diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-12-20 14:22:11 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-12-20 14:22:11 +0000 |
| commit | 0c872e02b2c822e3397515ec324051ff540f0cd5 (patch) | |
| tree | ce2fb6ce7030e4dad0f4118d21ab6453e5938cdd /doc | |
| parent | f7e05a6853b12f02911494c4b3fe53d9540d74fc (diff) | |
| download | gitlab-ce-0c872e02b2c822e3397515ec324051ff540f0cd5.tar.gz | |
Add latest changes from gitlab-org/gitlab@15-7-stable-eev15.7.0-rc42
Diffstat (limited to 'doc')
751 files changed, 18118 insertions, 10710 deletions
diff --git a/doc/.vale/gitlab/CurrentStatus.yml b/doc/.vale/gitlab/CurrentStatus.yml index 57b95dcf4ac..9972573b406 100644 --- a/doc/.vale/gitlab/CurrentStatus.yml +++ b/doc/.vale/gitlab/CurrentStatus.yml @@ -1,14 +1,13 @@ --- -# Suggestion: gitlab.CurrentStatus +# Warning: gitlab.CurrentStatus # # Checks for words that indicate a product or feature may change in the future. # # For a list of all options, see https://vale.sh/docs/topics/styles/ extends: existence message: "Remove '%s'. The documentation reflects the current state of the product." -level: suggestion +level: warning ignorecase: true link: https://docs.gitlab.com/ee/development/documentation/versions.html#promising-features-in-future-versions tokens: - currently - - yet diff --git a/doc/.vale/gitlab/ElementDescriptors.yml b/doc/.vale/gitlab/ElementDescriptors.yml index f3573f5ce65..f806f5878e1 100644 --- a/doc/.vale/gitlab/ElementDescriptors.yml +++ b/doc/.vale/gitlab/ElementDescriptors.yml @@ -1,13 +1,14 @@ --- -# Suggestion: gitlab.ElementDescriptors +# Warning: gitlab.ElementDescriptors # -# Suggests the correct way to describe elements in a form. +# Suggests the correct way to describe a button. # # For a list of all options, see https://vale.sh/docs/topics/styles/ -extends: substitution -message: "When describing elements, %s '%s'." -link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#language -level: suggestion +extends: existence +message: "If possible, rewrite to remove 'button'." +link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html#button +level: warning ignorecase: true -swap: - button: 'if possible, rewrite to remove' +scope: raw +raw: + - \*\*.+?\*\* button diff --git a/doc/.vale/gitlab/InclusionAbleism.yml b/doc/.vale/gitlab/InclusionAbleism.yml index 7419430c8a2..3ee356155cd 100644 --- a/doc/.vale/gitlab/InclusionAbleism.yml +++ b/doc/.vale/gitlab/InclusionAbleism.yml @@ -1,5 +1,5 @@ --- -# Suggestion: gitlab.InclusionAbleism +# Warning: gitlab.InclusionAbleism # # Suggests alternatives for words that foster ableism. # @@ -7,7 +7,7 @@ extends: substitution message: "Use inclusive language. Consider '%s' instead of '%s'." link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html -level: suggestion +level: warning ignorecase: true swap: sanity (?:check|test): check for completeness diff --git a/doc/.vale/gitlab/InclusionGender.yml b/doc/.vale/gitlab/InclusionGender.yml index ce8861b6a09..dcf8c1930ec 100644 --- a/doc/.vale/gitlab/InclusionGender.yml +++ b/doc/.vale/gitlab/InclusionGender.yml @@ -1,5 +1,5 @@ --- -# Suggestion: gitlab.InclusionGender +# Warning: gitlab.InclusionGender # # Suggests alternatives for words that are gender-specific. # @@ -7,7 +7,7 @@ extends: substitution message: "Use inclusive language. Consider '%s' instead of '%s'." link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html -level: suggestion +level: warning ignorecase: true swap: mankind: humanity, people diff --git a/doc/.vale/gitlab/Normal.yml b/doc/.vale/gitlab/Normal.yml new file mode 100644 index 00000000000..0ced9f3318e --- /dev/null +++ b/doc/.vale/gitlab/Normal.yml @@ -0,0 +1,14 @@ +--- +# Warning: gitlab.Normal +# +# Suggests alternatives for 'normal'. +# +# For a list of all options, see https://vale.sh/docs/topics/styles/ +extends: substitution +message: "Use '%s' instead of '%s'." +link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html +level: warning +ignorecase: true +swap: + normally: "usually' or 'typically" + normal: "typical' or 'standard" diff --git a/doc/.vale/gitlab/Simplicity.yml b/doc/.vale/gitlab/Simplicity.yml index 89169c1aa46..fd9b1c5e5a6 100644 --- a/doc/.vale/gitlab/Simplicity.yml +++ b/doc/.vale/gitlab/Simplicity.yml @@ -1,12 +1,12 @@ --- -# Suggestion: gitlab.Simplicity +# Warning: gitlab.Simplicity # # Checks for words implying ease of use, to avoid cognitive dissonance for frustrated users. # # For a list of all options, see https://vale.sh/docs/topics/styles/ extends: existence message: "Remove '%s'. Be precise instead of subjective." -level: suggestion +level: warning ignorecase: true link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html tokens: diff --git a/doc/.vale/gitlab/SubstitutionSuggestions.yml b/doc/.vale/gitlab/SubstitutionSuggestions.yml deleted file mode 100644 index 21cabf1e0a7..00000000000 --- a/doc/.vale/gitlab/SubstitutionSuggestions.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -# Suggestion: gitlab.SubstitutionSuggestions -# -# Suggests better options for frequently misused terms that are often - but not always - incorrect. -# SubstitutionWarning.yml and Substitutions.yml also exist. -# -# For a list of all options, see https://vale.sh/docs/topics/styles/ -extends: substitution -message: "Consider '%s' instead of '%s'." -link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html -level: suggestion -ignorecase: true -swap: - active user: "billable user" - active users: "billable users" - docs: "documentation" - e-mail: "email" - GLFM: "GitLab Flavored Markdown" - it is recommended: "you should" - we recommend: "you should" - navigate: go - OAuth2: "OAuth 2.0" - once that: "after that" - once the: "after the" - once you: "after you" - since: "because' or 'after" - sub-group: "subgroup" - sub-groups: "subgroups" - within: "in" diff --git a/doc/.vale/gitlab/SubstitutionWarning.yml b/doc/.vale/gitlab/SubstitutionWarning.yml index 8d6c18c1520..383ae38da16 100644 --- a/doc/.vale/gitlab/SubstitutionWarning.yml +++ b/doc/.vale/gitlab/SubstitutionWarning.yml @@ -1,16 +1,18 @@ --- # Warning: gitlab.SubstitutionWarning # -# Checks for misused terms or common shorthand that should never be used at GitLab, but can't be flagged as errors. -# Substitutions.yml and SubstitionSuggestions.yml also exist. +# Checks for misused terms or common shorthand that should not be used at GitLab, but can't be flagged as errors. +# Substitutions.yml also exists. # # For a list of all options, see https://vale.sh/docs/topics/styles/ extends: substitution -message: "If possible, use %s instead of '%s'." -link: https://about.gitlab.com/handbook/communication/#top-misused-terms +message: "Use '%s' instead of '%s' when possible." +link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html level: warning ignorecase: true swap: + active user: "billable user" + active users: "billable users" air(?:-| )?gapped: "offline environment" bullet: "list item" click: "select" @@ -19,11 +21,26 @@ swap: deselect: "clear" deselected: "cleared" distro: "distribution" + docs: "documentation" + e-mail: "email" + ex: "for example" file name: "filename" filesystem: "file system" - GFM: "'GitLab Flavored Markdown' or 'GitHub Flavored Markdown'" + GLFM: "GitLab Flavored Markdown" + GFM: "GitLab Flavored Markdown' or 'GitHub Flavored Markdown" info: "information" + it is recommended: "you should" n/a: "not applicable" + navigate to: "go to" + OAuth2: "OAuth 2.0" + once that: "after that" + once the: "after the" + once you: "after you" repo: "repository" + since: "because' or 'after" + sub-group: "subgroup" + sub-groups: "subgroups" timezone: "time zone" utilize: "use" + we recommend: "you should" + within: "in" diff --git a/doc/.vale/gitlab/Substitutions.yml b/doc/.vale/gitlab/Substitutions.yml index 92791486491..675abc6ef6d 100644 --- a/doc/.vale/gitlab/Substitutions.yml +++ b/doc/.vale/gitlab/Substitutions.yml @@ -2,7 +2,7 @@ # Error: gitlab.Substitutions # # Checks for misused terms that should never be used at GitLab. -# SubstitutionWarning.yml and SubstitionSuggestions.yml also exist. +# SubstitutionWarning.yml also exists. # # For a list of all options, see https://vale.sh/docs/topics/styles/ extends: substitution @@ -60,3 +60,4 @@ swap: reporter access: the Reporter role reporter permission: the Reporter role reporter permissions: the Reporter role + at least the Owner role: the Owner role diff --git a/doc/.vale/gitlab/Units.yml b/doc/.vale/gitlab/Units.yml index 4211fdee38b..e6263ad66b4 100644 --- a/doc/.vale/gitlab/Units.yml +++ b/doc/.vale/gitlab/Units.yml @@ -1,5 +1,5 @@ --- -# Suggestion: gitlab.Units +# Warning: gitlab.Units # # Recommends a space between a number and a unit of measure. # @@ -8,7 +8,7 @@ extends: existence message: "Add a space between the number and the unit in '%s'." link: 'https://docs.gitlab.com/ee/development/documentation/styleguide/' nonword: true -level: suggestion +level: warning ignorecase: true tokens: - \d+(?:B|kB|KiB|MB|MiB|GB|GiB|TB|TiB) diff --git a/doc/.vale/gitlab/Uppercase.yml b/doc/.vale/gitlab/Uppercase.yml index f53e1c72dcb..039ad7c5f03 100644 --- a/doc/.vale/gitlab/Uppercase.yml +++ b/doc/.vale/gitlab/Uppercase.yml @@ -39,6 +39,7 @@ exceptions: - CORE - CORS - CPU + - CRAN - CRIME - CRM - CRUD @@ -188,6 +189,7 @@ exceptions: - SAST - SATA - SBOM + - SBT - SCIM - SCM - SCP diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index 0fc7c9703ac..81b21f026f4 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -1,35 +1,53 @@ accessor accessors +ACLs +Adafruit +Airbnb +Airtable Akismet Alertmanager Algolia Alibaba +aliuid Aliyun allowlist allowlisted allowlisting allowlists AlmaLinux +AMIs anonymization anonymized Ansible Anthos -Apdex +Anycast +apdex +API +APIs Apparmor +Appetize approvers +Appsec architected architecting archiver Arel arity +Arkose +armhf +ARNs Artifactory Asana Asciidoctor asdf Assembla +Astro +async Atlassian auditability +auditable Auth0 +authenticator Authentiq Authy autocomplete @@ -40,18 +58,22 @@ autogenerated autoloaded autoloader autoloading +automatable autoscale autoscaled autoscaler +autoscalers autoscales autoscaling awardable awardables Axios Ayoa +AZs Azure B-tree backfilling +backfills backport backported backporting @@ -61,11 +83,16 @@ backtraced backtraces backtracing badging +balancer +balancer's Bamboo Bazel +bcrypt +Beamer Bhyve Bitbucket Bitnami +Bittrex blockquote blockquoted blockquotes @@ -73,6 +100,8 @@ blockquoting boolean booleans Bootsnap +bot +bot's Bottlerocket browsable bugfix @@ -88,15 +117,24 @@ bundlers burndown burnup burstable +CA cacheable Caddy +callout +callouts callstack callstacks +camelCase +camelCased Camo canonicalization canonicalized captcha +CAPTCHAs +Capybara Casdoor +CDNs +CE CentOS Ceph Certbot @@ -110,28 +148,57 @@ ChaosKube chatbot chatbots ChatOps +checksummable checksummed checksumming Chemlab +chipset +chipsets +CIDRs Citrix Citus +Civo +Cleartext +Clickhouse +CLIs +Clojars clonable Cloudwatch +clusterized +CMake +CMK +CMKs +CNAs +CNs Cobertura Codeception +Codecov +codenames Codepen +CodeSandbox Cognito +Coinbase +colocate colocated colocating +commit's +CommonMark compilable composable composables Conda +config +Configs Consul Contentful Corosync +corpuses Coursier +CPU +CPUs +CRAN cron +crond cronjob cronjobs crons @@ -142,31 +209,53 @@ crosslinking crosslinks Crossplane Crowdin +crypto +CSSComb CSV +CSVs +CTAs +CTEs +CUnit customappsso +CVEs +CWEs cybersecurity +CycloneDX Dangerfile +DAST +Databricks Datadog datasource datasources +datastore +datastores +datestamp datetime +DBeaver Debian +debloating +decodable Decompressor decryptable +dedupe deduplicate deduplicated deduplicates deduplicating deduplication +delegators deliverables +denormalization denormalize denormalized denormalizes denormalizing +dentry denylist denylisted denylisting denylists +Depesz deployer deployers deprovision @@ -180,14 +269,22 @@ deserialization deserialize deserializers deserializes +desugar +desugars +desynchronized +Dev DevOps Dhall +dialogs disambiguates discoverability dismissable Disqus Distroless Divio +DLE +DNs +Docker Dockerfile Dockerfiles Dockerize @@ -196,20 +293,37 @@ Dockerizing dogfood dogfooding dogfoods +DOMPurify dotenv +doublestar downvoted downvotes Dpl +dput Dreamweaver +DRIs +DSLs +Dynatrace Ecto +eden +EGit ElastiCache Elasticsearch +Eleventy enablement +Encrypt enqueued enqueues +enricher +enrichers enum enums +Enviroments +ESLint +ESXi ETag +ETags +Etsy Excon exfiltration ExifTool @@ -220,35 +334,49 @@ failovers failsafe Falco falsy +Fanout Fargate fastlane +Fastly Fastzip favicon favorited +ffaker Figma Filebeat +Filestore +Finicity +Finnhub Fio firewalled firewalling fixup +flamegraph +flamegraphs Flawfinder -Flowdock +Flickr Fluentd +Flutterwave Flycheck +focusable Forgerock formatters Fortinet +FQDNs +FreshBooks +frontend Fugit fuzzer +fuzzing Gantt Gbps Gemfile Gemnasium Gemojione Getter -getters Getters gettext +GIDs Git Gitaly Gitea @@ -258,25 +386,32 @@ gitlabsos Gitleaks Gitpod Gitter +GLab globals +globbing Gmail Godep +Golang Gollum Google goroutine goroutines Gosec +GPUs Gradle Grafana Grafonnet gravatar Grype +GUIs Gzip Hackathon Haml +HAProxy hardcode hardcoded hardcodes +HashiCorp Haswell heatmap heatmaps @@ -284,6 +419,8 @@ Helm Helmfile Heroku Herokuish +heuristical +hexdigest Hexo HipChat hostname @@ -292,21 +429,34 @@ hotfix hotfixed hotfixes hotfixing +hotspots http https +hyperparameter +hyperparameters +iCalendar +iCloud idempotence idmapper Iglu +IIFEs Immer inclusivity +inflector +inflectors Ingress initializer initializers +injective innersource innersourcing +inodes interdependencies interdependency interruptible +inviter +IPs +IPython irker issuables Istio @@ -317,16 +467,23 @@ JavaScript Jenkins Jenkinsfile Jira +Jitsu jq jQuery +JRuby +JSDoc jsdom Jsonnet +JUnit JupyterHub +JWT +JWTs Kaminari kanban kanbans kaniko Karma +KCachegrind Kerberos Keycloak keyset @@ -339,18 +496,24 @@ Klar Knative Kramdown Kroki +kubeconfig Kubecost kubectl Kubernetes Kubesec +Kucoin Kustomize +kwargs Laravel +LaunchDarkly ldapsearch Lefthook Leiningen libFuzzer +Libgcrypt Libravatar liveness +lockfile lockfiles Lodash Lograge @@ -361,14 +524,19 @@ lookahead lookaheads lookbehind lookbehinds +Lookbook lookups loopback +Lua Lucene +macOS +Mailchimp Maildir Mailgun Mailroom Makefile Makefiles +malloc Markdown markdownlint Marketo @@ -379,15 +547,19 @@ Mattermost mbox memoization memoize +memoizes memoized memoizing Memorystore mergeability mergeable metaprogramming +metric's +microformat Microsoft middleware middlewares +migratable migratus minikube MinIO @@ -401,6 +573,8 @@ mitigations mitmproxy mixin mixins +MLFlow +Mmap mockup mockups ModSecurity @@ -408,52 +582,82 @@ Monokai monorepo monorepos monospace +MRs +MSBuild multiline mutex nameserver nameservers namespace namespaced +namespace's namespaces namespacing namespacings Nanoc +NAT +navigations negatable Netlify +NGINX +ngrok +njsscan Nokogiri nosniff noteable noteables npm +NuGet nullability nullable Nurtch +NVMe nyc OAuth Octokit offboarded offboarding offboards +OIDs +OKRs Okta OmniAuth onboarding OpenID OpenShift Opsgenie +Opstrace +ORMs +OS +OSs outdent Overcommit Packagist +packfile +packfiles +Packwerk +paginator parallelization parallelizations +PascalCase +PascalCased +passthrough +passthroughs passwordless +parsable Patroni +PDFs performant PgBouncer +pgFormatter pgLoader +pgMustard Phabricator phaser phasers phpenv +PHPUnit +PIDs pipenv Pipfile Pipfiles @@ -464,25 +668,34 @@ polyfill polyfills pooler postfixed +Postgres postgres.ai PostgreSQL +Praefect's +prebuild +prebuilds precompile precompiled preconfigure preconfigured preconfigures +prefetch +prefetching prefill prefilled prefilling prefills preload +preloaded preloading preloads prepend prepended prepending prepends +prepopulate prepopulated +presentationals Prettifier Pritaly Priyanka @@ -499,15 +712,20 @@ pseudocode pseudonymization pseudonymized pseudonymizer +Pulumi Puma +Pumble +PyPI pytest Python Qualys queryable Quicktime Rackspace +railties Raspbian rbenv +rbspy rbtrace Rclone Rdoc @@ -523,7 +741,9 @@ rebase rebased rebases rebasing +rebinding reCAPTCHA +recoverability Redcarpet redirection redirections @@ -534,8 +754,13 @@ referer referers reflog reflogs +refname refspec refspecs +regexes +Rego +reimplementation +reimplemented reindex reindexed reindexes @@ -545,15 +770,19 @@ reinitializing relicensing remediations renderers +renderless replicables -rpcs repmgr repmgrd +reposts repurposing +requestee +requesters requeue requeued requeues requeuing +resolver Restlet resync resynced @@ -567,10 +796,15 @@ reusability reverified reverifies reverify +reviewee +RIs roadmap roadmaps +rock rollout rollouts +routable +RPCs RSpec rsync rsynced @@ -580,6 +814,8 @@ Rubinius Rubix RuboCop Rubular +RubyGems +Rugged ruleset rulesets runbook @@ -590,13 +826,22 @@ runtimes Salesforce sandboxing sanitization +SBOMs +SBT sbt scalers scatterplot scatterplots +schedulable Schemastore +scriptable scrollable +SDKs +segmentations +SELinux Semgrep +Sendbird +Sendinblue Sendmail Sentry serializer @@ -605,10 +850,13 @@ serializing serverless setuptools severities +SFCs sharded sharding +SHAs shfmt Shimo +Shippo Shopify Sidekiq Silverlight @@ -618,10 +866,16 @@ skippable skopeo Slack Slackbot +SLAs +SLIs Slony +SLOs smartcard smartcards +snake_case +snake_cased snapshotting +Snowplow Snyk Sobelow Solargraph @@ -631,9 +885,14 @@ Spamcheck spammable sparkline sparklines +Speedscope spidering Splunk SpotBugs +Squarespace +SREs +SSDs +SSGs Stackdriver Stackprof starrer @@ -649,6 +908,7 @@ subchart subcharts subcommand subcommands +subcomponent subfolder subfolders subgraph @@ -661,6 +921,8 @@ sublicense sublicensed sublicenses sublicensing +submodule +submodule's subnet subnets subnetting @@ -677,15 +939,20 @@ subtask subtasks subtest subtests +subtransaction +subtransactions subtree subtrees sudo +sunsetting supercookie supercookies +supergroup superset supersets supertype supertypes +SVGs swappiness swimlane swimlanes @@ -695,8 +962,12 @@ syscall syscalls syslog systemd +tablespace +tablespaces tanuki +taskscaler tcpdump +teardown templated Thanos Thoughtbot @@ -707,7 +978,9 @@ timeboxed timeboxes timeboxing timecop -tiptap +timelog +timelogs +Tiptap todos tokenizer Tokenizers @@ -717,6 +990,7 @@ toolchain toolchains toolkit toolkits +toolset tooltip tooltips transactionally @@ -725,6 +999,7 @@ transpiled transpiles transpiling Trello +Trendline triaged triages triaging @@ -733,9 +1008,13 @@ Truststore truthy Twilio Twitter +Typeform TypeScript +TZInfo Ubuntu Udemy +UI +UIDs unapplied unapprove unapproved @@ -749,6 +1028,8 @@ unassign unassigning unassigns unban +unbans +uncached uncheck unchecked unchecking @@ -757,6 +1038,7 @@ uncomment uncommented uncommenting uncordon +underperforming unencode unencoded unencoder @@ -782,6 +1064,7 @@ unoptimized unoptimizes unoptimizing unpatched +unpause unprioritized unprotect unprotected @@ -794,6 +1077,7 @@ unpublish unpublished unpublishes unpublishing +unpullable unpushed unreferenced unregister @@ -804,6 +1088,7 @@ unresolve unresolved unresolving unreviewed +unrevoke unsanitized unschedule unscoped @@ -835,10 +1120,14 @@ upstreams upvote upvoted upvotes +urgencies URIs +URL +UUIDs Vagrantfile validator validators +vCPUs vendored vendoring versionless @@ -846,13 +1135,18 @@ viewport viewports virtualized virtualizing +Vite +VMs +VPCs Vue Vuex +waitlist walkthrough walkthroughs WebdriverIO Webex webpack +WEBrick webserver Webservice websocket @@ -866,17 +1160,22 @@ wireframing Wireshark Wordpress Workato +workstream worktree worktrees Worldline Xcode Xeon +XPath +Yandex YouTrack ytt Yubico Zabbix +ZAProxy Zeitwerk Zendesk ZenTao zsh Zstandard +Zuora diff --git a/doc/administration/application_settings_cache.md b/doc/administration/application_settings_cache.md index d04056beb96..30019df44aa 100644 --- a/doc/administration/application_settings_cache.md +++ b/doc/administration/application_settings_cache.md @@ -20,7 +20,7 @@ To change the expiry value: ::Tabs -:::TabTitle Omnibus package +:::TabTitle Linux package (Omnibus) 1. Edit `/etc/gitlab/gitlab.rb`: @@ -36,7 +36,7 @@ To change the expiry value: gitlab-ctl restart ``` -:::TabTitle Source +:::TabTitle Self-compiled (Source) 1. Edit `config/gitlab.yml`: diff --git a/doc/administration/audit_event_streaming.md b/doc/administration/audit_event_streaming.md index 0af1af12a60..4d0e6518ebb 100644 --- a/doc/administration/audit_event_streaming.md +++ b/doc/administration/audit_event_streaming.md @@ -18,6 +18,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - Custom HTTP headers UI [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) in GitLab 15.3. [Feature flag `custom_headers_streaming_audit_events_ui`](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) removed. > - [Improved user experience](https://gitlab.com/gitlab-org/gitlab/-/issues/367963) in GitLab 15.3. > - User-specified verification token API support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360813) in GitLab 15.4. +> - Event type filters API [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344845) in GitLab 15.7. Users can set a streaming destination for a top-level group to receive all audit events about the group, its subgroups, and projects as structured JSON. @@ -143,6 +144,7 @@ query { id } } + eventTypeFilters } } } @@ -284,6 +286,63 @@ Users with the Owner role for a group can list streaming destinations and see th 1. On the main area, select the **Streams**. 1. View the verification token on the right side of each item. +## Event type filters + +> Event type filters API [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344845) in GitLab 15.7. + +When this feature is enabled for a group, you can use an API to permit users to filter streamed audit events per destination. +If the feature is enabled with no filters, the destination receives all audit events. + +A streaming destination that has an event type filter set has a **filtered** (**{filter}**) label. + +### Use the API to add an event type filter + +Prerequisites: + +- You must have the Owner role for the group. + +You can add a list of event type filters using the `auditEventsStreamingDestinationEventsAdd` query type: + +```graphql +mutation { + auditEventsStreamingDestinationEventsAdd(input: { + destinationId: "gid://gitlab/AuditEvents::ExternalAuditEventDestination/1", + eventTypeFilters: ["list of event type filters"]}){ + errors + eventTypeFilters + } +} +``` + +Event type filters are added if: + +- The returned `errors` object is empty. +- The API responds with `200 OK`. + +### Use the API to remove an event type filter + +Prerequisites: + +- You must have the Owner role for the group. + +You can remove a list of event type filters using the `auditEventsStreamingDestinationEventsRemove` query type: + +```graphql +mutation { + auditEventsStreamingDestinationEventsRemove(input: { + destinationId: "gid://gitlab/AuditEvents::ExternalAuditEventDestination/1", + eventTypeFilters: ["list of event type filters"] + }){ + errors + } +} +``` + +Event type filters are removed if: + +- The returned `errors` object is empty. +- The API responds with `200 OK`. + ## Payload schema > Documentation for an audit event streaming schema was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358149) in GitLab 15.3. diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index 0aa0d163972..1951ab5e2c7 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -12,6 +12,8 @@ You can use audit events to track, for example: - Who changed the permission level of a particular user for a GitLab project, and when. - Who added a new user or removed a user, and when. +Audit events are similar to the [log system](logs/index.md). + The GitLab API, database, and `audit_json.log` record many audit events. Some audit events are only available through [streaming audit events](audit_event_streaming.md). @@ -21,56 +23,148 @@ NOTE: You can't configure a retention policy for audit events, but epic [7917](https://gitlab.com/groups/gitlab-org/-/epics/7917) proposes to change this. -## List of events +## Time zones -There are two kinds of events logged: +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/242014) in GitLab 15.7, GitLab UI shows dates and times in the user's local time zone instead of UTC. -- Events scoped to the group or project, used by group and project managers - to look up who made a change. -- Instance events scoped to the whole GitLab instance, used by your Compliance team to - perform formal audits. +The time zone used for audit events depends on where you view them: -NOTE: -Some events are recorded and available only as [streaming audit events](audit_event_streaming.md). +- In GitLab UI, your local time zone (GitLab 15.7 and later) or UTC (GitLab 15.6 and earlier) is used. +- The [Audit Events API](../api/audit_events.md) returns dates and times in UTC by default, or the + [configured time zone](timezone.md) on a self-managed GitLab instance. +- In `audit_json.log`, UTC is used. +- In CSV exports, UTC is used. -### Impersonation data +## View audit events -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/536) in GitLab 13.0. +Depending on the events you want to view, at a minimum you must have: -When a user is being [impersonated](../user/admin_area/index.md#user-impersonation), their actions are logged as audit events as usual, with two additional details: +- For group audit events of all users in the group, the Owner role for the group. +- For project audit events of all users in the project, the Maintainer role for the project. +- For group and project audit events based on your own actions, the Developer role for the group or project. +- [Auditor users](auditor_users.md) can see group and project events for all users. -1. Usual audit events include information about the impersonating administrator. These audit events are visible in their - respective audit event pages depending on their type (group, project, or user). -1. Extra audit events are recorded for the start and stop of the administrator's impersonation session. These audit events - are visible in the: - - Instance audit events. - - Group audit events for all groups the user belongs to (GitLab 14.8 and later). For performance reasons, group audit - events are limited to the oldest 20 groups to which you belong. +You can view audit events scoped to a group or project. - +To view a group's audit events: -### Group events +1. Go to the group. +1. On the left sidebar, select **Security & Compliance > Audit Events**. -A user with: +Group events do not include project audit events. Group events can also be accessed using the +[Group Audit Events API](../api/audit_events.md#group-audit-events). Group event queries are limited to a maximum of 30 +days. -- Owner role (or above) can retrieve group audit events of all users. -- Developer or Maintainer role is limited to group audit events based on their individual actions. +To view a project's audit events: -Group events do not include project audit events. +1. Go to the project. +1. On the left sidebar, select **Security & Compliance > Audit Events**. -To view a group's audit events: +Project events can also be accessed using the [Project Audit Events API](../api/audit_events.md#project-audit-events). +Project event queries are limited to a maximum of 30 days. -1. Go to the group. -1. On the left sidebar, select **Security & Compliance > Audit Events**. +## View instance audit events **(PREMIUM SELF)** -From there, you can see the following actions: +You can view audit events from user actions across an entire GitLab instance. + +To view instance audit events: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Monitoring > Audit Events**. + +### Export to CSV + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1449) in GitLab 13.4. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/285441) in GitLab 13.7. + +You can export the current view (including filters) of your instance audit events as a CSV file. To export the instance +audit events to CSV: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Monitoring > Audit Events**. +1. Select the available search [filters](#filter-audit-events). +1. Select **Export as CSV**. + +The exported file: + +- Is sorted by `created_at` in ascending order. +- Is limited to a maximum of 100 000 events. The remaining records are truncated when this limit is reached. + +Data is encoded with: + +- Comma as the column delimiter. +- `"` to quote fields if necessary. +- New lines separate rows. + +The first row contains the headers, which are listed in the following table along with a description of the values: + +| Column | Description | +|:---------------------|:---------------------------------------------------| +| **ID** | Audit event `id`. | +| **Author ID** | ID of the author. | +| **Author Name** | Full name of the author. | +| **Entity ID** | ID of the scope. | +| **Entity Type** | Type of the scope (`Project`, `Group`, or `User`). | +| **Entity Path** | Path of the scope. | +| **Target ID** | ID of the target. | +| **Target Type** | Type of the target. | +| **Target Details** | Details of the target. | +| **Action** | Description of the action. | +| **IP Address** | IP address of the author who performed the action. | +| **Created At (UTC)** | Formatted as `YYYY-MM-DD HH:MM:SS`. | + +## View sign-in events **(FREE)** + +Successful sign-in events are the only audit events available at all tiers. To see successful sign-in events: + +1. Select your avatar. +1. Select **Edit profile > Authentication log**. + +After upgrading to a paid tier, you can see also see successful sign-in events on audit event pages. + +## Filter audit events + +From audit events pages, different filters are available depending on the page you're on. + +| Audit event page | Available filter | +|:-----------------|:-----------------------------------------------------------------------------------------------------------------------| +| Project | User (member of the project) who performed the action. | +| Group | User (member of the group) who performed the action. | +| Instance | Group, project, or user. | +| All | Date range buttons and pickers (maximum range of 31 days). Default is from the first day of the month to today's date. | + +## User impersonation + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/536) in GitLab 13.0. +> - Impersonation session events included in group audit events in GitLab 14.8. + +When a user is [impersonated](../user/admin_area/index.md#user-impersonation), their actions are logged as audit events +with additional details: + +- Audit events include information about the impersonating administrator. These audit events are visible in audit event + pages depending on the audit event type (group, project, or user). +- Extra audit events are recorded for the start and end of the administrator's impersonation session. These audit events + are visible as: + - Instance audit events. + - Group audit events for all groups the user belongs to. For performance reasons, group audit events are limited to + the oldest 20 groups you belong to. + + + +## Available audit events + +You can view different events depending on the version of GitLab you have. + +### Group events + +The following actions on groups generate group audit events: - Group name or path changed. - Group repository size limit changed. - Group created or deleted. - Group changed visibility. - User was added to group and with which [permissions](../user/permissions.md). -- User sign-in via [Group SAML](../user/group/saml_sso/index.md). +- User sign-in using [Group SAML](../user/group/saml_sso/index.md). - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8071) in GitLab 14.5, changes to the following [group SAML](../user/group/saml_sso/index.md) configuration: - Enabled status. @@ -85,8 +179,8 @@ From there, you can see the following actions: - Permissions changes of a user assigned to a group. - Removed user from group. - Project repository imported into group. -- [Project shared with group](../user/project/members/share_project_with_groups.md) - and with which [permissions](../user/permissions.md). +- [Project shared with group](../user/project/members/share_project_with_groups.md) and with which + [permissions](../user/permissions.md). - Removal of a previously shared group with a project. - LFS enabled or disabled. - Shared runners minutes limit changed. @@ -94,36 +188,36 @@ From there, you can see the following actions: - Request access enabled or disabled. - 2FA enforcement or grace period changed. - Roles allowed to create project changed. -- Group CI/CD variable added, removed, or protected status changed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30857) in GitLab 13.3. -- Compliance framework created, updated, or deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340649) in GitLab 14.5. -- Event streaming destination created, updated, or deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344664) in GitLab 14.6. -- Instance administrator started or stopped impersonation of a group member. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/300961) in GitLab 14.8. -- Group deploy token was successfully created, revoked, or deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. -- Failed attempt to create a group deploy token. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. -- [IP restrictions](../user/group/access_and_permissions.md#restrict-access-to-groups-by-ip-address) changed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358986) in GitLab 15.0. +- Group CI/CD variable added, removed, or protected status changed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30857) in GitLab 13.3. +- Compliance framework created, updated, or deleted. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340649) in GitLab 14.5. +- Event streaming destination created, updated, or deleted. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344664) in GitLab 14.6. +- Instance administrator started or stopped impersonation of a group member. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/300961) in GitLab 14.8. +- Group deploy token was successfully created, revoked, or deleted. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. +- Failed attempt to create a group deploy token. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) + in GitLab 14.9. +- [IP restrictions](../user/group/access_and_permissions.md#restrict-access-to-groups-by-ip-address) changed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358986) in GitLab 15.0. - Changes to push rules. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227629) in GitLab 15.0. -- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356152) in GitLab 15.1, changes to the following merge request approvals settings: +- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356152) in GitLab 15.1, changes to the following merge + request approvals settings: - Prevent approval by author. - Prevent approvals by users who add commits. - Prevent editing approval rules in projects and merge requests. - Require user password to approve. - Remove all approvals when commits are added to the source branch. -- Changes to streaming audit destination custom HTTP headers. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/366350) in GitLab 15.3. -- Group had a security policy project linked, changed, or unlinked. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/377877) in GitLab 15.6) - -Group events can also be accessed via the [Group Audit Events API](../api/audit_events.md#group-audit-events) +- Changes to streaming audit destination custom HTTP headers. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/366350) in GitLab 15.3. +- Group had a security policy project linked, changed, or unlinked. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/377877) in GitLab 15.6. ### Project events -A user with a Maintainer role (or above) can retrieve project audit events of all users. -A user with a Developer role is limited to project audit events based on their individual actions. - -To view a project's audit events: - -1. Go to the project. -1. On the left sidebar, select **Security & Compliance > Audit Events**. - -From there, you can see the following actions: +The following actions on projects generate project audit events: - Added or removed deploy keys - Project created, deleted, renamed, moved (transferred), changed path @@ -138,68 +232,87 @@ From there, you can see the following actions: - Added, removed, or updated protected branches - Release was added to a project - Release was updated -- Release was deleted ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/94793/) in GitLab 15.3) +- Release was deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/94793/) in GitLab 15.3. - Release milestone associations changed -- Permission to approve merge requests by committers was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9) +- Permission to approve merge requests by committers was updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9. - Permission to approve merge requests by committers was updated. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9. - Message for event [changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72623/diffs) in GitLab 14.6. - -- Permission to approve merge requests by authors was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9) -- Number of required approvals was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9) -- Added or removed users and groups from project approval groups ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213603) in GitLab 13.2) -- Project CI/CD variable added, removed, or protected status changed ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30857) in GitLab 13.4) -- Project access token was successfully created or revoked ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230007) in GitLab 13.9) -- Failed attempt to create or revoke a project access token ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230007) in GitLab 13.9) -- When default branch changes for a project ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/52339) in GitLab 13.9) -- Created, updated, or deleted DAST profiles, DAST scanner profiles, and DAST site profiles - ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217872) in GitLab 14.1) -- Changed a project's compliance framework ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/329362) in GitLab 14.1) -- User password required for approvals was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2) -- Permission to modify merge requests approval rules in merge requests was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2) -- New approvals requirement when new commits are added to an MR was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2) -- When [strategies for feature flags](../operations/feature_flags.md#feature-flag-strategies) are changed ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68408) in GitLab 14.3) -- Allowing force push to protected branch changed ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338873) in GitLab 14.3) -- Code owner approval requirement on merge requests targeting protected branch changed ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338873) in GitLab 14.3) -- Users and groups allowed to merge and push to protected branch added or removed ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338873) in GitLab 14.3) -- Project deploy token was successfully created, revoked or deleted ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353451) in GitLab 14.9) -- Failed attempt to create a project deploy token ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353451) in GitLab 14.9) -- When merge method is updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) -- Merged results pipelines enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) -- Merge trains enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) -- Automatically resolve merge request diff discussions enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) -- Show link to create or view a merge request when pushing from the command line enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) -- Delete source branch option by default enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) -- Squash commits when merging is updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) -- Pipelines must succeed enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) -- Skipped pipelines are considered successful enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) -- All discussions must be resolved enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) -- Commit message suggestion is updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) -- Status check is added, edited, or deleted ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0) -- Merge commit message template is updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0) -- Squash commit message template is updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0) -- Default description template for merge requests is updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0) -- Project was scheduled for deletion due to inactivity ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689) in GitLab 15.0) -- Project had a security policy project linked, changed, or unlinked ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/377877) in GitLab 15.6) - -Project events can also be accessed via the [Project Audit Events API](../api/audit_events.md#project-audit-events). - -Project event queries are limited to a maximum of 30 days. +- Permission to approve merge requests by authors was updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9. +- Number of required approvals was updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9. +- Added or removed users and groups from project approval groups. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213603) in GitLab 13.2. +- Project CI/CD variable added, removed, or protected status changed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30857) in GitLab 13.4. +- Project access token was successfully created or revoked. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230007) in GitLab 13.9. +- Failed attempt to create or revoke a project access token. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230007) in GitLab 13.9. +- When default branch changes for a project. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/52339) in GitLab 13.9. +- Created, updated, or deleted DAST profiles, DAST scanner profiles, and DAST site profiles. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217872) in GitLab 14.1. +- Changed a project's compliance framework. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/329362) in GitLab 14.1. +- User password required for approvals was updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2. +- Permission to modify merge requests approval rules in merge requests was updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2. +- New approvals requirement when new commits are added to an MR was updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2. +- When [strategies for feature flags](../operations/feature_flags.md#feature-flag-strategies) are changed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68408) in GitLab 14.3. +- Allowing force push to protected branch changed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338873) in GitLab 14.3. +- Code owner approval requirement on merge requests targeting protected branch changed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338873) in GitLab 14.3. +- Users and groups allowed to merge and push to protected branch added or removed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338873) in GitLab 14.3. +- Project deploy token was successfully created, revoked or deleted. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353451) in GitLab 14.9. +- Failed attempt to create a project deploy token. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353451) in GitLab 14.9. +- When merge method is updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. +- Merged results pipelines enabled or disabled. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. +- Merge trains enabled or disabled. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. +- Automatically resolve merge request diff discussions enabled or disabled. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. +- Show link to create or view a merge request when pushing from the command line enabled or disabled. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. +- Delete source branch option by default enabled or disabled. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. +- Squash commits when merging is updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. +- Pipelines must succeed enabled or disabled. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. +- Skipped pipelines are considered successful enabled or disabled. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. +- All discussions must be resolved enabled or disabled. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. +- Commit message suggestion is updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. +- Status check is added, edited, or deleted. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0. +- Merge commit message template is updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0. +- Squash commit message template is updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0. +- Default description template for merge requests is updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0. +- Project was scheduled for deletion due to inactivity. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689) in GitLab 15.0. +- Project had a security policy project linked, changed, or unlinked. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/377877) in GitLab 15.6. ### Instance events **(PREMIUM SELF)** -Server-wide audit events introduce the ability to observe user actions across -the entire instance of your GitLab server, making it easy to understand who -changed what and when for audit purposes. - -Instance events do not include group or project audit events. - -To view the server-wide audit events: - -1. On the top bar, select **Main menu > Admin**. -1. On the left sidebar, select **Monitoring > Audit Events**. - -The following user actions are recorded: +The following user actions on a GitLab instance generate instance audit events: - Sign-in events and the authentication type (such as standard, LDAP, or OmniAuth) - Failed sign-ins @@ -209,123 +322,47 @@ The following user actions are recorded: - Ask for password reset - Grant OAuth access - Started or stopped user impersonation -- Changed username ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7797) in GitLab 12.8) -- User was deleted ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8) -- User was added ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8) -- User requests access to an instance ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/298783) in GitLab 13.9) -- User was approved via Admin Area ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276250) in GitLab 13.6) -- User was rejected via Admin Area ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/298783) in GitLab 13.9) -- User was blocked via Admin Area ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8) -- User was blocked via API ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25872) in GitLab 12.9) -- Failed second-factor authentication attempt ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16826) in GitLab 13.5) -- A user's personal access token was successfully created or revoked ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276921) in GitLab 13.6) -- A failed attempt to create or revoke a user's personal access token ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276921) in GitLab 13.6) -- Administrator added or removed ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/323905) in GitLab 14.1) -- Removed SSH key ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220127) in GitLab 14.1) -- Added or removed GPG key ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220127) in GitLab 14.1) -- A user's two-factor authentication was disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238177) in GitLab 15.1) - -Instance events can also be accessed via the [Instance Audit Events API](../api/audit_events.md#instance-audit-events). - -### Sign-in events **(FREE)** - -Successful sign-in events are the only Audit Events available at all tiers. To see -successful sign-in events: - -1. Select your avatar. -1. Select **Edit profile > Authentication log**. - -After upgrading from GitLab Free to a paid tier, successful sign-in events are the only Audit -Events visible in Audit Events views until more events are logged. - -### "Deleted User" events - -Audit events can be created for a user after the user is deleted. The user name associated with the event is set to -"Deleted User" because the actual user name is unknowable. For example, if a deleted user's access to a project is -removed automatically due to expiration, the audit event is created for "Deleted User". We are [investigating](https://gitlab.com/gitlab-org/gitlab/-/issues/343933) -whether this is avoidable. - -### Missing events - -Some events are not tracked in audit events. See the following -epics for more detail on which events are not being tracked, and our progress -on adding these events into GitLab: - -- [Project settings and activity](https://gitlab.com/groups/gitlab-org/-/epics/474) -- [Group settings and activity](https://gitlab.com/groups/gitlab-org/-/epics/475) -- [Instance-level settings and activity](https://gitlab.com/groups/gitlab-org/-/epics/476) - -Don't see the event you want in any of the epics linked above? You can either: +- Changed username. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7797) in GitLab 12.8. +- User was deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8. +- User was added. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8. +- User requests access to an instance. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/298783) in GitLab 13.9. +- User was approved using the Admin Area. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276250) in GitLab 13.6. +- User was rejected using the Admin Area. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/298783) in GitLab 13.9. +- User was blocked using the Admin Area. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8. +- User was blocked using the API. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25872) in GitLab 12.9. +- Failed second-factor authentication attempt. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16826) in + GitLab 13.5. +- A user's personal access token was successfully created or revoked. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276921) in GitLab 13.6. +- A failed attempt to create or revoke a user's personal access token. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276921) in GitLab 13.6. +- Administrator added or removed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/323905) in GitLab 14.1. +- Removed SSH key. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220127) in GitLab 14.1. +- Added or removed GPG key. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220127) in GitLab 14.1. +- A user's two-factor authentication was disabled. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238177) in + GitLab 15.1. +- Enabled Admin Mode. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362101) in GitLab 15.7. + +Instance events can also be accessed using the [Instance Audit Events API](../api/audit_events.md#instance-audit-events). + +## "Deleted User" events + +Audit events created after users are deleted are created for "Deleted User". For example, if a deleted user's access to +a project is removed automatically due to expiration. + +Issue [343933](https://gitlab.com/gitlab-org/gitlab/-/issues/343933) proposes to change this behavior. + +## Unsupported events + +Some events are not tracked in audit events. The following epics propose support for more events: + +- [Project settings and activity](https://gitlab.com/groups/gitlab-org/-/epics/474). +- [Group settings and activity](https://gitlab.com/groups/gitlab-org/-/epics/475). +- [Instance-level settings and activity](https://gitlab.com/groups/gitlab-org/-/epics/476). + +If you don't see the event you want in any of the epics, you can either: - Use the **Audit Event Proposal** issue template to [create an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Audit%20Event%20Proposal) to request it. - [Add it yourself](../development/audit_event_guide/index.md). - -### Removed events - -> - Repositories push events was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/337993) in GitLab 14.3. -> - Repositories push events was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/337993) in GitLab 15.0. - -The repositories push events feature was: - -- [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/337993) in GitLab 14.3. -- [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/337993) in GitLab 15.0. - -## Search - -The search filters you can see depends on which audit level you are at. - -| Filter | Available options | -| ------ | ----------------- | -| Scope (Project level) | A specific user who performed the action. | -| Scope (Group level) | A specific user (in a group) who performed the action. | -| Scope (Instance level) | A specific group, project, or user that the action was scoped to. | -| Date range | Either via the date range buttons or pickers (maximum range of 31 days). Default is from the first day of the month to today's date. | - - - -## Export to CSV **(PREMIUM SELF)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1449) in GitLab 13.4. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/285441) in GitLab 13.7. - -Export to CSV allows customers to export the current filter view of your audit events as a -CSV file, which stores tabular data in plain text. The data provides a comprehensive view with respect to -audit events. - -To export the audit events to CSV: - -1. On the top bar, select **Main menu > Admin**. -1. On the left sidebar, select **Monitoring > Audit Events**. -1. Select the available search [filters](#search). -1. Select **Export as CSV**. - -### Sort - -Exported events are always sorted by `created_at` in ascending order. - -### Format - -Data is encoded with a comma as the column delimiter, with `"` used to quote fields if needed, and newlines to separate rows. -The first row contains the headers, which are listed in the following table along with a description of the values: - -| Column | Description | -|---------|-------------| -| ID | Audit event `id` | -| Author ID | ID of the author | -| Author Name | Full name of the author | -| Entity ID | ID of the scope | -| Entity Type | Type of the scope (`Project`/`Group`/`User`) | -| Entity Path | Path of the scope | -| Target ID | ID of the target | -| Target Type | Type of the target | -| Target Details | Details of the target | -| Action | Description of the action | -| IP Address | IP address of the author who performed the action | -| Created At (UTC) | Formatted as `YYYY-MM-DD HH:MM:SS` | - -### Limitation - -The audit events CSV file is limited to a maximum of `100,000` events. -The remaining records are truncated when this limit is reached. diff --git a/doc/administration/auth/ldap/index.md b/doc/administration/auth/ldap/index.md index 6243f3da2d2..2cb9bac7af9 100644 --- a/doc/administration/auth/ldap/index.md +++ b/doc/administration/auth/ldap/index.md @@ -87,9 +87,10 @@ with `start_tls` and `ssl` was replaced with `simple_tls`. LDAP users must have a set email address, regardless of whether or not it's used to sign in. -### Example Omnibus GitLab configuration +### Example Linux package (Omnibus) configuration -This example shows configuration for Omnibus GitLab instances: +This example shows a sample configuration for a GitLab instance that +was installed by using the Linux package (Omnibus): ```ruby gitlab_rails['ldap_enabled'] = true @@ -135,9 +136,14 @@ gitlab_rails['ldap_servers'] = { } ``` -### Example source install configuration +### Example Helm chart (Kubernetes) configuration -This example shows configuration for source install instances: +View [how to configure LDAP for a GitLab instance that was installed by using the Helm chart](https://docs.gitlab.com/charts/charts/globals.html#ldap). + +### Example self-compiled (source) configuration + +This example shows a sample configuration for a GitLab instance that +was installed by using the self-compiled source: ```yaml production: @@ -358,7 +364,9 @@ This can lead to several confusing issues such as creating links or namespaces w GitLab can automatically lowercase usernames provided by the LDAP server by enabling the configuration option `lowercase_usernames`. By default, this configuration option is `false`. -**Omnibus configuration** +::Tabs + +:::TabTitle Linux package (Omnibus) 1. Edit `/etc/gitlab/gitlab.rb`: @@ -373,7 +381,7 @@ the configuration option `lowercase_usernames`. By default, this configuration o 1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -**Source configuration** +:::TabTitle Self-compiled (source) 1. Edit `config/gitlab.yaml`: @@ -388,6 +396,8 @@ the configuration option `lowercase_usernames`. By default, this configuration o 1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. +::EndTabs + ### Disable LDAP web sign in It can be useful to prevent using LDAP credentials through the web UI when @@ -398,7 +408,9 @@ checks like custom 2FA. When LDAP web sign in is disabled, users don't see an **LDAP** tab on the sign-in page. This does not disable using LDAP credentials for Git access. -**Omnibus configuration** +::Tabs + +:::TabTitle Linux package (Omnibus) 1. Edit `/etc/gitlab/gitlab.rb`: @@ -408,7 +420,30 @@ This does not disable using LDAP credentials for Git access. 1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -**Source configuration** +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: + + ```yaml + global: + appConfig: + ldap: + preventSignin: true + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Self-compiled (source) 1. Edit `config/gitlab.yaml`: @@ -420,6 +455,8 @@ This does not disable using LDAP credentials for Git access. 1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. +::EndTabs + ### Use encrypted credentials Instead of having the LDAP integration credentials stored in plaintext in the configuration files, you can optionally @@ -439,7 +476,9 @@ The supported configuration items for the encrypted file are: The encrypted contents can be configured with the [LDAP secret edit Rake command](../../raketasks/ldap.md#edit-secret). -**Omnibus configuration** +::Tabs + +:::TabTitle Linux package (Omnibus) If initially your LDAP configuration looked like: @@ -473,7 +512,7 @@ If initially your LDAP configuration looked like: 1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -**Source configuration** +:::TabTitle Self-compiled (source) If initially your LDAP configuration looked like: @@ -507,6 +546,24 @@ If initially your LDAP configuration looked like: 1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. +::EndTabs + +## Updating LDAP DN and email + +When an LDAP server creates a user in GitLab, the user's LDAP distinguished name (DN) is linked to their GitLab account +as an identifier. + +When a user tries to sign in with LDAP, GitLab tries to find the user using the DN saved on that user's account. + +- If GitLab finds the user by the DN and the user's email address: + - Matches the GitLab account's email address, GitLab does not take any further action. + - Has changed, GitLab updates its record of the user's email to match the one in LDAP. +- If GitLab cannot find a user by their DN, it tries to find the user by their email. If GitLab: + - Finds the user by their email, GitLab updates the DN stored in the user's GitLab account. Both values now + match the information stored in LDAP. + - Cannot find the user by their email address (both the DN **and** the email address have changed), see + [User DN and email have changed](ldap-troubleshooting.md#user-dn-and-email-have-changed). + ## Disable anonymous LDAP authentication GitLab doesn't support TLS client authentication. Complete these steps on your LDAP server. @@ -543,7 +600,7 @@ Updating user email addresses must be done on the LDAP server that manages the u The updated user's previous email address becomes the secondary email address to preserve that user's commit history. -You can find more details on the expected behavior of user updates in our [LDAP troubleshooting section](ldap-troubleshooting.md#user-dn-orand-email-have-changed). +You can find more details on the expected behavior of user updates in our [LDAP troubleshooting section](ldap-troubleshooting.md#user-dn-and-email-have-changed). ## Google Secure LDAP diff --git a/doc/administration/auth/ldap/ldap-troubleshooting.md b/doc/administration/auth/ldap/ldap-troubleshooting.md index 499c3c64af7..21ec4b293d4 100644 --- a/doc/administration/auth/ldap/ldap-troubleshooting.md +++ b/doc/administration/auth/ldap/ldap-troubleshooting.md @@ -541,7 +541,7 @@ Usually this is not a cause for concern. If you think a particular user should already exist in GitLab, but you're seeing this entry, it could be due to a mismatched DN stored in GitLab. See -[User DN and/or email have changed](#user-dn-orand-email-have-changed) to update the user's LDAP identity. +[User DN and email have changed](#user-dn-and-email-have-changed) to update the user's LDAP identity. ```shell User with DN `uid=john0,ou=people,dc=example,dc=com` should have access @@ -624,26 +624,16 @@ does not do this: 1. Wait until LDAP group synchronization has finished running. 1. Remove the user from the LDAP group. -### User DN or/and email have changed +### User DN and email have changed -When an LDAP user is created in GitLab, their LDAP DN is stored for later reference. +If both the primary email **and** the DN change in LDAP, GitLab cannot identify the correct LDAP record of a user. As a +result, GitLab blocks that user. So that GitLab can find the LDAP record, update the user's existing GitLab profile with +at least either: -If GitLab cannot find a user by their DN, it falls back -to finding the user by their email. If the lookup is successful, GitLab -updates the stored DN to the new value so both values now match what's in -LDAP. +- The new primary email. +- DN values. -If the email has changed and the DN has not, GitLab finds the user with -the DN and updates its own record of the user's email to match the one in LDAP. - -However, if the primary email _and_ the DN change in LDAP, then GitLab -has no way of identifying the correct LDAP record of the user and, as a -result, the user is blocked. To rectify this, the user's existing -profile must be updated with at least one of the new values (primary -email or DN) so the LDAP record can be found. - -The following script updates the emails for all provided users so they -aren't blocked or unable to access their accounts. +The following script updates the emails for all provided users so they aren't blocked or unable to access their accounts. NOTE: The following script requires that any new accounts with the new @@ -669,7 +659,7 @@ end You can then [run a UserSync](#sync-all-users) **(PREMIUM SELF)** to sync the latest DN for each of these users. -## Could not authenticate you from ldapmain because "Unknown provider" +## `Could not authenticate you from Ldapmain because "Unknown provider"` You can receive the following error when authenticating with an LDAP server: @@ -831,6 +821,38 @@ ldapsearch -D "cn=admin,dc=ldap-testing,dc=example,dc=com" \ The `bind_dn`, `password`, `port`, `host`, and `base` are all identical to what's configured in the `gitlab.rb`. +#### Use ldapsearch with `start_tls` encryption + +The previous example performs an LDAP test in plaintext to port 389. If you are using [`start_tls` encryption](index.md#basic-configuration-settings), in +the `ldapsearch` command include: + +- The `-Z` flag. +- The FQDN of the LDAP server. + +You must include these because, during TLS negotiation, the FQDN of the LDAP server is evaluated against its certificate: + +```shell +ldapsearch -D "cn=admin,dc=ldap-testing,dc=example,dc=com" \ + -w Password1 \ + -p 389 \ + -h "testing.ldap.com" \ + -b "dc=ldap-testing,dc=example,dc=com" -Z +``` + +#### Use ldapsearch with `simple_tls` encryption + +If you are using [`simple_tls` encryption](index.md#basic-configuration-settings) (usually on port 636), include the following in the `ldapsearch` command: + +- The LDAP server FQDN with the `-H` flag and the port. +- The full constructed URI. + +```shell +ldapsearch -D "cn=admin,dc=ldap-testing,dc=example,dc=com" \ + -w Password1 \ + -H "ldaps://testing.ldap.com:636" \ + -b "dc=ldap-testing,dc=example,dc=com" +``` + For more information, see the [official `ldapsearch` documentation](https://linux.die.net/man/1/ldapsearch). ### Using **AdFind** (Windows) diff --git a/doc/administration/auth/ldap/ldap_synchronization.md b/doc/administration/auth/ldap/ldap_synchronization.md index 02b04861844..5c8c7f7baf1 100644 --- a/doc/administration/auth/ldap/ldap_synchronization.md +++ b/doc/administration/auth/ldap/ldap_synchronization.md @@ -55,7 +55,9 @@ use a [crontab generator](http://www.crontabgenerator.com). The example below shows how to set LDAP user sync to run once every 12 hours at the top of the hour. -**Omnibus installations** +::Tabs + +:::TabTitle Linux package (Omnibus) 1. Edit `/etc/gitlab/gitlab.rb`: @@ -63,19 +65,77 @@ sync to run once every 12 hours at the top of the hour. gitlab_rails['ldap_sync_worker_cron'] = "0 */12 * * *" ``` -1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: + + ```yaml + global: + appConfig: + cron_jobs: + ldap_sync_worker: + cron: "0 */12 * * *" + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['ldap_sync_worker_cron'] = "0 */12 * * *" + ``` + +1. Save the file and restart GitLab: -**Source installations** + ```shell + docker compose up -d + ``` -1. Edit `config/gitlab.yaml`: +:::TabTitle Self-compiled (source) + +1. Edit `/home/git/gitlab/config/gitlab.yml`: ```yaml - cron_jobs: - ldap_sync_worker_cron: - "0 */12 * * *" + production: &base + ee_cron_jobs: + ldap_sync_worker: + cron: "0 */12 * * *" ``` -1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +::EndTabs ## Group sync @@ -91,41 +151,103 @@ GitLab group membership to be automatically updated based on LDAP group members. The `group_base` configuration should be a base LDAP 'container', such as an 'organization' or 'organizational unit', that contains LDAP groups that should be available to GitLab. For example, `group_base` could be -`ou=groups,dc=example,dc=com`. In the configuration file it looks like the +`ou=groups,dc=example,dc=com`. In the configuration file, it looks like the following. -**Omnibus configuration** +::Tabs + +:::TabTitle Linux package (Omnibus) 1. Edit `/etc/gitlab/gitlab.rb`: ```ruby gitlab_rails['ldap_servers'] = { - 'main' => { - # snip... - 'group_base' => 'ou=groups,dc=example,dc=com', - } + 'main' => { + 'group_base' => 'ou=groups,dc=example,dc=com', + } } ``` -1. [Apply your changes to GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` -**Source configuration** +1. Edit `gitlab_values.yaml`: + + ```yaml + global: + appConfig: + ldap: + servers: + main: + group_base: ou=groups,dc=example,dc=com + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['ldap_servers'] = { + 'main' => { + 'group_base' => 'ou=groups,dc=example,dc=com', + } + } + ``` + +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) 1. Edit `/home/git/gitlab/config/gitlab.yml`: ```yaml - production: + production: &base ldap: servers: main: - # snip... group_base: ou=groups,dc=example,dc=com ``` -1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +::EndTabs To take advantage of group sync, group Owners or users with the [Maintainer role](../../../user/permissions.md) must -[create one or more LDAP group links](#add-group-links). +[create one or more LDAP group links](../../../user/group/access_and_permissions.md#manage-group-memberships-via-ldap). ### Add group links @@ -146,37 +268,101 @@ as opposed to the full DN. Additionally, if an LDAP user has an `admin` role, but is not a member of the `admin_group` group, GitLab revokes their `admin` role when syncing. -**Omnibus configuration** +::Tabs + +:::TabTitle Linux package (Omnibus) 1. Edit `/etc/gitlab/gitlab.rb`: ```ruby gitlab_rails['ldap_servers'] = { - 'main' => { - # snip... - 'group_base' => 'ou=groups,dc=example,dc=com', - 'admin_group' => 'my_admin_group', - } + 'main' => { + 'group_base' => 'ou=groups,dc=example,dc=com', + 'admin_group' => 'my_admin_group', + } } ``` -1. [Apply your changes to GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: + + ```yaml + global: + appConfig: + ldap: + servers: + main: + group_base: ou=groups,dc=example,dc=com + admin_group: my_admin_group + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` -**Source configuration** +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['ldap_servers'] = { + 'main' => { + 'group_base' => 'ou=groups,dc=example,dc=com', + 'admin_group' => 'my_admin_group', + } + } + ``` + +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) 1. Edit `/home/git/gitlab/config/gitlab.yml`: ```yaml - production: + production: &base ldap: servers: main: - # snip... group_base: ou=groups,dc=example,dc=com admin_group: my_admin_group ``` -1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +::EndTabs ### Global group memberships lock @@ -218,7 +404,9 @@ You can manually configure LDAP group sync times by setting the following configuration values. The example below shows how to set group sync to run once every two hours at the top of the hour. -**Omnibus installations** +::Tabs + +:::TabTitle Linux package (Omnibus) 1. Edit `/etc/gitlab/gitlab.rb`: @@ -226,56 +414,176 @@ sync to run once every two hours at the top of the hour. gitlab_rails['ldap_group_sync_worker_cron'] = "0 */2 * * * *" ``` -1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: + + ```yaml + global: + appConfig: + cron_jobs: + ldap_group_sync_worker: + cron: "*/30 * * * *" + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['ldap_group_sync_worker_cron'] = "0 */2 * * * *" + ``` + +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` -**Source installations** +:::TabTitle Self-compiled (source) -1. Edit `config/gitlab.yaml`: +1. Edit `/home/git/gitlab/config/gitlab.yml`: ```yaml - cron_jobs: - ldap_group_sync_worker_cron: - "*/30 * * * *" + production: &base + ee_cron_jobs: + ldap_group_sync_worker: + cron: "*/30 * * * *" ``` -1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +::EndTabs ### External groups Using the `external_groups` setting allows you to mark all users belonging -to these groups as [external users](../../../user/permissions.md#external-users). +to these groups as [external users](../../../user/admin_area/external_users.md). Group membership is checked periodically through the `LdapGroupSync` background task. -**Omnibus configuration** +::Tabs + +:::TabTitle Linux package (Omnibus) 1. Edit `/etc/gitlab/gitlab.rb`: ```ruby gitlab_rails['ldap_servers'] = { - 'main' => { - # snip... - 'external_groups' => ['interns', 'contractors'], - } + 'main' => { + 'external_groups' => ['interns', 'contractors'], + } } ``` -1. [Apply your changes to GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` -**Source configuration** +:::TabTitle Helm chart (Kubernetes) -1. Edit `config/gitlab.yaml`: +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: ```yaml - production: + global: + appConfig: + ldap: + servers: + main: + external_groups: ['interns', 'contractors'] + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['ldap_servers'] = { + 'main' => { + 'external_groups' => ['interns', 'contractors'], + } + } + ``` + +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: &base ldap: servers: main: - # snip... external_groups: ['interns', 'contractors'] ``` -1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +::EndTabs ### Group sync technical details diff --git a/doc/administration/clusters/kas.md b/doc/administration/clusters/kas.md index d7e1c9af1de..79dd69183a6 100644 --- a/doc/administration/clusters/kas.md +++ b/doc/administration/clusters/kas.md @@ -69,7 +69,7 @@ To enable the agent server on multiple nodes: - `gitlab_kas['api_secret_key']` is the shared secret used for authentication between KAS and GitLab. This value must be Base64-encoded and exactly 32 bytes long. - `gitlab_kas['private_api_secret_key']` is the shared secret used for authentication between different KAS instances. This value must be Base64-encoded and exactly 32 bytes long. -1. For each application node, follow the steps in: [Use an external installation](../clusters/kas.md#use-an-external-installation). +1. For each application node, follow the steps in [Use an external installation](../clusters/kas.md#use-an-external-installation). If the agent server is enabled on the application node, do not include `gitlab_kas['enable'] = false` in the configuration for that node. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). ### For GitLab Helm Chart diff --git a/doc/administration/configure.md b/doc/administration/configure.md index 91fec753f7e..bf618345a94 100644 --- a/doc/administration/configure.md +++ b/doc/administration/configure.md @@ -16,7 +16,7 @@ Customize and configure your self-managed GitLab installation. Here are some qui - [Packages](packages/index.md) The following tables are intended to guide you to choose the right combination of capabilities based on your requirements. It is common to want the most -available, quickly recoverable, highly performant and fully data resilient solution. However, there are tradeoffs. +available, quickly recoverable, highly performant, and fully data resilient solution. However, there are tradeoffs. The tables lists features on the left and provides their capabilities to the right along with known trade-offs. @@ -24,9 +24,9 @@ The tables lists features on the left and provides their capabilities to the rig | | Availability | Recoverability | Data Resiliency | Performance | Risks/Trade-offs| |-|--------------|----------------|-----------------|-------------|-----------------| -|Gitaly Cluster | Very high - tolerant of node failures | RTO for a single node of 10s with no manual intervention | Data is stored on multiple nodes | Good - While writes may take slightly longer due to voting, read distribution improves read speeds | **Trade-off** - Slight decrease in write speed for redundant, strongly-consistent storage solution. **Risks** - [Does not currently support snapshot backups](gitaly/index.md#snapshot-backup-and-recovery-limitations), GitLab backup task can be slow for large data sets | +|Gitaly Cluster | Very high - tolerant of node failures | RTO for a single node of 10 s with no manual intervention | Data is stored on multiple nodes | Good - While writes may take slightly longer due to voting, read distribution improves read speeds | **Trade-off** - Slight decrease in write speed for redundant, strongly-consistent storage solution. **Risks** - [Does not support snapshot backups](gitaly/index.md#snapshot-backup-and-recovery-limitations), GitLab backup task can be slow for large data sets | |Gitaly Shards | Single storage location is a single point of failure | Would need to restore only shards which failed | Single point of failure | Good - can allocate repositories to shards to spread load | **Trade-off** - Need to manually configure repositories into different shards to balance loads / storage space **Risks** - Single point of failure relies on recovery process when single-node failure occurs | -|Gitaly + NFS | Single storage location is a single point of failure | Single node failure requires restoration from backup | Single point of failure | Average - NFS is not ideally suited to large quantities of small reads / writes which can have a detrimental impact on performance | **Trade-off** - Easy and familiar administration though NFS is not ideally suited to Git demands **Risks** - Many instances of NFS compatibility issues which provide very poor customer experiences | +|Gitaly + NFS | Single storage location is a single point of failure | Single node failure requires restoration from backup | Single point of failure | Average - NFS is not ideally suited to large quantities of small reads / writes which can have a detrimental impact on performance | **Trade-off** - Familiar administration though NFS is not ideally suited to Git demands **Risks** - Many instances of NFS compatibility issues which provide very poor customer experiences | ## Geo Capabilities @@ -34,7 +34,7 @@ If your availability needs to span multiple zones or multiple locations, read ab | | Availability | Recoverability | Data Resiliency | Performance | Risks/Trade-offs| |-|--------------|----------------|-----------------|-------------|-----------------| -|Geo| Depends on the architecture of the Geo site. It is possible to deploy secondaries in single and multiple node configurations. | Eventually consistent. Recovery point depends on replication lag, which depends on a number of factors such as network speeds. Geo supports failover from a primary to secondary site using manual commands that are scriptable. | Geo currently replicates 100% of planned data types and verifies 50%. See [limitations table](geo/replication/datatypes.md#limitations-on-replicationverification) for more detail. | Improves read/clone times for users of a secondary. | Geo is not intended to replace other backup/restore solutions. Because of replication lag and the possibility of replicating bad data from a primary, we recommend that customers also take regular backups of their primary site and test the restore process. | +|Geo| Depends on the architecture of the Geo site. It is possible to deploy secondaries in single and multiple node configurations. | Eventually consistent. Recovery point depends on replication lag, which depends on a number of factors such as network speeds. Geo supports failover from a primary to secondary site using manual commands that are scriptable. | Geo replicates 100% of planned data types and verifies 50%. See [limitations table](geo/replication/datatypes.md#limitations-on-replicationverification) for more detail. | Improves read/clone times for users of a secondary. | Geo is not intended to replace other backup/restore solutions. Because of replication lag and the possibility of replicating bad data from a primary, customers should also take regular backups of their primary site and test the restore process. | ## Scenarios for failure modes and available mitigation paths diff --git a/doc/administration/external_pipeline_validation.md b/doc/administration/external_pipeline_validation.md index e6f825f7cb8..2dec3857f75 100644 --- a/doc/administration/external_pipeline_validation.md +++ b/doc/administration/external_pipeline_validation.md @@ -9,9 +9,6 @@ type: reference, howto You can use an external service to validate a pipeline before it's created. -WARNING: -This is an experimental feature and subject to change without notice. - GitLab sends a POST request to the external service URL with the pipeline data as payload. The response code from the external service determines if GitLab should accept or reject the pipeline. If the response is: diff --git a/doc/administration/feature_flags.md b/doc/administration/feature_flags.md index f2a40b60536..f7237b167e5 100644 --- a/doc/administration/feature_flags.md +++ b/doc/administration/feature_flags.md @@ -45,8 +45,7 @@ Features that are disabled by default may change or be removed without notice in Data corruption, stability degradation, performance degradation, or security issues might occur if you enable a feature that's disabled by default. Problems caused by using a default -disabled feature aren't covered by GitLab support, unless you were directed by GitLab -to enable the feature. +disabled feature aren't covered by GitLab Support. Security issues found in features that are disabled by default are patched in regular releases and do not follow our regular [maintenance policy](../policy/maintenance.md#security-releases) diff --git a/doc/administration/file_hooks.md b/doc/administration/file_hooks.md index f8b55a7c680..b74ee22d584 100644 --- a/doc/administration/file_hooks.md +++ b/doc/administration/file_hooks.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Source Code +stage: Manage +group: Integrations info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" type: reference --- diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index dfa8d09e6ef..a78350d9dba 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -174,7 +174,7 @@ Use `gitlab-ctl geo promote` instead. #### Promoting a **secondary** site with multiple nodes running GitLab 14.5 and later -1. SSH to every Sidekiq, PostgresSQL, and Gitaly node in the **secondary** site and run one of the following commands: +1. SSH to every Sidekiq, PostgreSQL, and Gitaly node in the **secondary** site and run one of the following commands: - To promote the node on the secondary site to primary: @@ -252,7 +252,7 @@ do this manually. #### Promoting a **secondary** site with a Patroni standby cluster running GitLab 14.5 and later -1. SSH to every Sidekiq, PostgresSQL, and Gitaly node in the **secondary** site and run one of the following commands: +1. SSH to every Sidekiq, PostgreSQL, and Gitaly node in the **secondary** site and run one of the following commands: - To promote the secondary site to primary: @@ -364,7 +364,7 @@ with the **secondary** site: sudo -u $PG_SUPERUSER $PG_CTL_BINARY -D $PG_DATA_DIRECTORY promote ``` -1. SSH to every Sidekiq, PostgresSQL, and Gitaly node in the **secondary** site and run one of the following commands: +1. SSH to every Sidekiq, PostgreSQL, and Gitaly node in the **secondary** site and run one of the following commands: - To promote the secondary site to primary: diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index 80707afacca..e9eb154d398 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -217,7 +217,7 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary - All replication meters reach 100% replicated, 0% failures. - All verification meters reach 100% verified, 0% failures. - - Database replication lag is 0ms. + - Database replication lag is 0 ms. - The Geo log cursor is up to date (0 events behind). 1. On the **secondary** site: diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md index 8c18aaa944d..cef0101dd40 100644 --- a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md @@ -72,7 +72,7 @@ On the **secondary** site: 1. On the left sidebar, select **Geo > Sites** to see its status. Replicated objects (shown in green) should be close to 100%, and there should be no failures (shown in red). If a large proportion of - objects aren't yet replicated (shown in gray), consider giving the site more + objects aren't replicated (shown in gray), consider giving the site more time to complete.  @@ -162,7 +162,7 @@ follow these steps to avoid unnecessary data loss: - All replication meters reach 100% replicated, 0% failures. - All verification meters reach 100% verified, 0% failures. - - Database replication lag is 0ms. + - Database replication lag is 0 ms. - The Geo log cursor is up to date (0 events behind). 1. On the **secondary** site: @@ -215,7 +215,7 @@ follow these steps to avoid unnecessary data loss: `initctl stop gitlab-runsvvdir && echo 'manual' > /etc/init/gitlab-runsvdir.override && initctl reload-configuration`. - If you do not have SSH access to the **primary** site, take the machine offline and - prevent it from rebooting. Since there are many ways you may prefer to accomplish + prevent it from rebooting. As there are many ways you may prefer to accomplish this, we avoid a single recommendation. You may have to: - Reconfigure the load balancers. @@ -228,7 +228,7 @@ follow these steps to avoid unnecessary data loss: ### Promoting the **secondary** site running GitLab 14.5 and later -1. SSH to every Sidekiq, PostgresSQL, and Gitaly node in the **secondary** site and run one of the following commands: +1. SSH to every Sidekiq, PostgreSQL, and Gitaly node in the **secondary** site and run one of the following commands: - To promote the secondary site to primary: @@ -276,7 +276,7 @@ WARNING: If you encounter an `ActiveRecord::RecordInvalid: Validation failed: Name has already been taken` error during this process, read [the troubleshooting advice](../../replication/troubleshooting.md#fixing-errors-during-a-failover-or-when-promoting-a-secondary-to-a-primary-site). -The `gitlab-ctl promote-to-primary-node` command cannot be used yet in +The `gitlab-ctl promote-to-primary-node` command cannot be used in conjunction with multiple servers, as it can only perform changes on a **secondary** with only a single machine. Instead, you must do this manually. diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md index f9d99095951..085765ec51e 100644 --- a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md @@ -147,7 +147,7 @@ follow these steps to avoid unnecessary data loss: - All replication meters reach 100% replicated, 0% failures. - All verification meters reach 100% verified, 0% failures. - - Database replication lag is 0ms. + - Database replication lag is 0 ms. - The Geo log cursor is up to date (0 events behind). 1. On the **secondary** site: diff --git a/doc/administration/geo/replication/configuration.md b/doc/administration/geo/replication/configuration.md index 55c5d3784c2..d625b2a0324 100644 --- a/doc/administration/geo/replication/configuration.md +++ b/doc/administration/geo/replication/configuration.md @@ -264,7 +264,7 @@ Install the correct certificate based on your certificate type: - **Multi-domain certificate** that includes both primary and secondary site domains: Install the certificate at `/etc/gitlab/ssl` on all **Rails, Sidekiq, and Gitaly** nodes in the **secondary** site. - **Single-domain certificate** where the certificates are specific to each Geo site domain: Generate a valid certificate for your **secondary** site's domain and install it at `/etc/gitlab/ssl` per [these instructions](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) on all **Rails, Sidekiq, and Gitaly** nodes in the **secondary** site. -#### Connecting to external services that use customer certificates +#### Connecting to external services that use custom certificates A copy of the self-signed certificate for the external service needs to be added to the trust store on all the **primary** site's nodes that require access to the service. diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index 0198d2a63e8..022fe114a33 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -201,7 +201,7 @@ successfully, you must replicate their data using some other means. |[CI job artifacts](../../../ci/pipelines/job_artifacts.md) | **Yes** (10.4) | **Yes** (14.10) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Verification is behind the feature flag `geo_job_artifact_replication`, enabled by default in 14.10. | |[CI Pipeline Artifacts](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/ci/pipeline_artifact.rb) | [**Yes** (13.11)](https://gitlab.com/gitlab-org/gitlab/-/issues/238464) | [**Yes** (13.11)](https://gitlab.com/gitlab-org/gitlab/-/issues/238464) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Persists additional artifacts after a pipeline completes. | |[CI Secure Files](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/ci/secure_file.rb) | [**Yes** (15.3)](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91430) | [**Yes** (15.3)](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91430) | [**Yes** (15.3)](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91430) | [No](object_storage.md#verification-of-files-in-object-storage) | Verification is behind the feature flag `geo_ci_secure_file_replication`, enabled by default in 15.3. | -|[Container Registry](../../packages/container_registry.md) | **Yes** (12.3)* | No | No | No | Replication is behind feature flag `geo_container_repository_replication`, enabled by default. Requires additional configuration. See [instructions](container_registry.md) to set up the Container Registry replication. | +|[Container Registry](../../packages/container_registry.md) | **Yes** (12.3)* | No | No | No | See [instructions](container_registry.md) to set up the Container Registry replication. | |[Infrastructure Registry](../../../user/packages/infrastructure_registry/index.md) | **Yes** (14.0) | **Yes** (14.0) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Behind feature flag `geo_package_file_replication`, enabled by default. | |[Project designs repository](../../../user/project/issues/design_management.md) | **Yes** (12.7) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/32467) | N/A | N/A | Designs also require replication of LFS objects and Uploads. | |[Package Registry](../../../user/packages/package_registry/index.md) | **Yes** (13.2) | **Yes** (13.10) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Behind feature flag `geo_package_file_replication`, enabled by default. | diff --git a/doc/administration/geo/replication/geo_validation_tests.md b/doc/administration/geo/replication/geo_validation_tests.md index f09422d1e26..a12dd8d9d68 100644 --- a/doc/administration/geo/replication/geo_validation_tests.md +++ b/doc/administration/geo/replication/geo_validation_tests.md @@ -198,7 +198,7 @@ The following are additional validation tests we performed. [Validate Object storage replication using Azure based object storage](https://gitlab.com/gitlab-org/gitlab/-/issues/348804#note_821294631): -- Description: Tested the average time it takes for a single image to replicate from the primary object storage location to the secondary when using Azure based object storage replication and [GitLab based object storage replication](object_storage.md#enabling-gitlab-managed-object-storage-replication). This was tested by uploading a 1mb image to a project on the primary site every second for 60 seconds. The time was then measured until a image was available on the secondary site. This was achieved using a [Ruby Script](https://gitlab.com/gitlab-org/quality/geo-replication-tester). +- Description: Tested the average time it takes for a single image to replicate from the primary object storage location to the secondary when using Azure based object storage replication and [GitLab based object storage replication](object_storage.md#enabling-gitlab-managed-object-storage-replication). This was tested by uploading a 1 MB image to a project on the primary site every second for 60 seconds. The time was then measured until a image was available on the secondary site. This was achieved using a [Ruby Script](https://gitlab.com/gitlab-org/quality/geo-replication-tester). - Outcome: When using Azure based replication the average time for an image to replicate from the primary object storage to the secondary was recorded as 40 seconds, the longest replication time was 70 seconds and the quickest was 11 seconds. When using GitLab based replication the average time for replication to complete was 5 seconds, the longest replication time was 10 seconds and the quickest was 3 seconds. - Follow up issue: - [Validate Cross Region Object storage replication using Azure based object storage](https://gitlab.com/gitlab-org/gitlab/-/issues/358154) @@ -207,12 +207,12 @@ The following are additional validation tests we performed. [Validate Object storage replication using AWS based object storage](https://gitlab.com/gitlab-org/gitlab/-/issues/351463): -- Description: Tested the average time it takes for a single image to replicate from the primary object storage location to the secondary when using AWS based object storage replication and [GitLab based object storage replication](object_storage.md#enabling-gitlab-managed-object-storage-replication). This was tested by uploading a 1mb image to a project on the primary site every second for 60 seconds. The time was then measured until a image was available on the secondary site. This was achieved using a [Ruby Script](https://gitlab.com/gitlab-org/quality/geo-replication-tester). -- Outcome: When using AWS managed replication the average time for an image to replicate between sites is about 49 seconds, this is true for when sites are located within the same region and when they are further apart (Europe to America). When using Geo managed replication within the same region the average time for replication took just 5 seconds, however when replicating cross region the average time rose to 33 seconds. +- Description: Tested the average time it takes for a single image to replicate from the primary object storage location to the secondary when using AWS based object storage replication and [GitLab based object storage replication](object_storage.md#enabling-gitlab-managed-object-storage-replication). This was tested by uploading a 1 MB image to a project on the primary site every second for 60 seconds. The time was then measured until a image was available on the secondary site. This was achieved using a [Ruby Script](https://gitlab.com/gitlab-org/quality/geo-replication-tester). +- Outcome: When using AWS managed replication the average time for an image to replicate between sites is about 49 seconds, this is true for when sites are located in the same region and when they are further apart (Europe to America). When using Geo managed replication in the same region the average time for replication took just 5 seconds, however when replicating cross region the average time rose to 33 seconds. [Validate Object storage replication using GCP based object storage](https://gitlab.com/gitlab-org/gitlab/-/issues/351464): -- Description: Tested the average time it takes for a single image to replicate from the primary object storage location to the secondary when using GCP based object storage replication and [GitLab based object storage replication](object_storage.md#enabling-gitlab-managed-object-storage-replication). This was tested by uploading a 1mb image to a project on the primary site every second for 60 seconds. The time was then measured until a image was available on the secondary site. This was achieved using a [Ruby Script](https://gitlab.com/gitlab-org/quality/geo-replication-tester). +- Description: Tested the average time it takes for a single image to replicate from the primary object storage location to the secondary when using GCP based object storage replication and [GitLab based object storage replication](object_storage.md#enabling-gitlab-managed-object-storage-replication). This was tested by uploading a 1 MB image to a project on the primary site every second for 60 seconds. The time was then measured until a image was available on the secondary site. This was achieved using a [Ruby Script](https://gitlab.com/gitlab-org/quality/geo-replication-tester). - Outcome: GCP handles replication differently than other Cloud Providers. In GCP, the process is to a create single bucket that is either multi, dual, or single region based. This means that the bucket automatically stores replicas in a region based on the option chosen. Even when using multi region, this only replicates in a single continent, the options being America, Europe, or Asia. At current there doesn't seem to be any way to replicate objects between continents using GCP based replication. For Geo managed replication the average time when replicating in the same region was 6 seconds, and when replicating cross region this rose to just 9 seconds. ## Other tests diff --git a/doc/administration/geo/replication/location_aware_git_url.md b/doc/administration/geo/replication/location_aware_git_url.md index dbe543f5a62..460de5f3232 100644 --- a/doc/administration/geo/replication/location_aware_git_url.md +++ b/doc/administration/geo/replication/location_aware_git_url.md @@ -104,7 +104,7 @@ on the external URL of the current host. For example: You can customize the: -- SSH remote URL to use the location-aware `git.example.com`. To do so, change the SSH remote URL's +- SSH remote URL to use the location-aware `git.example.com`. To do so, change the SSH remote URL host by setting `gitlab_rails['gitlab_ssh_host']` in `gitlab.rb` of web nodes. - HTTP remote URL as shown in [Custom Git clone URL for HTTP(S)](../../../user/admin_area/settings/visibility_and_access_controls.md#customize-git-clone-url-for-https). diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index fa668091c90..1dcced781ce 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -49,6 +49,8 @@ health check manually to get this information and a few more details. #### Health check Rake task +> The use of a custom NTP server was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105514) in GitLab 15.7. + This Rake task can be run on a **Rails** node in the **primary** or **secondary** Geo sites: @@ -80,6 +82,22 @@ All projects are in hashed storage? ... yes Checking Geo ... Finished ``` +You can also specify a custom NTP server using environment variables. For example: + +```shell +export NTP_HOST="ntp.ubuntu.com" +export NTP_TIMEOUT="30" +sudo gitlab-rake gitlab:geo:check +``` + +The following environment variables are supported. + +| Variable | Description | Default value | +| ----------- | ----------- | ------------- | +|`NTP_HOST` | The NTP host. | `pool.ntp.org` | +|`NTP_PORT` | The NTP port the host listens on. |`ntp`| +|`NTP_TIMEOUT`| The NTP timeout in seconds. | The value defined in the `net-ntp` Ruby library ([60 seconds](https://github.com/zencoder/net-ntp/blob/3d0990214f439a5127782e0f50faeaf2c8ca7023/lib/net/ntp/ntp.rb#L6)). | + #### Sync status Rake task Current sync information can be found manually by running this Rake task on any @@ -204,7 +222,7 @@ Commands that change data can cause damage if not run correctly or under the rig ``` 1. This will cause the primary to start checksumming all Uploads. -1. When a primary successfully checksums a record, then all secondaries rechecksum as well, and they compare the values. +1. When a primary successfully checksums a record, then all secondaries recalculate the checksum as well, and they compare the values. A similar thing can be done for all Models handled by the [Geo Self-Service Framework](../../../development/geo/framework.md) which have implemented verification: @@ -1335,7 +1353,7 @@ status ``` 1. This will cause the primary to start checksumming all Uploads. -1. When a primary successfully checksums a record, then all secondaries rechecksum as well, and they compare the values. +1. When a primary successfully checksums a record, then all secondaries recalculate the checksum as well, and they compare the values. For other SSF data types replace `Upload` in the command above with the desired model class. @@ -1476,7 +1494,7 @@ Failed to contact primary https://primary.domain.com/namespace/push_test.git\\nE The partial failover to a secondary Geo *site* may be the result of a temporary/transient issue. Therefore, first attempt to run the promote command again. -1. SSH into every Sidekiq, PostgresSQL, Gitaly, and Rails node in the **secondary** site and run one of the following commands: +1. SSH into every Sidekiq, PostgreSQL, Gitaly, and Rails node in the **secondary** site and run one of the following commands: - To promote the secondary site to primary: @@ -1495,7 +1513,7 @@ The partial failover to a secondary Geo *site* may be the result of a temporary/ If the above steps are **not successful**, proceed through the next steps: -1. SSH to every Sidekiq, PostgresSQL, Gitaly and Rails node in the **secondary** site and perform the following operations: +1. SSH to every Sidekiq, PostgreSQL, Gitaly and Rails node in the **secondary** site and perform the following operations: - Create a `/etc/gitlab/gitlab-cluster.json` file with the following content: diff --git a/doc/administration/geo/replication/tuning.md b/doc/administration/geo/replication/tuning.md index ab9263ad344..4dc3ba93d66 100644 --- a/doc/administration/geo/replication/tuning.md +++ b/doc/administration/geo/replication/tuning.md @@ -30,7 +30,7 @@ However, this may not lead to more downloads in parallel unless the number of available Sidekiq threads is also increased. For example, if repository synchronization concurrency is increased from 25 to 50, you may also want to increase the number of Sidekiq threads from 25 to 50. See the -[Sidekiq concurrency documentation](../../sidekiq/extra_sidekiq_processes.md#number-of-threads) +[Sidekiq concurrency documentation](../../sidekiq/extra_sidekiq_processes.md#concurrency) for more details. ## Repository re-verification diff --git a/doc/administration/geo/replication/version_specific_upgrades.md b/doc/administration/geo/replication/version_specific_upgrades.md index 9aad5cdeaa7..d981656f748 100644 --- a/doc/administration/geo/replication/version_specific_upgrades.md +++ b/doc/administration/geo/replication/version_specific_upgrades.md @@ -39,10 +39,6 @@ results in a loop that consistently fails for all objects stored in object stora For information on how to fix this, see [Troubleshooting - Failed syncs with GitLab-managed object storage replication](troubleshooting.md#failed-syncs-with-gitlab-managed-object-storage-replication). -## Upgrading to 14.6 - -[Geo proxying](../secondary_proxy/index.md) was [enabled by default for unified URLs](https://gitlab.com/gitlab-org/gitlab/-/issues/325732) in 14.6. This may be a breaking change. If needed, you may [disable Geo proxying](../secondary_proxy/index.md#disable-geo-proxying). - ## Upgrading to 14.4 There is [an issue in GitLab 14.4.0 through 14.4.2](../../../update/index.md#1440) that can affect Geo and other features that rely on cronjobs. We recommend upgrading to GitLab 14.4.3 or later. @@ -144,7 +140,7 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab ## Upgrading to GitLab 13.9 -### Error during zero-downtime upgrade: "cannot drop column asset_proxy_whitelist" +### Error during zero-downtime upgrade: `cannot drop column asset_proxy_whitelist` We've detected an issue [with a column rename](https://gitlab.com/gitlab-org/gitlab/-/issues/324160) that prevents upgrades to GitLab 13.9.0, 13.9.1, 13.9.2 and 13.9.3 when following the zero-downtime steps. It is necessary diff --git a/doc/administration/geo/secondary_proxy/index.md b/doc/administration/geo/secondary_proxy/index.md index 2786982bb51..ac8b88a91d5 100644 --- a/doc/administration/geo/secondary_proxy/index.md +++ b/doc/administration/geo/secondary_proxy/index.md @@ -70,39 +70,6 @@ a single URL used by all Geo sites, including the primary. In Kubernetes, you can use the same domain under `global.hosts.domain` as for the primary site. -## Disable Geo proxying - -You can disable the secondary proxying on each Geo site, separately, by following these steps with Omnibus-based packages: - -1. SSH into each application node (serving user traffic directly) on your secondary Geo site - and add the following environment variable: - - ```shell - sudo editor /etc/gitlab/gitlab.rb - ``` - - ```ruby - gitlab_workhorse['env'] = { - "GEO_SECONDARY_PROXY" => "0" - } - ``` - -1. Reconfigure the updated nodes for the change to take effect: - - ```shell - gitlab-ctl reconfigure - ``` - -In Kubernetes, you can use `--set gitlab.webservice.extraEnv.GEO_SECONDARY_PROXY="0"`, -or specify the following in your values file: - -```yaml -gitlab: - webservice: - extraEnv: - GEO_SECONDARY_PROXY: "0" -``` - ## Geo proxying with Separate URLs > Geo secondary proxying for separate URLs is [enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/346112) in GitLab 15.1. @@ -116,11 +83,18 @@ You can also add feedback in the epic about any use-cases that are not possible anymore with proxying enabled. If you run into issues, to disable this feature, disable the `geo_secondary_proxy_separate_urls` feature flag. -SSH into one node running Rails on your primary Geo site and run: -```shell -sudo gitlab-rails runner "Feature.disable(:geo_secondary_proxy_separate_urls)" -``` +1. SSH into one node running Rails on your primary Geo site and run: + + ```shell + sudo gitlab-rails runner "Feature.disable(:geo_secondary_proxy_separate_urls)" + ``` + +1. Restart Puma on all of the nodes running Rails on your secondary Geo site: + + ```shell + sudo gitlab-ctl restart puma + ``` In Kubernetes, you can run the same command in the toolbox pod. Refer to the [Kubernetes cheat sheet](https://docs.gitlab.com/charts/troubleshooting/kubernetes_cheat_sheet.html#gitlab-specific-kubernetes-information) @@ -192,3 +166,42 @@ It does not cover all data types, more will be added in the future as they are t 1. Git reads are served from the local secondary while pushes get proxied to the primary. Selective sync or cases where repositories don't exist locally on the Geo secondary throw a "not found" error. 1. Pages can use the same URL (without access control), but must be configured separately and are not proxied. + +## Disable Geo proxying + +Secondary proxying is enabled by default on a secondary site when it uses a unified URL, meaning, the same `external_url` as the primary site. Disabling proxying in this case tends to not be helpful due to completely different behavior being served at the same URL, depending on routing. + +Secondary proxying is enabled by default in GitLab 15.1 on a secondary site even without a unified URL. If proxying needs to be disabled on a secondary site, it is much easier to disable the feature flag in [Geo proxying with Separate URLs](#geo-proxying-with-separate-urls). However, if there are multiple secondary sites, then the instructions in this section can be used to disable secondary proxying per site. + +Additionally, the `gitlab-workhorse` service polls `/api/v4/geo/proxy` every 10 seconds. In GitLab 15.2 and later, it is only polled once, if Geo is not enabled. Prior to GitLab 15.2, you can stop this polling by disabling secondary proxying. + +You can disable the secondary proxying on each Geo site, separately, by following these steps with Omnibus-based packages: + +1. SSH into each application node (serving user traffic directly) on your secondary Geo site + and add the following environment variable: + + ```shell + sudo editor /etc/gitlab/gitlab.rb + ``` + + ```ruby + gitlab_workhorse['env'] = { + "GEO_SECONDARY_PROXY" => "0" + } + ``` + +1. Reconfigure the updated nodes for the change to take effect: + + ```shell + gitlab-ctl reconfigure + ``` + +In Kubernetes, you can use `--set gitlab.webservice.extraEnv.GEO_SECONDARY_PROXY="0"`, +or specify the following in your values file: + +```yaml +gitlab: + webservice: + extraEnv: + GEO_SECONDARY_PROXY: "0" +``` diff --git a/doc/administration/geo/setup/database.md b/doc/administration/geo/setup/database.md index 86caf5306b5..99f7b32be59 100644 --- a/doc/administration/geo/setup/database.md +++ b/doc/administration/geo/setup/database.md @@ -35,23 +35,23 @@ or trying to evaluate Geo for a future clusterized installation. A single instance can be expanded to a clusterized version using Patroni, which is recommended for a highly available architecture. -Follow below the instructions on how to set up PostgreSQL replication as a single instance database. +Follow the instructions below on how to set up PostgreSQL replication as a single instance database. Alternatively, you can look at the [Multi-node database replication](#multi-node-database-replication) instructions on setting up replication with a Patroni cluster. ### PostgreSQL replication The GitLab **primary** site where the write operations happen connects to -the **primary** database server, and **secondary** sites +the **primary** database server. **Secondary** sites connect to their own database servers (which are read-only). -We recommend using [PostgreSQL replication slots](https://medium.com/@tk512/replication-slots-in-postgresql-b4b03d277c75) +You should use [PostgreSQL's replication slots](https://medium.com/@tk512/replication-slots-in-postgresql-b4b03d277c75) to ensure that the **primary** site retains all the data necessary for the **secondary** sites to recover. See below for more details. The following guide assumes that: -- You are using Omnibus and therefore you are using PostgreSQL 12 or later +- You are using Omnibus and therefore you are using PostgreSQL 12 or later, which includes the [`pg_basebackup` tool](https://www.postgresql.org/docs/12/app-pgbasebackup.html). - You have a **primary** site already set up (the GitLab server you are replicating from), running Omnibus' PostgreSQL (or equivalent version), and @@ -120,8 +120,8 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o 1. Define a password for the database [replication user](https://wiki.postgresql.org/wiki/Streaming_Replication). - We will use the username defined in `/etc/gitlab/gitlab.rb` under the `postgresql['sql_replication_user']` - setting. The default value is `gitlab_replicator`, but if you changed it to something else, adapt + Use the username defined in `/etc/gitlab/gitlab.rb` under the `postgresql['sql_replication_user']` + setting. The default value is `gitlab_replicator`. If you changed the username to something else, adapt the instructions below. Generate a MD5 hash of the desired password: @@ -141,7 +141,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ``` If you are using an external database not managed by Omnibus GitLab, you need - to create the replicator user and define a password to it manually: + to create the `gitlab_replicator` user and define a password for that user manually: ```sql --- Create a new user 'replicator' @@ -155,16 +155,16 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o For security reasons, PostgreSQL does not listen on any network interfaces by default. However, Geo requires the **secondary** site to be able to - connect to the **primary** site's database. For this reason, we need the IP address of + connect to the **primary** site's database. For this reason, you need the IP address of each site. NOTE: For external PostgreSQL instances, see [additional instructions](external_database.md). - If you are using a cloud provider, you can lookup the addresses for each + If you are using a cloud provider, you can look up the addresses for each Geo site through your cloud provider's management console. - To lookup the address of a Geo site, SSH in to the Geo site and execute: + To look up the address of a Geo site, SSH into the Geo site and execute: ```shell ## @@ -187,7 +187,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o | `postgresql['md5_auth_cidr_addresses']` | **Primary** and **Secondary** sites' public or VPC private addresses. | If you are using Google Cloud Platform, SoftLayer, or any other vendor that - provides a virtual private cloud (VPC) you can use the **primary** and **secondary** sites + provides a virtual private cloud (VPC), you can use the **primary** and **secondary** sites' private addresses (corresponds to "internal address" for Google Cloud Platform) for `postgresql['md5_auth_cidr_addresses']` and `postgresql['listen_address']`. @@ -196,14 +196,14 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o for more details. NOTE: - If you need to use `0.0.0.0` or `*` as the listen_address, you also must add + If you need to use `0.0.0.0` or `*` as the `listen_address`, you also must add `127.0.0.1/32` to the `postgresql['md5_auth_cidr_addresses']` setting, to allow Rails to connect through `127.0.0.1`. For more information, see [omnibus-5258](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5258). - Depending on your network configuration, the suggested addresses may not - be correct. If your **primary** site and **secondary** sites connect over a local + Depending on your network configuration, the suggested addresses may + be incorrect. If your **primary** site and **secondary** sites connect over a local area network, or a virtual network connecting availability zones like - [Amazon's VPC](https://aws.amazon.com/vpc/) or [Google's VPC](https://cloud.google.com/vpc/) + [Amazon's VPC](https://aws.amazon.com/vpc/) or [Google's VPC](https://cloud.google.com/vpc/), you should use the **secondary** site's private address for `postgresql['md5_auth_cidr_addresses']`. Edit `/etc/gitlab/gitlab.rb` and add the following, replacing the IP @@ -286,12 +286,12 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ``` 1. Now that the PostgreSQL server is set up to accept remote connections, run - `netstat -plnt | grep 5432` to make sure that PostgreSQL is listening on port + `netstat -plnt | grep 5432` to ensure that PostgreSQL is listening on port `5432` to the **primary** site's private address. 1. A certificate was automatically generated when GitLab was reconfigured. This is used automatically to protect your PostgreSQL traffic from - eavesdroppers, but to protect against active ("man-in-the-middle") attackers, + eavesdroppers. To protect against active ("man-in-the-middle") attackers, the **secondary** site needs a copy of the certificate. Make a copy of the PostgreSQL `server.crt` file on the **primary** site by running this command: @@ -299,26 +299,26 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o cat ~gitlab-psql/data/server.crt ``` - Copy the output into a clipboard or into a local file. You + Copy the output to the clipboard or into a local file. You need it when setting up the **secondary** site! The certificate is not sensitive data. However, this certificate is created with a generic `PostgreSQL` Common Name. For this, you must use the `verify-ca` mode when replicating the database, otherwise, - the hostname mismatch will cause errors. + the hostname mismatch causes errors. 1. Optional. Generate your own SSL certificate and manually [configure SSL for PostgreSQL](https://docs.gitlab.com/omnibus/settings/database.html#configuring-ssl), instead of using the generated certificate. - You will need at least the SSL certificate and key, and set the `postgresql['ssl_cert_file']` and + You need at least the SSL certificate and key. Set the `postgresql['ssl_cert_file']` and `postgresql['ssl_key_file']` values to their full paths, as per the Database SSL docs. This allows you to use the `verify-full` SSL mode when replicating the database and get the extra benefit of verifying the full hostname in the CN. You can use this certificate (that you have also set in `postgresql['ssl_cert_file']`) instead - of the certificate from the point above going forward. This will allow you to use `verify-full` + of the certificate from the point above going forward. This allows you to use `verify-full` without replication errors if the CN matches. #### Step 2. Configure the **secondary** server @@ -337,7 +337,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ``` NOTE: - This step is important so we don't try to execute anything before the site is fully configured. + This step is important so you don't try to execute anything before the site is fully configured. 1. [Check TCP connectivity](../../raketasks/maintenance.md) to the **primary** site's PostgreSQL server: @@ -348,7 +348,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o NOTE: If this step fails, you may be using the wrong IP address, or a firewall may be preventing access to the site. Check the IP address, paying close - attention to the difference between public and private addresses and ensure + attention to the difference between public and private addresses. Ensure that, if a firewall is present, the **secondary** site is permitted to connect to the **primary** site on port 5432. @@ -389,14 +389,14 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ``` NOTE: - If you are using manually generated certificates and plan on using - `sslmode=verify-full` to benefit of the full hostname verification, - make sure to replace `verify-ca` to `verify-full` when + If you are using manually generated certificates and want to use + `sslmode=verify-full` to benefit from the full hostname verification, + replace `verify-ca` with `verify-full` when running the command. - When prompted enter the _plaintext_ password you set in the first step for the + When prompted, enter the _plaintext_ password you set in the first step for the `gitlab_replicator` user. If all worked correctly, you should see - the list of **primary** site's databases. + the list of the **primary** site's databases. A failure to connect here indicates that the TLS configuration is incorrect. Ensure that the contents of `~gitlab-psql/data/server.crt` on the **primary** site @@ -404,8 +404,8 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o 1. Configure PostgreSQL: - This step is similar to how we configured the **primary** instance. - We must enable this, even if using a single node. + This step is similar to how you configured the **primary** instance. + You must enable this, even if using a single node. Edit `/etc/gitlab/gitlab.rb` and add the following, replacing the IP addresses with addresses appropriate to your network configuration: @@ -450,12 +450,12 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o #### Step 3. Initiate the replication process -Below we provide a script that connects the database on the **secondary** site to -the database on the **primary** site, replicates the database, and creates the +Below is a script that connects the database on the **secondary** site to +the database on the **primary** site. This script replicates the database and creates the needed files for streaming replication. The directories used are the defaults that are set up in Omnibus. If you have -changed any defaults, configure it as you see fit replacing the directories and paths. +changed any defaults, configure the script accordingly, replacing any directories and paths. WARNING: Make sure to run this on the **secondary** site as it removes all PostgreSQL's @@ -469,7 +469,7 @@ data before running `pg_basebackup`. 1. Choose a database-friendly name to use for your **secondary** site to use as the replication slot name. For example, if your domain is - `secondary.geo.example.com`, you may use `secondary_example` as the slot + `secondary.geo.example.com`, use `secondary_example` as the slot name as shown in the commands below. 1. Execute the command below to start a backup/restore and begin the replication @@ -492,33 +492,36 @@ data before running `pg_basebackup`. ``` NOTE: - If you have generated custom PostgreSQL certificates, you will want to use + If you have generated custom PostgreSQL certificates, you need to use `--sslmode=verify-full` (or omit the `sslmode` line entirely), to benefit from the extra validation of the full host name in the certificate CN / SAN for additional security. - Otherwise, using the automatically created certificate with `verify-full` will fail, - as it has a generic `PostgreSQL` CN which will not match the `--host` value in this command. + Otherwise, using the automatically created certificate with `verify-full` fails, + as it has a generic `PostgreSQL` CN which doesn't match the `--host` value in this command. This command also takes a number of additional options. You can use `--help` - to list them all, but here are a couple of tips: + to list them all, but here are some tips: - - If PostgreSQL is listening on a non-standard port, add `--port=` as well. + - If PostgreSQL is listening on a non-standard port, add `--port=`. - If your database is too large to be transferred in 30 minutes, you need - to increase the timeout, for example, `--backup-timeout=3600` if you expect the + to increase the timeout. For example, use `--backup-timeout=3600` if you expect the initial replication to take under an hour. - Pass `--sslmode=disable` to skip PostgreSQL TLS authentication altogether (for example, you know the network path is secure, or you are using a site-to-site VPN). It is **not** safe over the public Internet! - You can read more details about each `sslmode` in the - [PostgreSQL documentation](https://www.postgresql.org/docs/12/libpq-ssl.html#LIBPQ-SSL-PROTECTION); - the instructions above are carefully written to ensure protection against + [PostgreSQL documentation](https://www.postgresql.org/docs/12/libpq-ssl.html#LIBPQ-SSL-PROTECTION). + The instructions above are carefully written to ensure protection against both passive eavesdroppers and active "man-in-the-middle" attackers. - Change the `--slot-name` to the name of the replication slot to be used on the **primary** database. The script attempts to create the replication slot automatically if it does not exist. - If you're repurposing an old site into a Geo **secondary** site, you must add `--force` to the command line. - - When not in a production machine you can disable backup step if you - really sure this is what you want by adding `--skip-backup` + - When not in a production machine, you can disable the backup step (if you + are certain this is what you want) by adding `--skip-backup`. + - If you are using PgBouncer, you need to target the database host directly. + - If you are using Patroni on your primary site, you must target the current leader host. + - If you are using a load balancer proxy (for example HAProxy) and it is targeting the Patroni leader for the primary, you should target the load balancer proxy instead. The replication process is now complete. @@ -528,9 +531,9 @@ The replication process is now complete. PostgreSQL connections, which can improve performance even when using in a single instance installation. -We recommend using PgBouncer if you use GitLab in a highly available +You should use PgBouncer if you use GitLab in a highly available configuration with a cluster of nodes supporting a Geo **primary** site and -two other clusters of nodes supporting a Geo **secondary** site. One for the +two other clusters of nodes supporting a Geo **secondary** site. You need two PgBouncer nodes: one for the main database and the other for the tracking database. For more information, see [High Availability with Omnibus GitLab](../../postgresql/replication_and_failover.md). @@ -542,7 +545,7 @@ when using Omnibus-managed PostgreSQL instances: On the GitLab Geo **primary** site: 1. The default value for the replication user is `gitlab_replicator`, but if you've set a custom replication - user in your `/etc/gitlab/gitlab.rb` under the `postgresql['sql_replication_user']` setting, make sure to + user in your `/etc/gitlab/gitlab.rb` under the `postgresql['sql_replication_user']` setting, ensure you adapt the following instructions for your own user. Generate an MD5 hash of the desired password: @@ -574,7 +577,7 @@ On the GitLab Geo **primary** site: ``` Until the password is updated on any **secondary** sites, the [PostgreSQL log](../../logs/index.md#postgresql-logs) on -the secondaries will report the following error message: +the secondaries report the following error message: ```console FATAL: could not connect to the primary server: FATAL: password authentication failed for user "gitlab_replicator" @@ -616,16 +619,16 @@ If you still haven't [migrated from repmgr to Patroni](#migrating-from-repmgr-to ### Migrating from repmgr to Patroni -1. Before migrating, we recommend that there is no replication lag between the **primary** and **secondary** sites and that replication is paused. In GitLab 13.2 and later, you can pause and resume replication with `gitlab-ctl geo-replication-pause` and `gitlab-ctl geo-replication-resume` on a Geo secondary database node. +1. Before migrating, you should ensure there is no replication lag between the **primary** and **secondary** sites and that replication is paused. In GitLab 13.2 and later, you can pause and resume replication with `gitlab-ctl geo-replication-pause` and `gitlab-ctl geo-replication-resume` on a Geo secondary database node. 1. Follow the [instructions to migrate repmgr to Patroni](../../postgresql/replication_and_failover.md#switching-from-repmgr-to-patroni). When configuring Patroni on each **primary** site database node, add `patroni['replication_slots'] = { '<slot_name>' => 'physical' }` -to `gitlab.rb` where `<slot_name>` is the name of the replication slot for your **secondary** site. This ensures that Patroni recognizes the replication slot as permanent and not drop it upon restarting. -1. If database replication to the **secondary** site was paused before migration, resume replication after Patroni is confirmed working on the **primary** site. +to `gitlab.rb` where `<slot_name>` is the name of the replication slot for your **secondary** site. This ensures that Patroni recognizes the replication slot as permanent and doesn't drop it upon restarting. +1. If database replication to the **secondary** site was paused before migration, resume replication after Patroni is confirmed as working on the **primary** site. ### Migrating a single PostgreSQL node to Patroni Before the introduction of Patroni, Geo had no Omnibus support for HA setups on the **secondary** site. -With Patroni it's now possible to support that. To migrate the existing PostgreSQL to Patroni: +With Patroni, this support is now possible. To migrate the existing PostgreSQL to Patroni: 1. Make sure you have a Consul cluster setup on the secondary (similar to how you set it up on the **primary** site). 1. [Configure a permanent replication slot](#step-1-configure-patroni-permanent-replication-slot-on-the-primary-site). @@ -634,23 +637,23 @@ With Patroni it's now possible to support that. To migrate the existing PostgreS 1. [Configure a Standby Cluster](#step-4-configure-a-standby-cluster-on-the-secondary-site) on that single node machine. -You end up with a “Standby Cluster” with a single node. That allows you to later on add additional Patroni nodes by following the same instructions above. +You end up with a “Standby Cluster” with a single node. That allows you to add additional Patroni nodes by following the same instructions above. ### Patroni support -Patroni is the official replication management solution for Geo. It +Patroni is the official replication management solution for Geo. Patroni can be used to build a highly available cluster on the **primary** and a **secondary** Geo site. -Using Patroni on a **secondary** site is optional and you don't have to use the same amount of +Using Patroni on a **secondary** site is optional and you don't have to use the same number of nodes on each Geo site. -For instructions about how to set up Patroni on the primary site, see the +For instructions on how to set up Patroni on the primary site, see the [PostgreSQL replication and failover with Omnibus GitLab](../../postgresql/replication_and_failover.md#patroni) page. #### Configuring Patroni cluster for a Geo secondary site In a Geo secondary site, the main PostgreSQL database is a read-only replica of the primary site's PostgreSQL database. -If you are currently using `repmgr` on your Geo primary site, see [these instructions](#migrating-from-repmgr-to-patroni) +If you are using `repmgr` on your Geo primary site, see [these instructions](#migrating-from-repmgr-to-patroni) for migrating from `repmgr` to Patroni. A production-ready and secure setup requires at least: @@ -661,14 +664,14 @@ A production-ready and secure setup requires at least: - 1 internal load-balancer _(primary site only)_ The internal load balancer provides a single endpoint for connecting to the Patroni cluster's leader whenever a new leader is -elected, and it is required for enabling cascading replication from the secondary sites. +elected. The load balancer is required for enabling cascading replication from the secondary sites. Be sure to use [password credentials](../../postgresql/replication_and_failover.md#database-authorization-for-patroni) and other database best practices. ##### Step 1. Configure Patroni permanent replication slot on the primary site -To set up database replication with Patroni on a secondary site, we must +To set up database replication with Patroni on a secondary site, you must configure a _permanent replication slot_ on the primary site's Patroni cluster, and ensure password authentication is used. @@ -734,8 +737,8 @@ Leader instance**: ##### Step 2. Configure the internal load balancer on the primary site To avoid reconfiguring the Standby Leader on the secondary site whenever a new -Leader is elected on the primary site, we must set up a TCP internal load -balancer which gives a single endpoint for connecting to the Patroni +Leader is elected on the primary site, you should set up a TCP internal load +balancer. This load balancer provides a single endpoint for connecting to the Patroni cluster's Leader. The Omnibus GitLab packages do not include a Load Balancer. Here's how you @@ -773,14 +776,14 @@ backend postgresql server patroni3.internal 10.6.0.23:5432 maxconn 100 check port 8008 ``` -Refer to your preferred Load Balancer's documentation for further guidance. +For further guidance, refer to the documentation for your preferred load balancer. ##### Step 3. Configure PgBouncer nodes on the secondary site A production-ready and highly available configuration requires at least -three Consul nodes, a minimum of one PgBouncer node, but it's recommended to have -one per database node. An internal load balancer (TCP) is required when there is -more than one PgBouncer service nodes. The internal load balancer provides a single +three Consul nodes and a minimum of one PgBouncer node. However, it is recommended to have +one PgBouncer node per database node. An internal load balancer (TCP) is required when there is +more than one PgBouncer service node. The internal load balancer provides a single endpoint for connecting to the PgBouncer cluster. For more information, see [High Availability with Omnibus GitLab](../../postgresql/replication_and_failover.md). @@ -841,7 +844,7 @@ On each node running a PgBouncer instance on the **secondary** site: NOTE: If you are converting a secondary site with a single PostgreSQL instance to a Patroni Cluster, you must start on the PostgreSQL instance. It becomes the Patroni Standby Leader instance, -and then you can switch over to another replica if you need. +and then you can switch over to another replica if you need to. For each node running a Patroni instance on the secondary site: @@ -895,7 +898,7 @@ For each node running a Patroni instance on the secondary site: ``` 1. Reconfigure GitLab for the changes to take effect. - This is required to bootstrap PostgreSQL users and settings. + This step is required to bootstrap PostgreSQL users and settings. - If this is a fresh installation of Patroni: @@ -915,13 +918,12 @@ For each node running a Patroni instance on the secondary site: ### Migrating a single tracking database node to Patroni -Before the introduction of Patroni, Geo had no Omnibus support for HA setups on +Before the introduction of Patroni, Geo provided no Omnibus support for HA setups on the secondary site. -With Patroni, it's now possible to support that. Due to some restrictions on the -Patroni implementation on Omnibus that do not allow us to manage two different -clusters on the same machine, we recommend setting up a new Patroni cluster for -the tracking database by following the same instructions above. +With Patroni, it's now possible to support HA setups. However, some restrictions in Patroni +prevent the management of two different clusters on the same machine. You should set up a new +Patroni cluster for the tracking database by following the same instructions above. The secondary nodes backfill the new tracking database, and no data synchronization is required. @@ -935,8 +937,8 @@ Omnibus automatically configures a tracking database when `roles(['geo_secondary If you want to run this database in a highly available configuration, don't use the `geo_secondary_role` above. Instead, follow the instructions below. -A production-ready and secure setup for the tracking PostgreSQL DB requires at least three Consul nodes, two -Patroni nodes and one PgBouncer node on the secondary site. +A production-ready and secure setup for the tracking PostgreSQL DB requires at least three Consul nodes: two +Patroni nodes, and one PgBouncer node on the secondary site. Because of [omnibus-6587](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6587), Consul can't track multiple services, so these must be different than the nodes used for the Standby Cluster database. @@ -1066,7 +1068,7 @@ On each node running a Patroni instance on the secondary site for the PostgreSQL ``` 1. Reconfigure GitLab for the changes to take effect. - This is required to bootstrap PostgreSQL users and settings: + This step is required to bootstrap PostgreSQL users and settings: ```shell gitlab-ctl reconfigure diff --git a/doc/administration/gitaly/configure_gitaly.md b/doc/administration/gitaly/configure_gitaly.md index e7a6fc35ced..d2f5282a69a 100644 --- a/doc/administration/gitaly/configure_gitaly.md +++ b/doc/administration/gitaly/configure_gitaly.md @@ -49,6 +49,9 @@ NOTE: When configured to run on their own servers, Gitaly servers must be [upgraded](../../update/package/index.md) before Gitaly clients in your cluster. +NOTE: +[Disk requirements](index.md#disk-requirements) apply to Gitaly nodes. + The process for setting up Gitaly on its own server is: 1. [Install Gitaly](#install-gitaly). @@ -750,14 +753,25 @@ settings: ## Limit RPC concurrency -Clone traffic can put a large strain on your Gitaly service. The bulk of the work gets done in the -either of the following RPCs: +WARNING: +Enabling limits on your environment should be done with caution and only +in select circumstances, such as to protect against unexpected traffic. +When reached, limits _do_ result in disconnects that negatively impact users. +For consistent and stable performance, you should first explore other options such as +adjusting node specifications, and [reviewing large repositories](../../user/project/repository/managing_large_repositories.md) or workloads. + +When cloning or pulling repositories, various RPCs run in the background. In particular, the Git pack RPCs: - `SSHUploadPackWithSidechannel` (for Git SSH). - `PostUploadPackWithSidechannel` (for Git HTTP). -To prevent such workloads from overwhelming your Gitaly server, you can set concurrency limits in -Gitaly's configuration file. For example: +These RPCs can consume a large amount of resources, which can have a significant impact in situations such as: + +- Unexpectedly high traffic. +- Running against [large repositories](../../user/project/repository/managing_large_repositories.md) that don't follow best practices. + +You can limit these processes from overwhelming your Gitaly server in these scenarios using the concurrency limits in Gitaly's configuration file. For +example: ```ruby # in /etc/gitlab/gitlab.rb @@ -795,30 +809,40 @@ repository. In the example above: - If a request waits in the queue for more than 1 second, it is rejected with an error. - If the queue grows beyond 10, subsequent requests are rejected with an error. +NOTE: +When these limits are reached, users are disconnected. + You can observe the behavior of this queue using the Gitaly logs and Prometheus. For more information, see the [relevant documentation](monitoring.md#monitor-gitaly-concurrency-limiting). ## Control groups +WARNING: +Enabling limits on your environment should be done with caution and only +in select circumstances, such as to protect against unexpected traffic. +When reached, limits _do_ result in disconnects that negatively impact users. +For consistent and stable performance, you should first explore other options such as +adjusting node specifications, and [reviewing large repositories](../../user/project/repository/managing_large_repositories.md) or workloads. + FLAG: On self-managed GitLab, by default repository cgroups are not available. To make it available, ask an administrator to [enable the feature flag](../feature_flags.md) named `gitaly_run_cmds_in_cgroup`. -Control groups (cgroups) in Linux allow limits to be imposed on how much memory and CPU can be consumed. +When enabling cgroups for memory, you should ensure that no swap is configured on the Gitaly nodes as +processes may switch to using that instead of being terminated. This situation could lead to notably compromised +performance. + +You can use control groups (cgroups) in Linux to impose limits on how much memory and CPU can be consumed by Gitaly processes. See the [`cgroups` Linux man page](https://man7.org/linux/man-pages/man7/cgroups.7.html) for more information. -cgroups can be useful for protecting the system against resource exhaustion because of over consumption of memory and CPU. +cgroups can be useful for protecting the system against unexpected resource exhaustion because of over consumption of memory and CPU. -Some Git operations are expensive by nature. `git clone`, for instance, -spawns a `git-upload-pack` process on the server that can consume a lot of memory -for large repositories. For example, a client that keeps on cloning a -large repository over and over again. This situation could potentially use up all of the -memory on a server, causing other operations to fail for other users. +Some Git operations can consume notable resources up to the point of exhaustion in situations such as: -A repository can consume large amounts of memory for many reasons when cloned or downloaded. -Using cgroups allows the kernel to kill these operations before they hog up all system resources. +- Unexpectedly high traffic. +- Operations running against large repositories that don't follow best practices. -Gitaly shells out to Git for many of its operations. Git can consume a lot of resources for certain operations, -especially for large repositories. +As a hard protection, it's possible to use cgroups that configure the kernel to terminate these operations before they hog up all system resources +and cause instability. Gitaly has built-in cgroups control. When configured, Gitaly assigns Git processes to a cgroup based on the repository the Git command is operating in. These cgroups are called repository cgroups. Each repository cgroup: @@ -832,8 +856,8 @@ When a repository cgroup reaches its: - Memory limit, the kernel looks through the processes for a candidate to kill. - CPU limit, processes are not killed, but the processes are prevented from consuming more CPU than allowed. -You configure repository cgroups for your GitLab installation to protect against system resource starvation from a few -large repositories or bad actors. +NOTE: +When these limits are reached, performance may be reduced and users may be disconnected. ### Configure repository cgroups (new method) @@ -913,8 +937,8 @@ gitaly['cgroups_cpu_enabled'] = true In the previous example using the new configuration method: -- The top level memory limit is capped at 60gb. -- Each of the 1000 cgroups in the repositories pool is capped at 20gb. +- The top level memory limit is capped at 60 GB. +- Each of the 1000 cgroups in the repositories pool is capped at 20 GB. This configuration leads to "oversubscription". Each cgroup in the pool has a much larger capacity than 1/1000th of the top-level memory limit. diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 1b7e1f0f7c6..32b44552b1a 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -54,9 +54,8 @@ Before deploying Gitaly Cluster, review: If you have: -- Not yet migrated to Gitaly Cluster and want to continue using NFS, remain on the service you are using. NFS is - supported in 14.x releases but is [deprecated](../../update/deprecations.md#nfs-for-git-repository-storage). - Support for storing Git repository data on NFS is scheduled to end for all versions of GitLab on November 22, 2022. +- Not yet migrated to Gitaly Cluster and want to continue using NFS, remain on the service you are using. However, NFS + is [no longer supported](../../update/removals.md#nfs-as-git-repository-storage-is-no-longer-supported). - Not yet migrated to Gitaly Cluster but want to migrate away from NFS, you have two options: - A sharded Gitaly instance. - Gitaly Cluster. @@ -90,6 +89,23 @@ If you are unable to use either method, contact customer support for restoration Contact customer support for immediate help in restoration or recovery. +## Disk requirements + +Gitaly and Gitaly Cluster require fast local storage to perform effectively because they are heavy I/O-based processes. Therefore, +we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). + +These SSDs should have a throughput of at least: + +- 8,000 input/output operations per second (IOPS) for read operations. +- 2,000 IOPS for write operations. + +These IOPS values are initial recommendations, and may be adjusted to greater or lesser values +depending on the scale of your environment's workload. If you’re running the environment on a +cloud provider, refer to their documentation about how to configure IOPS correctly. + +For repository data, only local storage is supported for Gitaly and Gitaly Cluster for performance and consistency reasons. +Alternatives such as [NFS](../nfs.md) or [cloud-based file systems](../nfs.md#avoid-using-cloud-based-file-systems) are not supported. + ## Directly accessing repositories GitLab doesn't advise directly accessing Gitaly repositories stored on disk with a Git client or any other tool, @@ -405,33 +421,6 @@ The leftover state is eventually cleaned up. Unlike Gitaly, Gitaly Cluster doesn't move the repositories in the storages but only virtually moves the repository by updating the relative path of the repository in the metadata store. -### Moving beyond NFS - -Engineering support for NFS for Git repositories is deprecated. Technical support is planned to be unavailable starting -November 22, 2022. See our [statement of support](https://about.gitlab.com/support/statement-of-support/#gitaly-and-nfs) -for more details. - -[Network File System (NFS)](https://en.wikipedia.org/wiki/Network_File_System) -is not well suited to Git workloads which are CPU and IOPS sensitive. -Specifically: - -- Git is sensitive to file system latency. Some operations require many - read operations. Operations that are fast on block storage can become an order of - magnitude slower. This significantly impacts GitLab application performance. -- NFS performance optimizations that prevent the performance gap between - block storage and NFS being even wider are vulnerable to race conditions. We have observed - [data inconsistencies](https://gitlab.com/gitlab-org/gitaly/-/issues/2589) - in production environments caused by simultaneous writes to different NFS - clients. Data corruption is not an acceptable risk. - -Gitaly Cluster is purpose built to provide reliable, high performance, fault -tolerant Git storage. - -Further reading: - -- Blog post: [The road to Gitaly v1.0 (aka, why GitLab doesn't require NFS for storing Git data anymore)](https://about.gitlab.com/blog/2018/09/12/the-road-to-gitaly-1-0/) -- Blog post: [How we spent two weeks hunting an NFS bug in the Linux kernel](https://about.gitlab.com/blog/2018/11/14/how-we-spent-two-weeks-hunting-an-nfs-bug/) - ### Components Gitaly Cluster consists of multiple components: @@ -672,7 +661,7 @@ To see if GitLab can access the repository file system directly, we use the foll - Gitaly ensures that the file system has a metadata file in its root with a UUID in it. - Gitaly reports this UUID to GitLab by using the `ServerInfo` RPC. -- GitLab Rails tries to read the metadata file directly. If it exists, and if the UUID's match, +- GitLab Rails tries to read the metadata file directly. If it exists, and if the UUIDs match, assume we have direct access. Direct Git access is: @@ -696,8 +685,3 @@ There are two facets to our efforts to remove direct Git access in GitLab: The second facet presents the only real solution. For this, we developed [Gitaly Cluster](#gitaly-cluster). - -## NFS deprecation notice - -Engineering support for NFS for Git repositories is deprecated. Technical support is planned to be -unavailable beginning November 22, 2022. For further information, see our [NFS Deprecation](../nfs.md#gitaly-and-nfs-deprecation) documentation. diff --git a/doc/administration/gitaly/monitoring.md b/doc/administration/gitaly/monitoring.md index 8d4f30c7c20..9024da269ca 100644 --- a/doc/administration/gitaly/monitoring.md +++ b/doc/administration/gitaly/monitoring.md @@ -49,7 +49,7 @@ the Gitaly logs and Prometheus: You can observe the status of [control groups (cgroups)](configure_gitaly.md#control-groups) using Prometheus: - `gitaly_cgroups_reclaim_attempts_total`, a gauge for the total number of times - there has been a memory relcaim attempt. This number resets each time a server is + there has been a memory reclaim attempt. This number resets each time a server is restarted. - `gitaly_cgroups_cpu_usage`, a gauge that measures CPU usage per cgroup. - `gitaly_cgroup_procs_total`, a gauge that measures the total number of diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 8cf32a6aaa3..9cc93b21ae9 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -28,6 +28,9 @@ The minimum recommended configuration for a Gitaly Cluster requires: - 3 Praefect nodes - 3 Gitaly nodes (1 primary, 2 secondary) +NOTE: +[Disk requirements](index.md#disk-requirements) apply to Gitaly nodes. + You should configure an odd number of Gitaly nodes so that transactions have a tie-breaker in case one of the Gitaly nodes fails in a mutating RPC call. @@ -43,15 +46,15 @@ default value. The default value depends on the GitLab version. Network latency for Gitaly Cluster should ideally be measurable in single-digit milliseconds. Latency is particularly important for: -- Gitaly node health checks. Nodes must be able to respond within 1 second. +- Gitaly node health checks. Nodes must be able to respond 1 second or faster. - Reference transactions that enforce [strong consistency](index.md#strong-consistency). Lower latencies mean Gitaly nodes can agree on changes faster. Achieving acceptable latency between Gitaly nodes: - On physical networks generally means high bandwidth, single location connections. -- On the cloud generally means within the same region, including allowing cross availability zone replication. These links - are designed for this type of synchronization. Latency of less than 2ms should be sufficient for Gitaly Cluster. +- On the cloud generally means in the same region, including allowing cross availability zone replication. These links + are designed for this type of synchronization. Latency of less than 2 ms should be sufficient for Gitaly Cluster. If you can't provide low network latencies for replication (for example, between distant locations), consider Geo. For more information, see [Comparison to Geo](index.md#comparison-to-geo). @@ -82,7 +85,7 @@ The requirements are relatively low because the database contains only metadata - Where repositories are located. - Some queued work. -It depends on the number of repositories, but a useful minimum is 5-10 GB, similar to the main +It depends on the number of repositories, but a good minimum is 5-10 GB, similar to the main GitLab application database. ## Setup Instructions @@ -103,7 +106,7 @@ If you [installed](https://about.gitlab.com/install/) GitLab using the Omnibus G Before beginning, you should already have a working GitLab instance. [Learn how to install GitLab](https://about.gitlab.com/install/). -Provision a PostgreSQL server. We recommend using the PostgreSQL that is shipped +Provision a PostgreSQL server. You should use the PostgreSQL that is shipped with Omnibus GitLab and use it to configure the PostgreSQL database. You can use an external PostgreSQL server (version 11 or newer) but you must set it up [manually](#manual-database-setup). @@ -1453,7 +1456,7 @@ To migrate existing clusters: 1. On the Praefect nodes, configure the election strategy in `/etc/gitlab/gitlab.rb` with `praefect['failover_election_strategy'] = 'per_repository'`. - 1. Run `gitlab-ctl reconfigure && gitlab-ctl start` to reconfigure and start the Praefects. + 1. Run `gitlab-ctl reconfigure && gitlab-ctl start` to reconfigure and start the Praefect nodes. - If downtime is unacceptable: diff --git a/doc/administration/gitaly/reference.md b/doc/administration/gitaly/reference.md index 8f7dc688e56..cec18960dfd 100644 --- a/doc/administration/gitaly/reference.md +++ b/doc/administration/gitaly/reference.md @@ -57,7 +57,7 @@ an empty string. It is possible to temporarily disable authentication with the `transitioning` setting. This allows you to monitor if all clients are authenticating correctly without causing a service outage for clients -that are not configured correctly yet: +that are still to be configured correctly: ```toml [auth] @@ -159,7 +159,7 @@ sum(rate(gitaly_catfile_cache_total{type="hit"}[5m])) / sum(rate(gitaly_catfile_ ### `gitaly-ruby` A Gitaly process uses one or more `gitaly-ruby` helper processes to -execute RPC's implemented in Ruby instead of Go. The `[gitaly-ruby]` +execute RPCs implemented in Ruby instead of Go. The `[gitaly-ruby]` section of the configuration file contains settings for these helper processes. These processes are known to occasionally suffer from memory leaks. @@ -169,7 +169,7 @@ Gitaly restarts its `gitaly-ruby` helpers when their memory exceeds the | Name | Type | Required | Description | | ---- | ---- | -------- | ----------- | | `dir` | string | yes | Path to where `gitaly-ruby` is installed (needed to boot the process).| -| `max_rss` | integer | no | Resident set size limit that triggers a `gitaly-ruby` restart, in bytes. Default is `200000000` (200MB). | +| `max_rss` | integer | no | Resident set size limit that triggers a `gitaly-ruby` restart, in bytes. Default is `200000000` (200 MB). | | `graceful_restart_timeout` | string | no | Grace period before a `gitaly-ruby` process is forcibly terminated after exceeding `max_rss`. Default is `10m` (10 minutes).| | `restart_delay` | string | no |Time that `gitaly-ruby` memory must remain high before a restart. Default is `5m` (5 minutes).| | `num_workers` | integer | no |Number of `gitaly-ruby` worker processes. Try increasing this number in case of `ResourceExhausted` errors. Default is `2`, minimum is `2`.| diff --git a/doc/administration/gitaly/troubleshooting.md b/doc/administration/gitaly/troubleshooting.md index 7edce840396..7f5d4b9e443 100644 --- a/doc/administration/gitaly/troubleshooting.md +++ b/doc/administration/gitaly/troubleshooting.md @@ -245,6 +245,28 @@ the application might be fetching this secret from a different file. Your Gitaly If that setting is missing, GitLab defaults to using `.gitlab_shell_secret` under `/opt/gitlab/embedded/service/gitlab-rails/.gitlab_shell_secret`. +### Repository pushes fail + +When attempting `git push`, you can see: + +- `401 Unauthorized` errors. +- The following in server logs: + + ```json + { + ... + "exception.class":"JWT::VerificationError", + "exception.message":"Signature verification raised", + ... + } + ``` + +This error occurs when the GitLab server has been upgraded to GitLab 15.5 or later but Gitaly has not yet been upgraded. + +From GitLab 15.5, GitLab [authenticates with GitLab Shell using a JWT token instead of a shared secret](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86148). +You should follow the [recommendations on upgrading external Gitaly](../../update/plan_your_upgrade.md#external-gitaly) and upgrade Gitaly before the GitLab +server. + ### Repository pushes fail with a `deny updating a hidden ref` error Due to [a change](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3426) @@ -456,6 +478,20 @@ in sync so the token check succeeds. This check helps identify the root cause of `permission denied` [errors being logged by Praefect](#permission-denied-errors-appearing-in-gitaly-or-praefect-logs-when-accessing-repositories). +For offline environments where access to public [`pool.ntp.org`](https://pool.ntp.org) servers is not possible, the Praefect `check` sub-command fails this +check with an error message similar to: + +```plaintext +checking with NTP service at and allowed clock drift 60000ms [correlation_id: <XXX>] +Failed (fatal) error: gitaly node at tcp://[gitlab.example-instance.com]:8075: rpc error: code = DeadlineExceeded desc = context deadline exceeded +``` + +To resolve this issue, set an environment variable on all Praefect servers to point to an accessible internal NTP server. For example: + +```shell +export NTP_HOST=ntp.example.com +``` + ### Praefect errors in logs If you receive an error, check `/var/log/gitlab/gitlab-rails/production.log`. @@ -608,7 +644,7 @@ Is [some cases](index.md#known-issues) the Praefect database can get out of sync a given repository is fully synced on all nodes, run the [`gitlab:praefect:replicas` Rake task](../raketasks/praefect.md#replica-checksums) that checksums the repository on all Gitaly nodes. -The [Praefect dataloss](recovery.md#check-for-data-loss) command only checks the state of the repository in the Praefect database, and cannot +The [Praefect `dataloss`](recovery.md#check-for-data-loss) command only checks the state of the repository in the Praefect database, and cannot be relied to detect sync problems in this scenario. ### Relation does not exist errors diff --git a/doc/administration/housekeeping.md b/doc/administration/housekeeping.md index 2d3e937e047..584f06ef537 100644 --- a/doc/administration/housekeeping.md +++ b/doc/administration/housekeeping.md @@ -45,11 +45,12 @@ be slow. ### Heuristical housekeeping -> - [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/2634) in GitLab 14.9 for the [manual trigger](#manual-trigger) and the [push-based trigger](#push-based-trigger) [with a flag](feature_flags.md) named `optimized_housekeeping`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/2634) in GitLab 14.9 for the [manual trigger](#manual-trigger) and the [push-based trigger](#push-based-trigger) [with a flag](feature_flags.md) named `optimized_housekeeping`. Enabled by default. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/353607) in GitLab 14.10. FLAG: -On self-managed GitLab, by default this feature is not available for the [manual trigger](#manual-trigger) and the [push-based trigger](#push-based-trigger). +On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](feature_flags.md) named `optimize_repository`. + To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named `optimized_housekeeping`. The heuristical (or "opportunistic") housekeeping strategy analyzes the @@ -76,9 +77,22 @@ based on the size of the repository: Gitaly does this to offset the fact that optimizing those data structures takes more time the bigger they get. It is especially important in large -monorepositories (which receive a lot of traffic) to avoid optimizing them too +monorepos (which receive a lot of traffic) to avoid optimizing them too frequently. +You can change how often Gitaly is asked to optimize a repository. + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > Repository**. +1. Expand **Repository maintenance**. +1. In the **Housekeeping** section, configure the housekeeping options. +1. Select **Save changes**. + +- **Enable automatic repository housekeeping**: Regularly ask Gitaly to run repository optimization. If you + keep this setting disabled for a long time, Git repository access on your GitLab server becomes + slower and your repositories use more disk space. +- **Optimize repository period**: Number of Git pushes after which Gitaly is asked to optimize a repository. + ## Running housekeeping tasks There are different ways in which GitLab runs housekeeping tasks: @@ -108,8 +122,13 @@ To trigger housekeeping tasks manually: This starts an asynchronous background worker for the project's repository. The background worker executes `git gc`, which performs a number of optimizations. +<!--- start_remove The following content will be removed on remove_date: '2023-04-22' --> + ### Push-based trigger +FLAG: +On self-managed GitLab, by default this feature is not available and superseded by [heuristical housekeeping](#heuristical-housekeeping). It is planned to be removed in 15.8. To enable the feature, ask an administrator to [disable the feature flag](feature_flags.md) named `optimize_repository`. + GitLab automatically runs repository housekeeping tasks after a configured number of pushes: @@ -208,7 +227,7 @@ of a repository. When creating the first fork, we: 1. Create an object pool repository that contains all objects of the repository that is about to be forked. -1. Link the repository to this new object pool via Git's altenates mechanism. +1. Link the repository to this new object pool via Git's alternates mechanism. 1. Repack the repository so that it uses objects from the object pool. It thus can drop its own copy of the objects. diff --git a/doc/administration/img/audit_events_v14_5.png b/doc/administration/img/audit_events_v14_5.png Binary files differdeleted file mode 100644 index 57190463d05..00000000000 --- a/doc/administration/img/audit_events_v14_5.png +++ /dev/null diff --git a/doc/administration/img/impersonated_audit_events_v13_8.png b/doc/administration/img/impersonated_audit_events_v13_8.png Binary files differdeleted file mode 100644 index 0a8548d515d..00000000000 --- a/doc/administration/img/impersonated_audit_events_v13_8.png +++ /dev/null diff --git a/doc/administration/img/impersonated_audit_events_v15_7.png b/doc/administration/img/impersonated_audit_events_v15_7.png Binary files differnew file mode 100644 index 00000000000..ef7decd984d --- /dev/null +++ b/doc/administration/img/impersonated_audit_events_v15_7.png diff --git a/doc/administration/inactive_project_deletion.md b/doc/administration/inactive_project_deletion.md index 88d8e3fc648..ea5658bef84 100644 --- a/doc/administration/inactive_project_deletion.md +++ b/doc/administration/inactive_project_deletion.md @@ -12,7 +12,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Administrators of large GitLab instances can find that over time, projects become inactive and are no longer used. These projects take up unnecessary disk space. With inactive project deletion, you can identify these projects, warn the maintainers ahead of time, and then delete the projects if they remain inactive. When an inactive project is -deleted, the action generates an audit event that it was performed by the first active administrator. +deleted, the action generates an audit event that it was performed by the @GitLab-Admin-Bot. For the default setting on GitLab.com, see the [GitLab.com settings page](../user/gitlab_com/index.md#inactive-project-deletion). diff --git a/doc/administration/incoming_email.md b/doc/administration/incoming_email.md index 433956bb066..826340ad967 100644 --- a/doc/administration/incoming_email.md +++ b/doc/administration/incoming_email.md @@ -10,7 +10,7 @@ GitLab has several features based on receiving incoming email messages: - [Reply by Email](reply_by_email.md): allow GitLab users to comment on issues and merge requests by replying to notification email. -- [New issue by email](../user/project/issues/managing_issues.md#by-sending-an-email): +- [New issue by email](../user/project/issues/create_issues.md#by-sending-an-email): allow GitLab users to create a new issue by sending an email to a user-specific email address. - [New merge request by email](../user/project/merge_requests/creating_merge_requests.md#by-sending-an-email): diff --git a/doc/administration/index.md b/doc/administration/index.md index 95db2b7a2e0..1059424da27 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -121,6 +121,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Creating users](../user/profile/account/create_accounts.md): Create users manually or through authentication integrations. - [Libravatar](libravatar.md): Use Libravatar instead of Gravatar for user avatars. - [Sign-up restrictions](../user/admin_area/settings/sign_up_restrictions.md): block email addresses of specific domains, or allow only specific domains. +- [Admin mode](../user/admin_area/settings/sign_in_restrictions.md#admin-mode): require that administrators authenticate separately to use administrative access, like `sudo`. - [Access restrictions](../user/admin_area/settings/visibility_and_access_controls.md#configure-enabled-git-access-protocols): Define which Git access protocols can be used to talk to GitLab (SSH, HTTP, HTTPS). - [Authentication and Authorization](auth/index.md): Configure external authentication with LDAP, SAML, CAS, and additional providers. - [Sync LDAP](auth/ldap/index.md) @@ -133,7 +134,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - Instances. - [Auditor users](auditor_users.md): Users with read-only access to all projects, groups, and other resources on the GitLab instance. - [Incoming email](incoming_email.md): Configure incoming emails to allow - users to [reply by email](reply_by_email.md), create [issues by email](../user/project/issues/managing_issues.md#by-sending-an-email) and + users to [reply by email](reply_by_email.md), create [issues by email](../user/project/issues/create_issues.md#by-sending-an-email) and [merge requests by email](../user/project/merge_requests/creating_merge_requests.md#by-sending-an-email), and to enable [Service Desk](../user/project/service_desk.md). - [Postfix for incoming email](reply_by_email_postfix_setup.md): Set up a basic Postfix mail server with IMAP authentication on Ubuntu for incoming @@ -208,7 +209,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. ## Troubleshooting - [Log system](logs/index.md): Where to look for logs. -- [Sidekiq Troubleshooting](troubleshooting/sidekiq.md): Debug when Sidekiq appears hung and is not processing jobs. +- [Sidekiq Troubleshooting](sidekiq/sidekiq_troubleshooting.md): Debug when Sidekiq appears hung and is not processing jobs. - [Navigating GitLab via Rails console](operations/rails_console.md) - [GitLab application limits](instance_limits.md) - [Responding to security incidents](../security/responding_to_security_incidents.md) diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 546c4667220..59746fc0f07 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -378,7 +378,7 @@ and to limit memory consumption. When using offset-based pagination in the REST API, there is a limit to the maximum requested offset into the set of results. This limit is only applied to endpoints that -support keyset-based pagination. More information about pagination options can be +also support keyset-based pagination. More information about pagination options can be found in the [API documentation section on pagination](../api/index.md#pagination). To set this limit for a self-managed installation, run the following in the @@ -581,7 +581,8 @@ limit value. For example, for a maximum frequency of: - Once per 10 minutes, the limit must be `144`. - Once per 60 minutes, the limit must be `24` -There is no limit for self-managed instances by default. +The minimum value is `24`, or one pipeline per 60 minutes. +There is no maximum value. To set this limit to `1440` on a self-managed installation, run the following in the [GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): @@ -672,6 +673,7 @@ setting is used: | `ci_max_artifact_size_network_referee` | 0 | | `ci_max_artifact_size_performance` | 0 | | `ci_max_artifact_size_requirements` | 0 | +| `ci_max_artifact_size_requirements_v2` | 0 | | `ci_max_artifact_size_sast` | 0 | | `ci_max_artifact_size_secret_detection` | 0 | | `ci_max_artifact_size_terraform` | 5 MB ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37018) in GitLab 13.3) | @@ -818,9 +820,9 @@ This limit is [enabled on GitLab.com](../user/gitlab_com/index.md#gitlab-cicd). You can set a limit on the maximum size of a dotenv artifact. This limit is checked every time a dotenv file is exported as an artifact. -Set the limit to `0` to disable it. Defaults to 5KB. +Set the limit to `0` to disable it. Defaults to 5 KB. -To set this limit to 5KB on a self-managed installation, run the following in the +To set this limit to 5 KB on a self-managed installation, run the following in the [GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): ```ruby diff --git a/doc/administration/integration/terminal.md b/doc/administration/integration/terminal.md index 14da70f6efb..a1522ccac31 100644 --- a/doc/administration/integration/terminal.md +++ b/doc/administration/integration/terminal.md @@ -41,7 +41,7 @@ In brief: - The WebSocket is handled in [Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse), rather than the Rails application server. - Workhorse queries Rails for connection details and user permissions. Rails - queries Kubernetes for them in the background using [Sidekiq](../troubleshooting/sidekiq.md). + queries Kubernetes for them in the background using [Sidekiq](../sidekiq/sidekiq_troubleshooting.md). - Workhorse acts as a proxy server between the user's browser and the Kubernetes API, passing WebSocket frames between the two. - Workhorse regularly polls Rails, terminating the WebSocket connection if the @@ -64,8 +64,8 @@ detail below. ## Enabling and disabling terminal support NOTE: -AWS Classic Load Balancers (CLBs) do not support web sockets. -If you want web terminals to work, use AWS Network Load Balancers (NLBs). +AWS Classic Load Balancers do not support web sockets. +If you want web terminals to work, use AWS Network Load Balancers. Read [AWS Elastic Load Balancing Product Comparison](https://aws.amazon.com/elasticloadbalancing/features/#compare) for more information. diff --git a/doc/administration/issue_closing_pattern.md b/doc/administration/issue_closing_pattern.md index e9150ae0650..bdbcdd0093c 100644 --- a/doc/administration/issue_closing_pattern.md +++ b/doc/administration/issue_closing_pattern.md @@ -17,36 +17,93 @@ in the project's default branch. ## Change the issue closing pattern -To change the pattern, you must have access to the server that GitLab -is installed on. - -The default pattern can be located in [`gitlab.yml.example`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/gitlab.yml.example) -under the "Automatic issue closing" section. +The [default issue closing pattern](../user/project/issues/managing_issues.md#default-closing-pattern) +covers a wide range of words. You can change the pattern to suit your needs. NOTE: You are advised to use <https://rubular.com> to test the issue closing pattern. -Because Rubular doesn't understand `%{issue_ref}`, you can replace this by +However, since Rubular doesn't understand `%{issue_ref}`, you can replace this by `#\d+` when testing your patterns, which matches only local issue references like `#123`. -**For Omnibus installations** +To change the default issue closing pattern: + +::Tabs -1. Open `/etc/gitlab/gitlab.rb` with your editor. -1. Change the value of `gitlab_rails['gitlab_issue_closing_pattern']` to a regular - expression of your liking: +:::TabTitle Linux package (Omnibus) + +1. Edit `/etc/gitlab/gitlab.rb` and change the `gitlab_rails['gitlab_issue_closing_pattern']` + value: ```ruby - gitlab_rails['gitlab_issue_closing_pattern'] = /\b((?:[Cc]los(?:e[sd]?|ing)|\b[Ff]ix(?:e[sd]|ing)?|\b[Rr]esolv(?:e[sd]?|ing)|\b[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?: *,? +and +| *,? *)?)|([A-Z][A-Z0-9_]+-\d+))+)/.source + gitlab_rails['gitlab_issue_closing_pattern'] = /<regular_expression>/.source + ``` + +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml` and change the `issueClosingPattern` value: + + ```yaml + global: + appConfig: + issueClosingPattern: "<regular_expression>" ``` -1. [Reconfigure](restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. +1. Save the file and apply the new values: -**For installations from source** + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml` and change the `gitlab_rails['gitlab_issue_closing_pattern']` + value: + + ```yaml + version: "3.6" + services: + gitlab: + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['gitlab_issue_closing_pattern'] = /<regular_expression>/.source + ``` + +1. Save the file and restart GitLab: -1. Open `gitlab.yml` with your editor. -1. Change the value of `issue_closing_pattern`: + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) + +1. Edit `/home/git/gitlab/config/gitlab.yml` and change the `issue_closing_pattern` value: ```yaml - issue_closing_pattern: "\b((?:[Cc]los(?:e[sd]?|ing)|\b[Ff]ix(?:e[sd]|ing)?|\b[Rr]esolv(?:e[sd]?|ing)|\b[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?: *,? +and +| *,? *)?)|([A-Z][A-Z0-9_]+-\d+))+)" + production: &base + gitlab: + issue_closing_pattern: "<regular_expression>" + ``` + +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart ``` -1. [Restart](restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. +::EndTabs diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index d79f322c3f5..fb2f73876e8 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -100,8 +100,7 @@ In a multi-server setup you must use one of the options to #### Object Storage Settings -NOTE: -In GitLab 13.2 and later, we recommend using the +In GitLab 13.2 and later, you should use the [consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). This section describes the earlier configuration format. @@ -235,7 +234,7 @@ by the `gitlab:artifacts:migrate` Rake task. To migrate back to local storage: 1. Run `gitlab-rake gitlab:artifacts:migrate_to_local`. -1. Disable object_storage for artifacts in `gitlab.rb`: +1. Disable object storage for artifacts in `gitlab.rb`: - Set `gitlab_rails['artifacts_object_store_enabled'] = false`. - Comment out all other `artifacts_object_store` settings, including the entire `artifacts_object_store_connection` section, including the closing `}`. @@ -636,24 +635,6 @@ If you need to manually remove **all** job artifacts associated with multiple jo - `3.months.ago` - `1.year.ago` -### Error `Downloading artifacts from coordinator... not found` - -When a job attempts to download artifacts from an earlier job, you might receive an error message similar to: - -```plaintext -Downloading artifacts from coordinator... not found id=12345678 responseStatus=404 Not Found -``` - -This can be caused by a `gitlab.rb` file with the following configuration: - -```ruby -gitlab_rails['artifacts_object_store_background_upload'] = false -gitlab_rails['artifacts_object_store_direct_upload'] = true -``` - -To prevent this, comment out or remove those lines, or switch to their [default values](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-config-template/gitlab.rb.template), and -then run `sudo gitlab-ctl reconfigure`. - ### Job artifact upload fails with error 500 If you are using object storage for artifacts and a job artifact fails to upload, diff --git a/doc/administration/job_logs.md b/doc/administration/job_logs.md index 5aa0e8f3948..9b25c70716b 100644 --- a/doc/administration/job_logs.md +++ b/doc/administration/job_logs.md @@ -158,7 +158,7 @@ To eliminate both file system requirements: The data flow is the same as described in the [data flow section](#data-flow) with one change: _the stored path of the first two phases is different_. This incremental log architecture stores chunks of logs in Redis and a persistent store (object storage or database) instead of -file storage. Redis is used as first-class storage, and it stores up-to 128KB +file storage. Redis is used as first-class storage, and it stores up-to 128 KB of data. After the full chunk is sent, it is flushed to a persistent store, either object storage (temporary directory) or database. After a while, the data in Redis and a persistent store is archived to [object storage](#uploading-logs-to-object-storage). @@ -169,7 +169,7 @@ Here is the detailed data flow: 1. The runner picks a job from GitLab 1. The runner sends a piece of log to GitLab 1. GitLab appends the data to Redis -1. After the data in Redis reaches 128KB, the data is flushed to a persistent store (object storage or the database). +1. After the data in Redis reaches 128 KB, the data is flushed to a persistent store (object storage or the database). 1. The above steps are repeated until the job is finished. 1. After the job is finished, GitLab schedules a Sidekiq worker to archive the log. 1. The Sidekiq worker archives the log to object storage and cleans up the log diff --git a/doc/administration/lfs/index.md b/doc/administration/lfs/index.md index 8fdc98bd12a..cf80b05a5e0 100644 --- a/doc/administration/lfs/index.md +++ b/doc/administration/lfs/index.md @@ -7,7 +7,8 @@ disqus_identifier: 'https://docs.gitlab.com/ee/workflow/lfs/lfs_administration.h # GitLab Git Large File Storage (LFS) Administration **(FREE SELF)** -Documentation about how to use Git LFS are under [Managing large binary files with Git LFS doc](../../topics/git/lfs/index.md). +This page contains information about configuring Git LFS in self-managed GitLab instances. +For user documentation about Git LFS, see [Git Large File Storage](../../topics/git/lfs/index.md). LFS is enabled in GitLab self-managed instances by default. @@ -62,27 +63,17 @@ to check which storage services can be integrated with GitLab. You can also use external object storage in a private local network. For example, [MinIO](https://min.io/) is a standalone object storage service that works with GitLab instances. -GitLab provides two different options for the uploading mechanism: "Direct upload" and "Background upload". - [Read more about using object storage with GitLab](../object_storage.md). NOTE: -In GitLab 13.2 and later, we recommend using the +In GitLab 13.2 and later, you should use the [consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). -This section describes the earlier configuration format. - -**Option 1. Direct upload** +This section describes the earlier configuration format. [Migration steps still apply](#migrating-to-object-storage). 1. User pushes an `lfs` file to the GitLab instance. 1. GitLab-workhorse uploads the file directly to the external object storage. 1. GitLab-workhorse notifies GitLab-rails that the upload process is complete. -**Option 2. Background upload** - -1. User pushes an `lfs` file to the GitLab instance. -1. GitLab-rails stores the file in the local file storage. -1. GitLab-rails then uploads the file to the external object storage asynchronously. - The following general settings are supported. | Setting | Description | Default | @@ -119,9 +110,7 @@ On Omnibus GitLab installations, the settings are prefixed by `lfs_object_store_ 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. [Migrate any existing local LFS objects to the object storage](#migrating-to-object-storage). - New LFS objects - are forwarded to object storage unless - `gitlab_rails['lfs_object_store_background_upload']` and `gitlab_rails['lfs_object_store_direct_upload']` is set to `false`. + New LFS objects are forwarded to object storage. ### S3 for installations from source @@ -150,9 +139,7 @@ For source installations the settings are nested under `lfs:` and then 1. Save the file, and then [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. 1. [Migrate any existing local LFS objects to the object storage](#migrating-to-object-storage). - New LFS objects - are forwarded to object storage unless - `background_upload` and `direct_upload` is set to `false`. + New LFS objects are forwarded to object storage. ### Migrating to object storage @@ -231,6 +218,12 @@ You can see the total storage used for LFS objects on groups and projects: - In the administration area. - In the [groups](../../api/groups.md) and [projects APIs](../../api/projects.md). +## Related topics + +- Blog post: [Getting started with Git LFS](https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/) +- User documentation: [Git Large File Storage (LFS)](../../topics/git/lfs/index.md) +- [Git LFS developer information](../../development/lfs.md) + ## Troubleshooting ### Missing LFS objects @@ -276,39 +269,6 @@ To delete these references: lfs_object.destroy ``` -### `Google::Apis::TransmissionError: execution expired` - -If LFS integration is configured with Google Cloud Storage and background uploads (`background_upload: true` and `direct_upload: false`), -Sidekiq workers may encounter this error. This is because the uploading timed out with very large files. -LFS files up to 6 GB can be uploaded without any extra steps, otherwise you need to use the following workaround. - -Sign in to Rails console: - -```shell -sudo gitlab-rails console -``` - -Set up timeouts: - -- These settings are only in effect for the same session. For example, they are not effective for Sidekiq workers. -- 20 minutes (1200 sec) is enough to upload 30GB LFS files: - -```ruby -::Google::Apis::ClientOptions.default.open_timeout_sec = 1200 -::Google::Apis::ClientOptions.default.read_timeout_sec = 1200 -::Google::Apis::ClientOptions.default.send_timeout_sec = 1200 -``` - -Upload LFS files manually (this process does not use Sidekiq at all): - -```ruby -LfsObject.where(file_store: [nil, 1]).find_each do |lfs_object| - lfs_object.file.migrate!(ObjectStorage::Store::REMOTE) if lfs_object.file.file.exists? -end -``` - -See more information in [!19581](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19581) - ### LFS commands fail on TLS v1.3 server If you configure GitLab to [disable TLS v1.2](https://docs.gitlab.com/omnibus/settings/nginx.html) @@ -332,6 +292,28 @@ To check an installed Git LFS client's version, run this command: git lfs version ``` +## Error viewing a PDF file + +When LFS has been configured with object storage and `proxy_download` set to +`false`, [you may see an error when previewing a PDF file from the Web browser](https://gitlab.com/gitlab-org/gitlab/-/issues/248100): + +```plaintext +An error occurred while loading the file. Please try again later. +``` + +This occurs due to Cross-Origin Resource Sharing (CORS) restrictions: +the browser attempts to load the PDF from object storage, but the object +storage provider rejects the request because the GitLab domain differs +from the object storage domain. + +To fix this issue, configure your object storage provider's CORS +settings to allow the GitLab domain. See the following documentation +for more details: + +1. [AWS S3](https://aws.amazon.com/premiumsupport/knowledge-center/s3-configure-cors/) +1. [Google Cloud Storage](https://cloud.google.com/storage/docs/configuring-cors) +1. [Azure Storage](https://learn.microsoft.com/en-us/rest/api/storageservices/cross-origin-resource-sharing--cors--support-for-the-azure-storage-services). + ## Known limitations - Only compatible with the Git LFS client versions 1.1.0 and later, or 1.0.2. diff --git a/doc/administration/logs/index.md b/doc/administration/logs/index.md index b0631c52a47..9893c6935a3 100644 --- a/doc/administration/logs/index.md +++ b/doc/administration/logs/index.md @@ -6,9 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Log system **(FREE SELF)** -GitLab has an advanced log system where everything is logged, so you -can analyze your instance using various system log files. In addition to -system log files, GitLab Enterprise Edition provides [Audit Events](../audit_events.md). +GitLab has an advanced log system where everything is logged, so you can analyze your instance using various system log +files. The log system is similar to [audit events](../audit_events.md). System log files are typically plain text in a standard log file format. This guide talks about how to read and use these system log files. @@ -86,24 +85,24 @@ except those captured by `runit`. | Log type | Managed by logrotate | Managed by svlogd/runit | |:------------------------------------------------|:------------------------|:------------------------| -| [Alertmanager Logs](#alertmanager-logs) | **{dotted-circle}** No | **{check-circle}** Yes | -| [Crond Logs](#crond-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [Alertmanager logs](#alertmanager-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [crond logs](#crond-logs) | **{dotted-circle}** No | **{check-circle}** Yes | | [Gitaly](#gitaly-logs) | **{check-circle}** Yes | **{check-circle}** Yes | | [GitLab Exporter for Omnibus](#gitlab-exporter) | **{dotted-circle}** No | **{check-circle}** Yes | -| [GitLab Pages Logs](#pages-logs) | **{check-circle}** Yes | **{check-circle}** Yes | +| [GitLab Pages logs](#pages-logs) | **{check-circle}** Yes | **{check-circle}** Yes | | GitLab Rails | **{check-circle}** Yes | **{dotted-circle}** No | -| [GitLab Shell Logs](#gitlab-shelllog) | **{check-circle}** Yes | **{dotted-circle}** No | -| [Grafana Logs](#grafana-logs) | **{dotted-circle}** No | **{check-circle}** Yes | -| [LogRotate Logs](#logrotate-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [GitLab Shell logs](#gitlab-shelllog) | **{check-circle}** Yes | **{dotted-circle}** No | +| [Grafana logs](#grafana-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [LogRotate logs](#logrotate-logs) | **{dotted-circle}** No | **{check-circle}** Yes | | [Mailroom](#mail_room_jsonlog-default) | **{check-circle}** Yes | **{check-circle}** Yes | | [NGINX](#nginx-logs) | **{check-circle}** Yes | **{check-circle}** Yes | -| [PostgreSQL Logs](#postgresql-logs) | **{dotted-circle}** No | **{check-circle}** Yes | -| [Praefect Logs](#praefect-logs) | **{dotted-circle}** Yes | **{check-circle}** Yes | -| [Prometheus Logs](#prometheus-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [PostgreSQL logs](#postgresql-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [Praefect logs](#praefect-logs) | **{dotted-circle}** Yes | **{check-circle}** Yes | +| [Prometheus logs](#prometheus-logs) | **{dotted-circle}** No | **{check-circle}** Yes | | [Puma](#puma-logs) | **{check-circle}** Yes | **{check-circle}** Yes | -| [Redis Logs](#redis-logs) | **{dotted-circle}** No | **{check-circle}** Yes | -| [Registry Logs](#registry-logs) | **{dotted-circle}** No | **{check-circle}** Yes | -| [Workhorse Logs](#workhorse-logs) | **{check-circle}** Yes | **{check-circle}** Yes | +| [Redis logs](#redis-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [Registry logs](#registry-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [Workhorse logs](#workhorse-logs) | **{check-circle}** Yes | **{check-circle}** Yes | ## `production_json.log` @@ -161,10 +160,14 @@ seconds: - `gitaly_duration_s`: Total time by Gitaly calls - `gitaly_calls`: Total number of calls made to Gitaly - `redis_calls`: Total number of calls made to Redis +- `redis_cross_slot_calls`: Total number of cross-slot calls made to Redis +- `redis_allowed_cross_slot_calls`: Total number of allowed cross-slot calls made to Redis - `redis_duration_s`: Total time to retrieve data from Redis - `redis_read_bytes`: Total bytes read from Redis - `redis_write_bytes`: Total bytes written to Redis - `redis_<instance>_calls`: Total number of calls made to a Redis instance +- `redis_<instance>_cross_slot_calls`: Total number of cross-slot calls made to a Redis instance +- `redis_<instance>_allowed_cross_slot_calls`: Total number of allowed cross-slot calls made to a Redis instance - `redis_<instance>_duration_s`: Total time to retrieve data from a Redis instance - `redis_<instance>_read_bytes`: Total bytes read from a Redis instance - `redis_<instance>_write_bytes`: Total bytes written to a Redis instance @@ -490,7 +493,7 @@ are logged to this file. For example: } ``` -## Sidekiq Logs +## Sidekiq logs NOTE: In Omnibus GitLab `12.10` or earlier, the Sidekiq log is at `/var/log/gitlab/gitlab-rails/sidekiq.log`. @@ -670,7 +673,7 @@ I, [2015-02-13T06:17:00.679433 #9291] INFO -- : Moving existing hooks directory User clone/fetch activity using SSH transport appears in this log as `executing git command <gitaly-upload-pack...`. -## Gitaly Logs +## Gitaly logs This file is in `/var/log/gitlab/gitaly/current` and is produced by [runit](http://smarden.org/runit/). `runit` is packaged with Omnibus GitLab and a brief explanation of its purpose @@ -697,7 +700,7 @@ This file is at `/var/log/gitlab/gitaly/gitaly_hooks.log` and is produced by `gitaly-hooks` command. It also contains records about failures received during processing of the responses from GitLab API. -## Puma Logs +## Puma logs ### `puma_stdout.log` @@ -980,11 +983,11 @@ can be used. } ``` -## Registry Logs +## Registry logs For Omnibus GitLab installations, Container Registry logs are in `/var/log/gitlab/registry/current`. -## NGINX Logs +## NGINX logs For Omnibus GitLab installations, NGINX logs are in: @@ -1003,7 +1006,7 @@ Below is the default GitLab NGINX access log format: $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ``` -## Pages Logs +## Pages logs For Omnibus GitLab installations, Pages logs are in `/var/log/gitlab/gitlab-pages/current`. @@ -1032,50 +1035,50 @@ For example: } ``` -## Mattermost Logs +## Mattermost logs For Omnibus GitLab installations, Mattermost logs are in these locations: - `/var/log/gitlab/mattermost/mattermost.log` - `/var/log/gitlab/mattermost/current` -## Workhorse Logs +## Workhorse logs For Omnibus GitLab installations, Workhorse logs are in `/var/log/gitlab/gitlab-workhorse/current`. -## PostgreSQL Logs +## PostgreSQL logs For Omnibus GitLab installations, PostgreSQL logs are in `/var/log/gitlab/postgresql/current`. -## Prometheus Logs +## Prometheus logs For Omnibus GitLab installations, Prometheus logs are in `/var/log/gitlab/prometheus/current`. -## Redis Logs +## Redis logs For Omnibus GitLab installations, Redis logs are in `/var/log/gitlab/redis/current`. -## Alertmanager Logs +## Alertmanager logs For Omnibus GitLab installations, Alertmanager logs are in `/var/log/gitlab/alertmanager/current`. <!-- vale gitlab.Spelling = NO --> -## Crond Logs +## crond logs For Omnibus GitLab installations, crond logs are in `/var/log/gitlab/crond/`. <!-- vale gitlab.Spelling = YES --> -## Grafana Logs +## Grafana logs For Omnibus GitLab installations, Grafana logs are in `/var/log/gitlab/grafana/current`. -## LogRotate Logs +## LogRotate logs For Omnibus GitLab installations, `logrotate` logs are in `/var/log/gitlab/logrotate/current`. -## GitLab Monitor Logs +## GitLab Monitor logs For Omnibus GitLab installations, GitLab Monitor logs are in `/var/log/gitlab/gitlab-monitor/`. @@ -1088,7 +1091,7 @@ For Omnibus GitLab installations, GitLab Exporter logs are in `/var/log/gitlab/g For Omnibus GitLab installations, GitLab agent server logs are in `/var/log/gitlab/gitlab-kas/current`. -## Praefect Logs +## Praefect logs For Omnibus GitLab installations, Praefect logs are in `/var/log/gitlab/praefect/`. diff --git a/doc/administration/maintenance_mode/index.md b/doc/administration/maintenance_mode/index.md index 9adb8ce2cd9..c30465b4df9 100644 --- a/doc/administration/maintenance_mode/index.md +++ b/doc/administration/maintenance_mode/index.md @@ -84,7 +84,7 @@ them to disable Maintenance Mode after it's been enabled. All users can sign in and out of the GitLab instance but no new users can be created. -If there are [LDAP syncs](../auth/ldap/index.md) scheduled for that time, they fail since user creation is disabled. Similarly, [user creations based on SAML](../../integration/saml.md#general-setup) fail. +If there are [LDAP syncs](../auth/ldap/index.md) scheduled for that time, they fail since user creation is disabled. Similarly, [user creations based on SAML](../../integration/saml.md#configure-saml-support-in-gitlab) fail. ### Git actions @@ -178,7 +178,7 @@ To monitor queues and disable jobs: ### Incident management -[Incident management](../../operations/incident_management/index.md) functions are limited. The creation of [alerts](../../operations/incident_management/alerts.md) and [incidents](../../operations/incident_management/incidents.md#incident-creation) are paused entirely. Notifications and paging on alerts and incidents are therefore disabled. +[Incident management](../../operations/incident_management/index.md) functions are limited. The creation of [alerts](../../operations/incident_management/alerts.md) and [incidents](../../operations/incident_management/manage_incidents.md#create-an-incident) are paused entirely. Notifications and paging on alerts and incidents are therefore disabled. ### Feature flags diff --git a/doc/administration/merge_request_diffs.md b/doc/administration/merge_request_diffs.md index 4dec9e52c19..25dba00beb9 100644 --- a/doc/administration/merge_request_diffs.md +++ b/doc/administration/merge_request_diffs.md @@ -104,8 +104,7 @@ be configured already. ### Object Storage Settings -NOTE: -In GitLab 13.2 and later, we recommend using the +In GitLab 13.2 and later, you should use the [consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). This section describes the earlier configuration format. diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md index 35dc64a0594..566bc070347 100644 --- a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md +++ b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md @@ -101,7 +101,7 @@ You can add custom metrics in the self-monitoring project by: A [bug](https://gitlab.com/gitlab-org/gitlab/-/issues/208676) causes project creation to fail with the following error in the log file when the first administrator user is an -[external user](../../../user/permissions.md#external-users): +[external user](../../../user/admin_area/external_users.md): ```plaintext Could not create instance administrators group. Errors: ["You don't have permission to create groups."] @@ -116,6 +116,6 @@ User.admins.active.first.external? If this returns true, the first administrator user is an external user. If you face this issue, you can temporarily -[make the administrator user a non-external user](../../../user/permissions.md#external-users) +[make the administrator user a non-external user](../../../user/admin_area/external_users.md) and then try to create the project. After the project is created, the administrator user can be changed back to an external user. diff --git a/doc/administration/monitoring/performance/performance_bar.md b/doc/administration/monitoring/performance/performance_bar.md index de6178a3151..171a441eaec 100644 --- a/doc/administration/monitoring/performance/performance_bar.md +++ b/doc/administration/monitoring/performance/performance_bar.md @@ -1,6 +1,6 @@ --- -stage: Monitor -group: Respond +stage: Data Stores +group: Application Performance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/administration/monitoring/prometheus/gitlab_exporter.md b/doc/administration/monitoring/prometheus/gitlab_exporter.md index bce35306505..1f4156349c5 100644 --- a/doc/administration/monitoring/prometheus/gitlab_exporter.md +++ b/doc/administration/monitoring/prometheus/gitlab_exporter.md @@ -1,6 +1,6 @@ --- -stage: Monitor -group: Respond +stage: Data Stores +group: Application Performance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index d3b28fd15bc..53ee028bc32 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -39,8 +39,9 @@ The following metrics are available: | `gitlab_cache_misses_total` | Counter | 10.2 | Cache read miss | `controller`, `action` | | `gitlab_cache_operation_duration_seconds` | Histogram | 10.2 | Cache access time | | | `gitlab_cache_operations_total` | Counter | 12.2 | Cache operations by controller or action | `controller`, `action`, `operation` | +| `gitlab_cache_read_multikey_count` | Histogram | 15.7 | Count of keys in multi-key cache read operations | `controller`, `action` | | `gitlab_ci_pipeline_builder_scoped_variables_duration` | Histogram | 14.5 | Time in seconds it takes to create the scoped variables for a CI/CD job -| `gitlab_ci_pipeline_creation_duration_seconds` | Histogram | 13.0 | Time in seconds it takes to create a CI/CD pipeline | | +| `gitlab_ci_pipeline_creation_duration_seconds` | Histogram | 13.0 | Time in seconds it takes to create a CI/CD pipeline | `gitlab` | | `gitlab_ci_pipeline_size_builds` | Histogram | 13.1 | Total number of builds within a pipeline grouped by a pipeline source | `source` | | `gitlab_ci_runner_authentication_success_total` | Counter | 15.2 | Total number of times that runner authentication has succeeded | `type` | | `gitlab_ci_runner_authentication_failure_total` | Counter | 15.2 | Total number of times that runner authentication has failed @@ -331,6 +332,16 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_dependency_proxy_blob_verification_total` | Gauge | 15.6 | Number of dependency proxy blobs verifications tried on secondary | | | `geo_dependency_proxy_blob_verified` | Gauge | 15.6 | Number of dependency proxy blobs verified on secondary | | | `geo_dependency_proxy_blob_verification_failed` | Gauge | 15.6 | Number of dependency proxy blobs verifications failed on secondary | | +| `geo_dependency_proxy_manifests` | Gauge | 15.6 | Number of dependency proxy manifests on primary | `url` | +| `geo_dependency_proxy_manifests_checksum_total` | Gauge | 15.6 | Number of dependency proxy manifests tried to checksum on primary | `url` | +| `geo_dependency_proxy_manifests_checksummed` | Gauge | 15.6 | Number of dependency proxy manifests successfully checksummed on primary | `url` | +| `geo_dependency_proxy_manifests_checksum_failed` | Gauge | 15.6 | Number of dependency proxy manifests failed to calculate the checksum on primary | `url` | +| `geo_dependency_proxy_manifests_synced` | Gauge | 15.6 | Number of syncable dependency proxy manifests synced on secondary | `url` | +| `geo_dependency_proxy_manifests_failed` | Gauge | 15.6 | Number of syncable dependency proxy manifests failed to sync on secondary | `url` | +| `geo_dependency_proxy_manifests_registry` | Gauge | 15.6 | Number of dependency proxy manifests in the registry | `url` | +| `geo_dependency_proxy_manifests_verification_total` | Gauge | 15.6 | Number of dependency proxy manifests verifications tried on secondary | `url` | +| `geo_dependency_proxy_manifests_verified` | Gauge | 15.6 | Number of dependency proxy manifests verified on secondary | `url` | +| `geo_dependency_proxy_manifests_verification_failed` | Gauge | 15.6 | Number of dependency proxy manifests verifications failed on secondary | `url` | ## Database load balancing metrics **(PREMIUM SELF)** diff --git a/doc/administration/nfs.md b/doc/administration/nfs.md index 85f35a1b188..972db315268 100644 --- a/doc/administration/nfs.md +++ b/doc/administration/nfs.md @@ -20,47 +20,14 @@ actions that read or write to Git repositories. For steps you can use to test file system performance, see [File System Performance Benchmarking](operations/filesystem_benchmarking.md). -## Gitaly and NFS deprecation +## Gitaly with NFS not supported -Starting with GitLab version 14.0, support for NFS to store Git repository data is deprecated. Technical customer support and engineering support is available for the 14.x releases. Engineering is fixing bugs and security vulnerabilities consistent with our [release and maintenance policy](../policy/maintenance.md#security-releases). +Technical and engineering support for using NFS to store Git repository data is officially at end-of-life. No product +changes or troubleshooting is provided through engineering, security or paid support channels. -Upon the release of GitLab 15.6 technical and engineering support for using NFS to store Git repository data is officially at end-of-life. There are no product changes or troubleshooting provided via Engineering, Security or Paid Support channels after the release date of 15.6, regardless of your GitLab version. - -Until the release of 15.6, for customers running 14.x releases, we continue to help with Git related tickets from customers running one or more Gitaly servers with its data stored on NFS. Examples may include: - -- Performance issues or timeouts accessing Git data -- Commits or branches vanish -- GitLab intermittently returns the wrong Git data (such as reporting that a repository has no branches) - -Assistance is limited to activities like: - -- Verifying developers' workflow uses features like protected branches -- Reviewing GitLab event data from the database to advise if it looks like a force push over-wrote branches -- Verifying that NFS client mount options match our [documented recommendations](#mount-options) -- Analyzing the GitLab Workhorse and Rails logs, and determining that `500` errors being seen in the environment are caused by slow responses from Gitaly - -GitLab support is unable to continue with the investigation if both: - -- The date of the request is on or after the release of GitLab version 15.6. -- Support Engineers and Management determine that all reasonable non-NFS root causes have been exhausted. - -If the issue is reproducible, or if it happens intermittently but regularly, GitLab Support can investigate providing the issue reproduces without the use of NFS. To reproduce without NFS, the affected repositories should be migrated to a different Gitaly shard, such as Gitaly cluster or a standalone Gitaly VM, backed with block storage. - -### Why remove NFS for Git repository data - -{:.no-toc} - -NFS is not well-suited to a workload consisting of many small files, like Git repositories. NFS does provide a number of configuration options designed to improve performance. However, over time, a number of these mount options have proven to result in inconsistencies across multiple nodes mounting the NFS volume, up to and including data loss. Addressing these inconsistencies consume extraordinary development and support engineer time that hamper our ability to develop [Gitaly Cluster](gitaly/praefect.md), our purpose-built solution to addressing the deficiencies of NFS in this environment. - -Gitaly Cluster provides highly-available Git repository storage. If this is not a requirement, single-node Gitaly backed by block storage is a suitable substitute. - -Engineering support for NFS for Git repositories is deprecated. Technical support is planned to be -unavailable from GitLab 15.0. No further enhancements are planned for this feature. - -Read: - -- [Moving beyond NFS](gitaly/index.md#moving-beyond-nfs). -- About the [correct mount options to use](#upgrade-to-gitaly-cluster-or-disable-caching-if-experiencing-data-loss). +If an issue is reproducible, or if it happens intermittently but regularly, GitLab Support can investigate providing the +issue can be reproduced without NFS. To reproduce without NFS, migrate the affected repositories to a different Gitaly +shard. For example, a Gitaly Cluster or a standalone Gitaly VM, backed with block storage. ## Known kernel version incompatibilities @@ -397,8 +364,8 @@ sudo ufw allow from <client_ip_address> to any port nfs ### Upgrade to Gitaly Cluster or disable caching if experiencing data loss WARNING: -Engineering support for NFS for Git repositories is deprecated. Read about -[moving beyond NFS](gitaly/index.md#moving-beyond-nfs). +Engineering support for NFS for Git repositories +[is unavailable](../update/removals.md#nfs-as-git-repository-storage-is-no-longer-supported). Customers and users have reported data loss on high-traffic repositories when using NFS for Git repositories. For example, we have seen: diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index d2c9c35148c..0cae46faf28 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -70,7 +70,7 @@ must be enabled, only the following providers can be used: - [Azure Blob storage](#azure-blob-storage) When consolidated object storage is used, direct upload is enabled -automatically. Background upload is not supported. For storage-specific +automatically. For storage-specific configuration, [direct upload may become the default](https://gitlab.com/gitlab-org/gitlab/-/issues/27331) because it does not require a shared folder. @@ -482,7 +482,7 @@ To migrate existing local data to object storage see the following guides: - [LFS objects](lfs/index.md#migrating-to-object-storage) - [Uploads](raketasks/uploads/migrate.md#migrate-to-object-storage) - [Merge request diffs](merge_request_diffs.md#using-object-storage) -- [Packages](packages/index.md#migrating-local-packages-to-object-storage) (optional feature) +- [Packages](packages/index.md#migrate-local-packages-to-object-storage) (optional feature) - [Dependency Proxy](packages/dependency_proxy.md#migrate-local-dependency-proxy-blobs-and-manifests-to-object-storage) - [Terraform state files](terraform_state.md#migrate-to-object-storage) - [Pages content](pages/index.md#migrate-pages-deployments-to-object-storage) @@ -536,8 +536,8 @@ supported by consolidated configuration form, refer to the following guides: | [Uploads](uploads.md#using-object-storage) | **{check-circle}** Yes | | [Container Registry](packages/container_registry.md#use-object-storage) (optional feature) | **{dotted-circle}** No | | [Merge request diffs](merge_request_diffs.md#using-object-storage) | **{check-circle}** Yes | -| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| **{dotted-circle}** No | -| [Packages](packages/index.md#using-object-storage) (optional feature) | **{check-circle}** Yes | +| [Mattermost](https://docs.mattermost.com/configure/file-storage-configuration-settings.html)| **{dotted-circle}** No | +| [Packages](packages/index.md#use-object-storage) (optional feature) | **{check-circle}** Yes | | [Dependency Proxy](packages/dependency_proxy.md#using-object-storage) (optional feature) | **{check-circle}** Yes | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | **{dotted-circle}** No | | [Terraform state files](terraform_state.md#using-object-storage) | **{check-circle}** Yes | @@ -638,9 +638,15 @@ if this access is not in place include: Received status code 403 from server: Forbidden ``` -Getting a `403 Forbidden` response is specifically called out on the -[package repository documentation](packages/index.md#using-object-storage) -as a side effect of how some build tools work. +- Object storage buckets need to allow Cross-Origin Resource Sharing + (CORS) access from the URL of the GitLab instance. Attempting to load + a PDF in the repository page may show the following error: + + ```plaintext + An error occurred while loading the file. Please try again later. + ``` + + See [the LFS documentation](lfs/index.md#error-viewing-a-pdf-file) for more details. Additionally for a short time period users could share pre-signed, time-limited object storage URLs with others without authentication. Also bandwidth charges may be incurred diff --git a/doc/administration/operations/extra_sidekiq_processes.md b/doc/administration/operations/extra_sidekiq_processes.md deleted file mode 100644 index 58858c54843..00000000000 --- a/doc/administration/operations/extra_sidekiq_processes.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../sidekiq/extra_sidekiq_processes.md' -remove_date: '2022-11-11' ---- - -This document was moved to [another location](../sidekiq/extra_sidekiq_processes.md). - -<!-- This redirect file can be deleted after <2022-11-11>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/operations/extra_sidekiq_routing.md b/doc/administration/operations/extra_sidekiq_routing.md deleted file mode 100644 index 072b6f63537..00000000000 --- a/doc/administration/operations/extra_sidekiq_routing.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../sidekiq/extra_sidekiq_routing.md' -remove_date: '2022-11-11' ---- - -This document was moved to [another location](../sidekiq/extra_sidekiq_routing.md). - -<!-- This redirect file can be deleted after <2022-11-11>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/operations/filesystem_benchmarking.md b/doc/administration/operations/filesystem_benchmarking.md index ec2975baf52..cd4ab1a9cf8 100644 --- a/doc/administration/operations/filesystem_benchmarking.md +++ b/doc/administration/operations/filesystem_benchmarking.md @@ -19,7 +19,7 @@ I/O. The information on this page can be used for either scenario. ### Benchmarking with `fio` -We recommend using +You should use [Fio](https://fio.readthedocs.io/en/latest/fio_doc.html) to test I/O performance. This test should be run both on the NFS server and on the application nodes that talk to the NFS server. @@ -35,8 +35,8 @@ Then run the following: fio --randrepeat=1 --ioengine=libaio --direct=1 --gtod_reduce=1 --name=test --bs=4k --iodepth=64 --readwrite=randrw --rwmixread=75 --size=4G --filename=/path/to/git-data/testfile ``` -This creates a 4GB file in `/path/to/git-data/testfile`. It performs -4KB reads and writes using a 75%/25% split in the file, with 64 +This creates a 4 GB file in `/path/to/git-data/testfile`. It performs +4 KB reads and writes using a 75%/25% split in the file, with 64 operations running at a time. Be sure to delete the file after the test completes. diff --git a/doc/administration/operations/index.md b/doc/administration/operations/index.md index d18f41becd5..527a72c5933 100644 --- a/doc/administration/operations/index.md +++ b/doc/administration/operations/index.md @@ -13,10 +13,9 @@ Keep your GitLab instance up and running smoothly. and more. - [Moving repositories](moving_repositories.md): Moving all repositories managed by GitLab to another file system or another server. -- [Sidekiq MemoryKiller](sidekiq_memory_killer.md): Configure Sidekiq MemoryKiller +- [Sidekiq MemoryKiller](../sidekiq/sidekiq_memory_killer.md): Configure Sidekiq MemoryKiller to restart Sidekiq. -- [Multiple Sidekiq processes](extra_sidekiq_processes.md): Configure multiple Sidekiq processes to ensure certain queues always have dedicated workers, no matter the number of jobs that must be processed. **(FREE SELF)** -- [Sidekiq routing rules](extra_sidekiq_routing.md): Configure the routing rules to route a job from a worker to a desirable queue. **(FREE SELF)** +- [Multiple Sidekiq processes](../sidekiq/extra_sidekiq_processes.md): Configure multiple Sidekiq processes to ensure certain queues always have dedicated workers, no matter the number of jobs that must be processed. **(FREE SELF)** - [Puma](puma.md): Understand Puma and puma-worker-killer. - Speed up SSH operations by [Authorizing SSH users via a fast, indexed lookup to the GitLab database](fast_ssh_key_lookup.md), and/or diff --git a/doc/administration/operations/puma.md b/doc/administration/operations/puma.md index eb326c06e6a..af595cdf297 100644 --- a/doc/administration/operations/puma.md +++ b/doc/administration/operations/puma.md @@ -66,9 +66,9 @@ From this output: - The formula that calculates the maximum memory value results in workers being killed before they reach the `per_worker_max_memory_mb` value. -- In GitLab 13.4 and earlier, the default values for the formula were 550MB for the primary - and 850MB for each worker. -- In GitLab 13.5 and later, the values are primary: 800MB, worker: 1024MB. +- In GitLab 13.4 and earlier, the default values for the formula were 550 MB for the primary + and 850 MB for each worker. +- In GitLab 13.5 and later, the values are primary: 800 MB, worker: 1024 MB. - The threshold for workers to be killed is set at 98% of the limit: ```plaintext @@ -110,11 +110,11 @@ To change the worker timeout to 600 seconds: WARNING: This is an experimental [Alpha feature](../../policy/alpha-beta-support.md#alpha-features) and subject to change without notice. The feature -is not ready for production use. If you want to use this feature, we recommend testing +is not ready for production use. If you want to use this feature, you should test outside of production first. See the [known issues](#puma-single-mode-known-issues) for additional details. -In a memory-constrained environment with less than 4GB of RAM available, consider disabling Puma +In a memory-constrained environment with less than 4 GB of RAM available, consider disabling Puma [clustered mode](https://github.com/puma/puma#clustered-mode). Set the number of `workers` to `0` to reduce memory usage by hundreds of MB: diff --git a/doc/administration/operations/sidekiq_memory_killer.md b/doc/administration/operations/sidekiq_memory_killer.md deleted file mode 100644 index b1977589fae..00000000000 --- a/doc/administration/operations/sidekiq_memory_killer.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../sidekiq/sidekiq_memory_killer.md' -remove_date: '2022-11-11' ---- - -This document was moved to [another location](../sidekiq/sidekiq_memory_killer.md). - -<!-- This redirect file can be deleted after <2022-11-11>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/package_information/index.md b/doc/administration/package_information/index.md index 4758b453135..bfa751a051f 100644 --- a/doc/administration/package_information/index.md +++ b/doc/administration/package_information/index.md @@ -16,9 +16,9 @@ The released package versions are in the format `MAJOR.MINOR.PATCH-EDITION.OMNIB | Component | Meaning | Example | |---------------------|---------|---------| -| `MAJOR.MINOR.PATCH` | The GitLab version this corresponds to. | 13.3.0 | -| `EDITION` | The edition of GitLab this corresponds to. | ee | -| `OMNIBUS_RELEASE` | The Omnibus GitLab release. Usually, this is 0. This is incremented if we need to build a new package without changing the GitLab version. | 0 | +| `MAJOR.MINOR.PATCH` | The GitLab version this corresponds to. | `13.3.0` | +| `EDITION` | The edition of GitLab this corresponds to. | `ee` | +| `OMNIBUS_RELEASE` | The Omnibus GitLab release. Usually, this is 0. This is incremented if we need to build a new package without changing the GitLab version. | `0` | ## Licenses diff --git a/doc/administration/package_information/licensing.md b/doc/administration/package_information/licensing.md index 0e46289a308..ece1597c521 100644 --- a/doc/administration/package_information/licensing.md +++ b/doc/administration/package_information/licensing.md @@ -25,8 +25,7 @@ Starting with version 9.2, the Omnibus GitLab package ships a `dependency_licenses.json` file containing version and license information of all bundled software, including software libraries, Ruby gems that the rails application uses, and JavaScript libraries that is required for the frontend -components. This file, being in JSON format, is easily machine parseable and -can be used for automated checks or validations. The file may be found at +components. Because it's in JSON format, GitLab can parse this file easily and use it for automated checks or validations. The file may be found at `/opt/gitlab/dependency_licenses.json`. Starting with version 11.3, we have also made the license information available diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index 2623f2afd8d..d76c728fb78 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -343,6 +343,9 @@ these deleted temporary upload artifacts are kept as non-current versions, there storage bucket size. To ensure that non-current versions are deleted after a given amount of time, you should configure an object lifecycle policy with your storage provider. +WARNING: +Do not directly modify the files or objects stored by the container registry. Anything other than the registry writing or deleting these entries can lead to instance-wide data consistency and instability issues from which recovery may not be possible. + You can configure the Container Registry to use various storage backends by configuring a storage driver. By default the GitLab Container Registry is configured to use the [file system driver](#use-file-system) @@ -585,6 +588,38 @@ you can pull from the Container Registry, but you cannot push. 1. Configure your registry to [use the S3 bucket for storage](#use-object-storage). 1. For the changes to take effect, set the Registry back to `read-write` mode and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +#### Moving to Azure Object Storage + +When moving from an existing file system or another object storage provider to Azure Object Storage, you must configure the registry to use the standard root directory. +This configuration is done by setting [`trimlegacyrootprefix: true]`](https://gitlab.com/gitlab-org/container-registry/-/blob/a3f64464c3ec1c5a599c0a2daa99ebcbc0100b9a/docs-gitlab/README.md#azure-storage-driver) in the Azure storage driver section of the registry configuration. +Without this configuration, the Azure storage driver uses `//` instead of `/` as the first section of the root path, rendering the migrated images inaccessible. + +**Omnibus GitLab installations** + +```ruby +registry['storage'] = { + 'azure' => { + 'accountname' => 'accountname', + 'accesskey' => 'base64encodedaccountkey', + 'container' => 'containername', + 'rootdirectory' => '/azure/virtual/container', + 'trimlegacyrootprefix' => 'true' + } +} +``` + +**Installations from source** + +```yaml +storage: + azure: + accountname: accountname + accountkey: base64encodedaccountkey + container: containername + rootdirectory: /azure/virtual/container + trimlegacyrootprefix: true +``` + ### Disable redirect for storage driver By default, users accessing a registry configured with a remote backend are redirected to the default backend for the storage driver. For example, registries can be configured using the `s3` storage driver, which redirects requests to a remote S3 bucket to alleviate load on the GitLab server. @@ -882,7 +917,7 @@ WARNING: If you're using a distributed architecture and Sidekiq is running on a different node, the cleanup policies don't work. To fix this, you must configure the `gitlab.rb` file on the Sidekiq nodes to point to the correct registry URL and copy the `registry.key` file to each Sidekiq node. For more -information, see the [Sidekiq configuration](../sidekiq.md) +information, see the [Sidekiq configuration](../sidekiq/index.md) page. To reduce the amount of [Container Registry disk space used by a given project](#registry-disk-space-usage-by-project), @@ -1091,6 +1126,9 @@ To enable the read-only mode: 1. Next, trigger one of the garbage collect commands: + WARNING: + You must use `/opt/gitlab/embedded/bin/registry` to recycle unused tags. If you use `gitlab-ctl registry-garbage-collect`, you **will bring the container registry down**. + ```shell # Recycling unused tags sudo /opt/gitlab/embedded/bin/registry garbage-collect /var/opt/gitlab/registry/config.yml @@ -1348,7 +1386,7 @@ level=error msg="response completed with error" err.code=unknown err.detail="une ``` To resolve the error specify a `chunksize` value in the Registry configuration. -Start with a value between `25000000` (25MB) and `50000000` (50MB). +Start with a value between `25000000` (25 MB) and `50000000` (50 MB). **For Omnibus installations** diff --git a/doc/administration/packages/dependency_proxy.md b/doc/administration/packages/dependency_proxy.md index f6eb6f85274..78efd75bbe3 100644 --- a/doc/administration/packages/dependency_proxy.md +++ b/doc/administration/packages/dependency_proxy.md @@ -121,15 +121,12 @@ To change the local storage path: ### Using object storage Instead of relying on the local storage, you can use an object storage to -store the blobs of the Dependency Proxy. +store the blobs of the Dependency Proxy. In GitLab 13.2 and later, you should use the +[consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). +This section describes the earlier configuration format. [Migration steps still apply](#migrate-local-dependency-proxy-blobs-and-manifests-to-object-storage). [Read more about using object storage with GitLab](../object_storage.md). -NOTE: -In GitLab 13.2 and later, we recommend using the -[consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). -This section describes the earlier configuration format. - **Omnibus GitLab installations** 1. Edit `/etc/gitlab/gitlab.rb` and add the following lines (uncomment where diff --git a/doc/administration/packages/index.md b/doc/administration/packages/index.md index 74d835eb744..6e53d77109f 100644 --- a/doc/administration/packages/index.md +++ b/doc/administration/packages/index.md @@ -6,11 +6,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Package Registry administration **(FREE SELF)** -GitLab Packages allows organizations to use GitLab as a private repository -for a variety of common package managers. Users are able to build and publish -packages, which can be easily consumed as a dependency in downstream projects. +To use GitLab as a private repository for a variety of common package managers, use the Package Registry. +You can build and publish +packages, which can be consumed as dependencies in downstream projects. -The Packages feature allows GitLab to act as a repository and supports the following formats: +## Supported formats + +The Package Registry supports the following formats: | Package type | GitLab version | |-------------------------------------------------------------------|----------------| @@ -49,204 +51,155 @@ guides you through the process. <!-- vale gitlab.Spelling = YES --> -## Enabling the Packages feature +## Rate limits -NOTE: -After the Packages feature is enabled, the repositories are available -for all new projects by default. To enable it for existing projects, users -explicitly do so in the project's settings. +When downloading packages as dependencies in downstream projects, many requests are made through the +Packages API. You may therefore reach enforced user and IP rate limits. To address this issue, you +can define specific rate limits for the Packages API. For more details, see [Package Registry Rate Limits](../../user/admin_area/settings/package_registry_rate_limits.md). -To enable the Packages feature: +## Change the storage path -**Omnibus GitLab installations** +By default, the packages are stored locally, but you can change the default +local location or even use object storage. -1. Edit `/etc/gitlab/gitlab.rb` and add the following line: +### Change the local storage path - ```ruby - gitlab_rails['packages_enabled'] = true - ``` +By default, the packages are stored in a local path, relative to the GitLab +installation: -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +- Linux package (Omnibus): `/var/opt/gitlab/gitlab-rails/shared/packages/` +- Self-compiled (source): `/home/git/gitlab/shared/packages/` -**Installations from source** +To change the local storage path: -1. After the installation is complete, you configure the `packages` - section in `config/gitlab.yml`. Set to `true` to enable it: +::Tabs - ```yaml - packages: - enabled: true +:::TabTitle Linux package (Omnibus) + +1. Edit `/etc/gitlab/gitlab.rb` and add the following line: + + ```ruby + gitlab_rails['packages_storage_path'] = "/mnt/packages" ``` -1. [Restart GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` -**Helm Chart installations** +:::TabTitle Self-compiled (source) -1. After the installation is complete, you configure the `packages` - section in `global.appConfig.packages`. Set to `true` to enable it: +1. Edit `/home/git/gitlab/config/gitlab.yml`: ```yaml - packages: - enabled: true + production: &base + packages: + enabled: true + storage_path: /mnt/packages ``` -1. [Restart GitLab](../restart_gitlab.md#helm-chart-installations) for the changes to take effect. +1. Save the file and restart GitLab: -## Rate limits + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target -When downloading packages as dependencies in downstream projects, many requests are made through the -Packages API. You may therefore reach enforced user and IP rate limits. To address this issue, you -can define specific rate limits for the Packages API. For more details, see [Package Registry Rate Limits](../../user/admin_area/settings/package_registry_rate_limits.md). + # For systems running SysV init + sudo service gitlab restart + ``` -## Changing the storage path +::EndTabs -By default, the packages are stored locally, but you can change the default -local location or even use object storage. +Docker and Kubernetes do not use local storage. -### Changing the local storage path +- For the Helm chart (Kubernetes): Use object storage instead. +- For Docker: The `/var/opt/gitlab/` directory is already + mounted in a directory on the host. There's no need to change the local + storage path inside the container. -The packages for Omnibus GitLab installations are stored under -`/var/opt/gitlab/gitlab-rails/shared/packages/` and for source -installations under `shared/packages/` (relative to the Git home directory). -To change the local storage path: +### Use object storage -**Omnibus GitLab installations** +Instead of relying on the local storage, you can use an object storage to store +packages. -1. Edit `/etc/gitlab/gitlab.rb` and add the following line: +For more information, see how to use the +[consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). - ```ruby - gitlab_rails['packages_storage_path'] = "/mnt/packages" - ``` +### Migrate local packages to object storage + +After [configuring the object storage](#use-object-storage), use the following task to +migrate existing packages from the local storage to the remote storage. +The processing is done in a background worker and requires **no downtime**. -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) - for the changes to take effect. +1. Migrate the packages. -**Installations from source** + ::Tabs -1. Edit the `packages` section in `config/gitlab.yml`: + :::TabTitle Linux package (Omnibus) - ```yaml - packages: - enabled: true - storage_path: shared/packages + ```shell + sudo gitlab-rake "gitlab:packages:migrate" ``` -1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. - -### Using object storage + :::TabTitle Self-compiled (source) -Instead of relying on the local storage, you can use an object storage to -store packages. + ```shell + RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:packages:migrate + ``` -[Read more about using object storage with GitLab](../object_storage.md). + ::EndTabs -NOTE: -We recommend using the [consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). The following instructions apply to the original configuration format. +1. Track the progress and verify that all packages migrated successfully using + the PostgreSQL console. -**Omnibus GitLab installations** + ::Tabs -1. Edit `/etc/gitlab/gitlab.rb` and add the following lines (uncomment where - necessary): + :::TabTitle Linux package (Omnibus) 14.1 and earlier - ```ruby - gitlab_rails['packages_enabled'] = true - gitlab_rails['packages_object_store_enabled'] = true - gitlab_rails['packages_object_store_remote_directory'] = "packages" # The bucket name. - gitlab_rails['packages_object_store_proxy_download'] = false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage. - gitlab_rails['packages_object_store_connection'] = { - ## - ## If the provider is AWS S3, uncomment the following - ## - #'provider' => 'AWS', - #'region' => 'eu-west-1', - #'aws_access_key_id' => 'AWS_ACCESS_KEY_ID', - #'aws_secret_access_key' => 'AWS_SECRET_ACCESS_KEY', - ## If an IAM profile is being used with AWS, omit the aws_access_key_id and aws_secret_access_key and uncomment - #'use_iam_profile' => true, - ## - ## If the provider is other than AWS (an S3-compatible one), uncomment the following - ## - #'host' => 's3.amazonaws.com', - #'aws_signature_version' => 4 # For creation of signed URLs. Set to 2 if provider does not support v4. - #'endpoint' => 'https://s3.amazonaws.com' # Useful for S3-compliant services such as DigitalOcean Spaces. - #'path_style' => false # If true, use 'host/bucket_name/object' instead of 'bucket_name.host/object'. - } + ```shell + sudo gitlab-rails dbconsole ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) - for the changes to take effect. - -**Installations from source** + :::TabTitle Linux package (Omnibus) 14.2 and later -1. Edit the `packages` section in `config/gitlab.yml` (uncomment where necessary): - - ```yaml - packages: - enabled: true - ## - ## The location where build packages are stored (default: shared/packages). - ## - # storage_path: shared/packages - object_store: - enabled: false - remote_directory: packages # The bucket name. - # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage. - connection: - ## - ## If the provider is AWS S3, use the following: - ## - provider: AWS - region: us-east-1 - aws_access_key_id: AWS_ACCESS_KEY_ID - aws_secret_access_key: AWS_SECRET_ACCESS_KEY - ## - ## If the provider is other than AWS (an S3-compatible one), comment out the previous 4 lines and use the following instead: - ## - # host: 's3.amazonaws.com' # default: s3.amazonaws.com. - # aws_signature_version: 4 # For creation of signed URLs. Set to 2 if provider does not support v4. - # endpoint: 'https://s3.amazonaws.com' # Useful for S3-compliant services such as DigitalOcean Spaces. - # path_style: false # If true, use 'host/bucket_name/object' instead of 'bucket_name.host/object'. + ```shell + sudo gitlab-rails dbconsole --database main ``` -1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. - -### Migrating local packages to object storage + :::TabTitle Self-compiled (source) -After [configuring the object storage](#using-object-storage), use the following task to -migrate existing packages from the local storage to the remote storage. -The processing is done in a background worker and requires **no downtime**. + ```shell + RAILS_ENV=production sudo -u git -H psql -d gitlabhq_production + ``` -For Omnibus GitLab: + ::EndTabs -```shell -sudo gitlab-rake "gitlab:packages:migrate" -``` +1. Verify that all packages migrated to object storage with the following SQL + query. The number of `objectstg` should be the same as `total`: -For installations from source: + ```shell + gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM packages_package_files; -```shell -RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:packages:migrate -``` + total | filesystem | objectstg + ------+------------+----------- + 34 | 0 | 34 + ``` -You can optionally track progress and verify that all packages migrated successfully using the -[PostgreSQL console](https://docs.gitlab.com/omnibus/settings/database.html#connecting-to-the-bundled-postgresql-database): +1. Finally, verify that there are no files on disk in the `packages` directory: -- `sudo gitlab-rails dbconsole` for Omnibus GitLab 14.1 and earlier. -- `sudo gitlab-rails dbconsole --database main` for Omnibus GitLab 14.2 and later. -- `sudo -u git -H psql -d gitlabhq_production` for source-installed instances. + ::Tabs -Verify `objectstg` below (where `file_store = '2'`) has count of all packages: + :::TabTitle Linux package (Omnibus) -```shell -gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM packages_package_files; + ```shell + sudo find /var/opt/gitlab/gitlab-rails/shared/packages -type f | grep -v tmp | wc -l + ``` -total | filesystem | objectstg -------+------------+----------- - 34 | 0 | 34 -``` + :::TabTitle Self-compiled (source) -Verify that there are no files on disk in the `packages` folder: + ```shell + sudo -u git find /home/git/gitlab/shared/packages -type f | grep -v tmp | wc -l + ``` -```shell -sudo find /var/opt/gitlab/gitlab-rails/shared/packages -type f | grep -v tmp | wc -l -``` + ::EndTabs diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index 3d31491a9d2..2a28df96ef4 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -247,20 +247,20 @@ control over how the Pages daemon runs and serves content in your environment. | `enable` | Enable or disable GitLab Pages on the current system. | | `external_http` | Configure Pages to bind to one or more secondary IP addresses, serving HTTP requests. Multiple addresses can be given as an array, along with exact ports, for example `['1.2.3.4', '1.2.3.5:8063']`. Sets value for `listen_http`. | | `external_https` | Configure Pages to bind to one or more secondary IP addresses, serving HTTPS requests. Multiple addresses can be given as an array, along with exact ports, for example `['1.2.3.4', '1.2.3.5:8063']`. Sets value for `listen_https`. | -| `server_shutdown_timeout` | GitLab Pages server shutdown timeout in seconds (default: 30s). | -| `gitlab_client_http_timeout` | GitLab API HTTP client connection timeout in seconds (default: 10s). | -| `gitlab_client_jwt_expiry` | JWT Token expiry time in seconds (default: 30s). | -| `gitlab_cache_expiry` | The maximum time a domain's configuration is stored in the cache (default: 600s). | -| `gitlab_cache_refresh` | The interval at which a domain's configuration is set to be due to refresh (default: 60s). | -| `gitlab_cache_cleanup` | The interval at which expired items are removed from the cache (default: 60s). | -| `gitlab_retrieval_timeout` | The maximum time to wait for a response from the GitLab API per request (default: 30s). | -| `gitlab_retrieval_interval` | The interval to wait before retrying to resolve a domain's configuration via the GitLab API (default: 1s). | +| `server_shutdown_timeout` | GitLab Pages server shutdown timeout in seconds (default: 30 s). | +| `gitlab_client_http_timeout` | GitLab API HTTP client connection timeout in seconds (default: 10 s). | +| `gitlab_client_jwt_expiry` | JWT Token expiry time in seconds (default: 30 s). | +| `gitlab_cache_expiry` | The maximum time a domain's configuration is stored in the cache (default: 600 s). | +| `gitlab_cache_refresh` | The interval at which a domain's configuration is set to be due to refresh (default: 60 s). | +| `gitlab_cache_cleanup` | The interval at which expired items are removed from the cache (default: 60 s). | +| `gitlab_retrieval_timeout` | The maximum time to wait for a response from the GitLab API per request (default: 30 s). | +| `gitlab_retrieval_interval` | The interval to wait before retrying to resolve a domain's configuration via the GitLab API (default: 1 s). | | `gitlab_retrieval_retries` | The maximum number of times to retry to resolve a domain's configuration via the API (default: 3). | | `domain_config_source` | This parameter was removed in 14.0, on earlier versions it can be used to enable and test API domain configuration source | | `gitlab_id` | The OAuth application public ID. Leave blank to automatically fill when Pages authenticates with GitLab. | | `gitlab_secret` | The OAuth application secret. Leave blank to automatically fill when Pages authenticates with GitLab. | | `auth_scope` | The OAuth application scope to use for authentication. Must match GitLab Pages OAuth application settings. Leave blank to use `api` scope by default. | -| `auth_cookie_session_timeout` | Authentication cookie session timeout in seconds (default: 600s). A value of `0` means the cookie is deleted after the browser session ends. | +| `auth_cookie_session_timeout` | Authentication cookie session timeout in seconds (default: 600 s). A value of `0` means the cookie is deleted after the browser session ends. | | `gitlab_server` | Server to use for authentication when access control is enabled; defaults to GitLab `external_url`. | | `headers` | Specify any additional http headers that should be sent to the client with each response. Multiple headers can be given as an array, header and value as one string, for example `['my-header: myvalue', 'my-other-header: my-other-value']` | | `enable_disk` | Allows the GitLab Pages daemon to serve content from disk. Shall be disabled if shared disk storage isn't available. | @@ -275,9 +275,9 @@ control over how the Pages daemon runs and serves content in your environment. | `max_uri_length` | The maximum length of URIs accepted by GitLab Pages. Set to 0 for unlimited length. [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/659) in GitLab 14.5. | `metrics_address` | The address to listen on for metrics requests. | | `redirect_http` | Redirect pages from HTTP to HTTPS, true/false. | -| `redirects_max_config_size` | The maximum size of the _redirects file, in bytes (default: 65536). | -| `redirects_max_path_segments` | The maximum number of path segments allowed in _redirects rules URLs (default: 25). | -| `redirects_max_rule_count` | The maximum number of rules allowed in _redirects (default: 1000). | +| `redirects_max_config_size` | The maximum size of the `_redirects` file, in bytes (default: 65536). | +| `redirects_max_path_segments` | The maximum number of path segments allowed in `_redirects` rules URLs (default: 25). | +| `redirects_max_rule_count` | The maximum number of rules allowed in `_redirects` (default: 1000). | | `sentry_dsn` | The address for sending Sentry crash reporting to. | | `sentry_enabled` | Enable reporting and logging with Sentry, true/false. | | `sentry_environment` | The environment for Sentry crash reporting. | @@ -294,7 +294,7 @@ control over how the Pages daemon runs and serves content in your environment. | `pages_path` | The directory on disk where pages are stored, defaults to `GITLAB-RAILS/shared/pages`. | | **`pages_nginx[]`** | | | `enable` | Include a virtual host `server{}` block for Pages inside NGINX. Needed for NGINX to proxy traffic back to the Pages daemon. Set to `false` if the Pages daemon should directly receive all requests, for example, when using [custom domains](index.md#custom-domains). | -| `FF_ENABLE_PLACEHOLDERS` | Feature flag to enable/disable rewrites (disabled by default). Read the [redirects documentation](../../user/project/pages/redirects.md#feature-flag-for-rewrites) for more information. | +| `FF_ENABLE_PLACEHOLDERS` | Feature flag for rewrites (enabled by default). See [Rewrites](../../user/project/pages/redirects.md#rewrites) for more information. | | `use_legacy_storage` | Temporarily-introduced parameter allowing to use legacy domain configuration source and storage. [Removed in 14.3](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6166). | | `rate_limit_source_ip` | Rate limit per source IP in number of requests per second. Set to `0` to disable this feature. | | `rate_limit_source_ip_burst` | Rate limit per source IP maximum burst allowed per second. | @@ -540,22 +540,22 @@ archive. You can modify the cache behavior by changing the following configurati | Setting | Description | | ------- | ----------- | -| `zip_cache_expiration` | The cache expiration interval of ZIP archives. Must be greater than zero to avoid serving stale content. Default is 60s. | -| `zip_cache_cleanup` | The interval at which archives are cleaned from memory if they have already expired. Default is 30s. | -| `zip_cache_refresh` | The time interval in which an archive is extended in memory if accessed before `zip_cache_expiration`. This works together with `zip_cache_expiration` to determine if an archive is extended in memory. See the [example below](#zip-cache-refresh-example) for important details. Default is 30s. | -| `zip_open_timeout` | The maximum time allowed to open a ZIP archive. Increase this time for big archives or slow network connections, as doing so may affect the latency of serving Pages. Default is 30s. | -| `zip_http_client_timeout` | The maximum time for the ZIP HTTP client. Default is 30m. | +| `zip_cache_expiration` | The cache expiration interval of ZIP archives. Must be greater than zero to avoid serving stale content. Default is 60 s. | +| `zip_cache_cleanup` | The interval at which archives are cleaned from memory if they have already expired. Default is 30 s. | +| `zip_cache_refresh` | The time interval in which an archive is extended in memory if accessed before `zip_cache_expiration`. This works together with `zip_cache_expiration` to determine if an archive is extended in memory. See the [example below](#zip-cache-refresh-example) for important details. Default is 30 s. | +| `zip_open_timeout` | The maximum time allowed to open a ZIP archive. Increase this time for big archives or slow network connections, as doing so may affect the latency of serving Pages. Default is 30 s. | +| `zip_http_client_timeout` | The maximum time for the ZIP HTTP client. Default is 30 m. | #### ZIP cache refresh example Archives are refreshed in the cache (extending the time they are held in memory) if they're accessed before `zip_cache_expiration`, and the time left before expiring is less than or equal to -`zip_cache_refresh`. For example, if `archive.zip` is accessed at time 0s, it expires in 60s (the -default for `zip_cache_expiration`). In the example below, if the archive is opened again after 15s -it is **not** refreshed because the time left for expiry (45s) is greater than `zip_cache_refresh` -(default 30s). However, if the archive is accessed again after 45s (from the first time it was +`zip_cache_refresh`. For example, if `archive.zip` is accessed at time 0 s, it expires in 60 s (the +default for `zip_cache_expiration`). In the example below, if the archive is opened again after 15 s +it is **not** refreshed because the time left for expiry (45 s) is greater than `zip_cache_refresh` +(default 30 s). However, if the archive is accessed again after 45 s (from the first time it was opened) it's refreshed. This extends the time the archive remains in memory from -`45s + zip_cache_expiration (60s)`, for a total of 105s. +`45s + zip_cache_expiration (60s)`, for a total of 105 s. After an archive reaches `zip_cache_expiration`, it's marked as expired and removed on the next `zip_cache_cleanup` interval. @@ -941,6 +941,10 @@ If you want to stop using and disconnect the NFS server, you need to #### S3-compatible connection settings +In GitLab 13.2 and later, you should use the +[consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). +This section describes the earlier configuration format. + See [the available connection settings for different providers](../object_storage.md#connection-settings). In Omnibus installations: diff --git a/doc/administration/postgresql/database_load_balancing.md b/doc/administration/postgresql/database_load_balancing.md index 805c4b2023f..15129770888 100644 --- a/doc/administration/postgresql/database_load_balancing.md +++ b/doc/administration/postgresql/database_load_balancing.md @@ -76,18 +76,20 @@ Database Load Balancing can be configured in one of two ways: ### Hosts -To configure a list of hosts, add the `gitlab_rails['db_load_balancing']` setting into the -`gitlab.rb` file in the GitLab Rails / Sidekiq nodes for each environment you want to balance. +To configure a list of hosts, perform these steps on all GitLab Rails (Sidekiq) +nodes for each environment you want to balance: -For example, on an environment that has PostgreSQL running on the hosts `host1.example.com`, -`host2.example.com` and `host3.example.com` and reachable on the same port configured with -`gitlab_rails['db_port']`: +1. Edit the `/etc/gitlab/gitlab.rb` file. +1. In `gitlab_rails['db_load_balancing']`, create an array of the read-only + replicas you want to balance. Do not add the primary host. For example, on + an environment with PostgreSQL running on the hosts `primary.example.com`, + `host1.example.com`, `host2.example.com`, and `host3.example.com`: -1. On each GitLab Rails / Sidekiq node, edit `/etc/gitlab/gitlab.rb` and add the following line: + ```ruby + gitlab_rails['db_load_balancing'] = { 'hosts' => ['host1.example.com', 'host2.example.com', `host3.example.com`] } + ``` - ```ruby - gitlab_rails['db_load_balancing'] = { 'hosts' => ['host1.example.com', 'host2.example.com', `host3.example.com`] } - ``` + These replicas must be reachable on the same port configured with `gitlab_rails['db_port']`. 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/postgresql/multiple_databases.md b/doc/administration/postgresql/multiple_databases.md new file mode 100644 index 00000000000..ffccbf861e7 --- /dev/null +++ b/doc/administration/postgresql/multiple_databases.md @@ -0,0 +1,141 @@ +--- +stage: Data Stores +group: Pods +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Multiple Databases **(FREE SELF)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/6168) in GitLab 15.7. + +WARNING: +This feature is not ready for production use + +By default, GitLab uses a single application database, referred to as the `main` database. + +To scale GitLab, you can configure GitLab to use multiple application databases. + +Due to [known issues](#known-issues), configuring GitLab with multiple databases is in [**Alpha**](../../policy/alpha-beta-support.md#alpha-features). + +## Known issues + +- Migrating data from the `main` database to the `ci` database is not supported or documented yet. +- Once data is migrated to the `ci` database, you cannot migrate it back. + +## Set up multiple databases + +Use the following content to set up multiple databases with a new GitLab installation. + +There is no documentation for existing GitLab installations yet. + +After you have set up multiple databases, GitLab uses a second application database for +[CI/CD features](../../ci/index.md), referred to as the `ci` database. For +example, GitLab reads and writes to the `ci_pipelines` table in the `ci` +database. + +WARNING: +You must stop GitLab before setting up multiple databases. This prevents +split-brain situations, where `main` data is written to the `ci` database, and +the other way around. + +### Installations from source + +1. [Back up GitLab](../../raketasks/backup_restore.md) + in case of unforeseen issues. + +1. Stop GitLab: + + ```shell + sudo service gitlab stop + ``` + +1. Open `config/database.yml`, and add a `ci:` section under + `production:`. See `config/database.yml.decomposed-postgresql` for possible + values for this new `ci:` section. Once modified, the `config/database.yml` should + look like: + + ```yaml + production: + main: + # ... + ci: + adapter: postgresql + encoding: unicode + database: gitlabhq_production_ci + # ... + ``` + +1. Save the `config/database.yml` file. + +1. Create the `gitlabhq_production_ci` database: + + ```shell + sudo -u postgres psql -d template1 -c "CREATE DATABASE gitlabhq_production OWNER git;" + sudo -u git -H bundle exec rake db:schema:load:ci + ``` + +1. Lock writes for `ci` tables in `main` database, and the other way around: + + ```shell + sudo -u git -H bundle exec rake gitlab:db:lock_writes + ``` + +1. Restart GitLab: + + ```shell + sudo service gitlab restart + ``` + +### Omnibus GitLab installations + +1. [Back up GitLab](../../raketasks/backup_restore.md) + in case of unforeseen issues. + +1. Stop GitLab: + + ```shell + sudo gitlab-ctl stop + ``` + +1. Edit `/etc/gitlab/gitlab.rb` and add the following lines: + + ```ruby + gitlab_rails['databases']['ci']['enable'] = true + gitlab_rails['databases']['ci']['db_database'] = 'gitlabhq_production_ci' + ``` + +1. Save the `/etc/gitlab/gitlab.rb` file. + +1. Reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +1. Optional. Reconfiguring GitLab should create the `gitlabhq_production_ci`. If it did not, manually create the `gitlabhq_production_ci`: + + ```shell + sudo gitlab-ctl start postgresql + sudo -u gitlab-psql /opt/gitlab/embedded/bin/psql -h /var/opt/gitlab/postgresql -d template1 -c "CREATE DATABASE gitlabhq_production_ci OWNER gitlab;" + sudo gitlab-rake db:schema:load:ci + +1. Lock writes for `ci` tables in `main` database, and the other way around: + + ```shell + sudo gitlab-ctl start postgresql + sudo gitlab-rake gitlab:db:lock_writes + ``` + +1. Restart GitLab: + + ```shell + sudo gitlab-ctl restart + ``` + +## Further information + +For more information on multiple databases, see [issue 6168](https://gitlab.com/groups/gitlab-org/-/epics/6168). + +For more information on how multiple databases work in GitLab, see the [development guide for multiple databases](../../development/database/multiple_databases.md). + +Since 2022-07-02, GitLab.com has been running with two separate databases. For more information, see this [blog post](https://about.gitlab.com/blog/2022/06/02/splitting-database-into-main-and-ci/). diff --git a/doc/administration/postgresql/pgbouncer.md b/doc/administration/postgresql/pgbouncer.md index 91c689fadea..25c4c940b97 100644 --- a/doc/administration/postgresql/pgbouncer.md +++ b/doc/administration/postgresql/pgbouncer.md @@ -220,8 +220,8 @@ the database. Each of the listed services below use the following formula to def - `puma` : `max_threads + headroom` (default `14`) - `max_threads` is configured via: `gitlab['puma']['max_threads']` (default: `4`) - `headroom` can be configured via `DB_POOL_HEADROOM` environment variable (default to `10`) -- `sidekiq` : `max_concurrency + 1 + headroom` (default: `61`) - - `max_concurrency` is configured via: `sidekiq['max_concurrency']` (default: `50`) +- `sidekiq` : `max_concurrency + 1 + headroom` (default: `31`) + - `max_concurrency` is configured via: `sidekiq['max_concurrency']` (default: `20`) - `headroom` can be configured via `DB_POOL_HEADROOM` environment variable (default to `10`) - `geo-logcursor`: `1+headroom` (default: `11`) - `headroom` can be configured via `DB_POOL_HEADROOM` environment variable (default to `10`) diff --git a/doc/administration/postgresql/replication_and_failover.md b/doc/administration/postgresql/replication_and_failover.md index ee90b120d05..ececa12f3ad 100644 --- a/doc/administration/postgresql/replication_and_failover.md +++ b/doc/administration/postgresql/replication_and_failover.md @@ -608,7 +608,7 @@ Here is a list and description of each machine and the assigned IP: All passwords are set to `toomanysecrets`. Do not use this password or derived hashes and the `external_url` for GitLab is `http://gitlab.example.com`. -After the initial configuration, if a failover occurs, the PostgresSQL leader node changes to one of the available secondaries until it is failed back. +After the initial configuration, if a failover occurs, the PostgreSQL leader node changes to one of the available secondaries until it is failed back. #### Example recommended setup for Consul servers @@ -1081,6 +1081,190 @@ Reverting the PostgreSQL upgrade with `gitlab-ctl revert-pg-upgrade` has the sam `gitlab-ctl pg-upgrade`. You should follow the same procedure by first stopping the replicas, then reverting the leader, and finally reverting the replicas. +### Near zero downtime upgrade of PostgreSQL in a Patroni cluster (Experimental) + +Patroni enables you to run a major PostgreSQL upgrade without shutting down the cluster. However, this +requires additional resources to host the new Patroni nodes with the upgraded PostgreSQL. In practice, with this +procedure, you are: + +- Creating a new Patroni cluster with a new version of PostgreSQL. +- Migrating the data from the existing cluster. + +This procedure is non-invasive, and does not impact your existing cluster before switching it off. +However, it can be both time- and resource-consuming. Consider their trade-offs with availability. + +The steps, in order: + +1. [Provision resources for the new cluster](#provision-resources-for-the-new-cluster). +1. [Preflight check](#preflight-check). +1. [Configure the leader of the new cluster](#configure-the-leader-of-the-new-cluster). +1. [Start publisher on the existing leader](#start-publisher-on-the-existing-leader). +1. [Copy the data from the existing cluster](#copy-the-data-from-the-existing-cluster). +1. [Replicate data from the existing cluster](#replicate-data-from-the-existing-cluster). +1. [Grow the new cluster](#grow-the-new-cluster). +1. [Switch the application to use the new cluster](#switch-the-application-to-use-the-new-cluster). +1. [Clean up](#clean-up). + +#### Provision resources for the new cluster + +You need a new set of resources for Patroni nodes. The new Patroni cluster does not require exactly the same number +of nodes as the existing cluster. You may choose a different number of nodes based on your requirements. The new +cluster uses the existing Consul cluster (with a different `patroni['scope']`) and PgBouncer nodes. + +Make sure that at least the leader node of the existing cluster is accessible from the nodes of the new +cluster. + +#### Preflight check + +We rely on PostgreSQL [logical replication](https://www.postgresql.org/docs/current/logical-replication.html) +to support near-zero downtime upgrades of Patroni clusters. The of +[logical replication requirements](https://www.postgresql.org/docs/current/logical-replication-restrictions.html) +must be met. In particular, `wal_level` must be `logical`. To check the `wal_level`, +run the following command with `gitlab-psql` on any node of the existing cluster: + +```sql +SHOW wal_level; +``` + +By default, Patroni sets `wal_level` to `replica`. You must increase it to `logical`. +Changing `wal_level` requires restarting PostgreSQL, so this step leads to a short +downtime (hence near-zero downtime). To do this on the Patroni **leader** node: + +1. Edit `gitlab.rb` by setting: + + ```ruby + patroni['postgresql']['wal_level'] = 'logical' + ``` + +1. Run `gitlab-ctl reconfigure`. This writes the configuration but does not restart PostgreSQL service. +1. Run `gitlab-ctl patroni restart` to restart PostgreSQL and apply the new `wal_level` without triggering + failover. For the duration of restart cycle, the cluster leader is unavailable. +1. Verify the change by running `SHOW wal_level` with `gitlab-psql`. + +#### Configure the leader of the new cluster + +Configure the first node of the new cluster. It becomes the leader of the new cluster. +You can use the configuration of the existing cluster, if it is compatible with the new +PostgreSQL version. Refer to the documentation on [configuring Patroni clusters](#configuring-patroni-cluster). + +In addition to the common configuration, you must apply the following in `gitlab.rb` to: + +1. Make sure that the new Patroni cluster uses a different scope. The scope is used to namespace the Patroni settings + in Consul, making it possible to use the same Consul cluster for the existing and the new clusters. + + ```ruby + patroni['scope'] = 'postgresql_new-ha' + ``` + +1. Make sure that Consul agents don't mix PostgreSQL services offered by the existing and the new Patroni + clusters. For this purpose, you must use an internal attribute that is currently undocumented: + + ```ruby + consul['internal']['postgresql_service_name'] = 'postgresql_new' + ``` + +#### Start publisher on the existing leader + +On the existing leader, run this SQL statement with `gitlab-psql` to start a logical replication publisher: + +```sql +CREATE PUBLICATION patroni_upgrade FOR ALL TABLES; +``` + +#### Copy the data from the existing cluster + +To dump the current database from the existing cluster, run these commands on the +**leader** of the new cluster: + +1. Optional. Copy global database objects: + + ```shell + pg_dumpall -h ${EXISTING_CLUSTER_LEADER} -U gitlab-psql -g | gitlab-psql + ``` + + You can ignore the errors about existing database objects, such as roles. They are + created when the node is configured for the first time. + +1. Copy the current database: + + ```shell + pg_dump -h ${EXISTING_CLUSTER_LEADER} -U gitlab-psql -d gitlabhq_production -s | gitlab-psql + ``` + + Depending on the size of your database, this command may take a while to complete. + +The `pg_dump` and `pg_dumpall` commands are in `/opt/gitlab/embedded/bin`. In these commands, +`EXISTING_CLUSTER_LEADER` is the host address of the leader node of the existing cluster. + +NOTE: +The `gitlab-psql` user must be able to authenticate the existing leader from the new leader node. + +#### Replicate data from the existing cluster + +After taking the initial data dump, you must keep the new leader in sync with the +latest changes of your existing cluster. On the new leader, run this SQL statement +with `gitlab-psql` to subscribe to publication of the existing leader: + +```sql +CREATE SUBSCRIPTION patroni_upgrade + CONNECTION 'host=EXISTING_CLUSTER_LEADER dbname=gitlabhq_production user=gitlab-psql' + PUBLICATION patroni_upgrade; +``` + +In this statement, `EXISTING_CLUSTER_LEADER` is the host address of the leader node +of the existing cluster. You can also use +[other parameters](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS) +to change the connection string. For example, you can pass the authentication password. + +To check the status of replication, run these queries: + +- `SELECT * FROM pg_replication_slots WHERE slot_name = 'patroni_upgrade'` on the existing leader (the publisher). +- `SELECT * FROM pg_stat_subscription` on the new leader (the subscriber). + +#### Grow the new cluster + +Configure other nodes of the new cluster in the way you +[configured the leader](#configure-the-leader-of-the-new-cluster). +Make sure that you use the same `patroni['scope']` and +`consul['internal']['postgresql_service_name']`. + +What happens here: + +- The application still uses the existing leader as its database backend. +- The logical replication ensures that the new leader keeps in sync. +- When other nodes are added to the new cluster, Patroni handles + the replication to the these nodes. + +It is a good idea to wait until the replica nodes of the new cluster are initialized and caught up on the replication +lag. + +#### Switch the application to use the new cluster + +Up to this point, you can stop the upgrade procedure without losing data on the +existing cluster. When you switch the database backend of the application and point +it to the new cluster, the old cluster does not receive new updates. It falls behind +the new cluster. After this point, any recovery must be done from the nodes of the new cluster. + +To do the switch on **all** PgBouncer nodes: + +1. Edit `gitlab.rb` by setting: + + ```ruby + consul['watchers'] = %w(postgresql_new) + consul['internal']['postgresql_service_name'] = 'postgresql_new' + ``` + +1. Run `gitlab-ctl reconfigure`. +1. You must also run `rm /var/opt/gitlab/consul/config.d/watcher_postgresql.json`. + This is a [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7293). + +#### Clean up + +After completing these steps, then you can clean up the resources of the old Patroni cluster. +They are no longer needed. However, before removing the resources, remove the +logical replication subscription on the new leader by running `DROP SUBSCRIPTION patroni_upgrade` +with `gitlab-psql`. + ## Troubleshooting ### Consul and PostgreSQL changes not taking effect diff --git a/doc/administration/raketasks/geo.md b/doc/administration/raketasks/geo.md index a4d027101dd..03b09e00f1c 100644 --- a/doc/administration/raketasks/geo.md +++ b/doc/administration/raketasks/geo.md @@ -4,7 +4,7 @@ group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Geo Rake Tasks **(PREMIUM SELF)** +# Geo Rake tasks **(PREMIUM SELF)** The following Rake tasks are for [Geo installations](../geo/index.md). See also [troubleshooting Geo](../geo/replication/troubleshooting.md) for additional Geo Rake tasks. diff --git a/doc/administration/raketasks/github_import.md b/doc/administration/raketasks/github_import.md index 224ed63d3e6..61f6137e1ed 100644 --- a/doc/administration/raketasks/github_import.md +++ b/doc/administration/raketasks/github_import.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# GitHub import **(FREE SELF)** +# GitHub import Rake task **(FREE SELF)** To retrieve and import GitHub repositories, you need a [GitHub personal access token](https://github.com/settings/tokens). A username should be passed as the second argument to the Rake task, diff --git a/doc/administration/raketasks/maintenance.md b/doc/administration/raketasks/maintenance.md index 293efb1b7ae..ba095b33bf5 100644 --- a/doc/administration/raketasks/maintenance.md +++ b/doc/administration/raketasks/maintenance.md @@ -301,7 +301,7 @@ sudo gitlab-rake gitlab:exclusive_lease:clear[project_housekeeping:4] ## Display status of database migrations -See the [upgrade documentation](../../update/index.md#checking-for-background-migrations-before-upgrading) +See the [background migrations documentation](../../update/background_migrations.md) for how to check that migrations are complete when upgrading GitLab. To check the status of specific migrations, you can use the following Rake task: diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index a2463c6ff88..88913eb1f7f 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -35,13 +35,13 @@ full list of reference architectures, see | GitLab Rails | 3 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | | Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Object storage<sup>4</sup> | - | - | - | - | -| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. - [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. - - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB clusters](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - Note that [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is **incompatible** with load balancing enabled by default in [14.4.0](../../update/index.md#1440). - Consul is primarily used for Omnibus PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However, Consul is also used optionally by Prometheus for Omnibus auto host discovery. 2. Can be optionally run on reputable third-party external PaaS Redis solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. @@ -189,7 +189,7 @@ CI pipelines alike. As such, large repositories come with notable cost and typically will require more resources to handle, significantly so in some cases. It's therefore **strongly** recommended then to review large repositories -to ensure they maintain good repo health and reduce their size wherever possible. +to ensure they maintain good health and reduce their size wherever possible. NOTE: If best practices aren't followed and large repositories are present on the environment, @@ -227,9 +227,6 @@ To set up GitLab and its components to accommodate up to 10,000 users: environment. 1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) - to have shared disk storage service as an alternative to Gitaly or object - storage. 1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, more advanced code search across your entire GitLab instance. @@ -1275,7 +1272,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - postgresql['max_connections'] = 200 + postgresql['max_connections'] = 500 # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1759,9 +1756,8 @@ To configure Praefect with TLS: Sidekiq requires connection to the [Redis](#configure-redis), [PostgreSQL](#configure-postgresql) and [Gitaly](#configure-gitaly) instances. -Since it's recommended to use [Object storage](#configure-the-object-storage) -over [NFS](#configure-nfs-optional) for data objects, the following examples -include the Object storage configuration. +Because you must use [Object storage](#configure-the-object-storage) instead of NFS for data objects, the following +examples include the Object storage configuration. - `10.6.0.101`: Sidekiq 1 - `10.6.0.102`: Sidekiq 2 @@ -1859,8 +1855,8 @@ Updates to example must be made at: # Set number of Sidekiq queue processes to the same number as available CPUs sidekiq['queue_groups'] = ['*'] * 4 - # Set number of Sidekiq threads per queue process to the recommend number of 10 - sidekiq['max_concurrency'] = 10 + # Set number of Sidekiq threads per queue process to the recommend number of 20 + sidekiq['max_concurrency'] = 20 # Monitoring consul['enable'] = true @@ -1918,7 +1914,7 @@ Updates to example must be made at: NOTE: If you find that the environment's Sidekiq job processing is slow with long queues, more nodes can be added as required. You can also tune your Sidekiq nodes to -run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). +run [multiple Sidekiq processes](../sidekiq/extra_sidekiq_processes.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1929,9 +1925,8 @@ run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). ## Configure GitLab Rails This section describes how to configure the GitLab application (Rails) component. -Since it's recommended to use [Object storage](#configure-the-object-storage) -over [NFS](#configure-nfs-optional) for data objects, the following examples -include the Object storage configuration. +Because you must use [Object storage](#configure-the-object-storage) instead of NFS for data objects, the following +examples include the Object storage configuration. The following IPs will be used as an example: @@ -2070,7 +2065,6 @@ On each node perform the following: 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. - 1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: ```shell @@ -2081,9 +2075,7 @@ On each node perform the following: [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -1. [Enable incremental logging](#enable-incremental-logging), unless you are using [NFS](#configure-nfs-optional). - +1. [Enable incremental logging](#enable-incremental-logging). 1. Confirm the node can connect to Gitaly: ```shell @@ -2209,9 +2201,6 @@ To configure the Monitoring node: ## Configure the object storage GitLab supports using an object storage service for holding numerous types of data. -It's recommended over [NFS](#configure-nfs-optional) and in general it's better -in larger setups as object storage is typically much more performant, reliable, -and scalable. GitLab has been tested on a number of object storage providers: @@ -2235,7 +2224,7 @@ NOTE: When using the [storage-specific form](../object_storage.md#storage-specific-configuration) in GitLab 14.x and earlier, you should enable [direct upload mode](../../development/uploads/index.md#direct-upload). The previous [background upload](../../development/uploads/index.md#direct-upload) mode, -which was deprecated in 14.9, requires shared storage such as [NFS](#configure-nfs-optional). +which was deprecated in 14.9, requires shared storage such as NFS. Using separate buckets for each data type is the recommended approach for GitLab. This ensures there are no collisions across the various types of data GitLab stores. @@ -2254,22 +2243,6 @@ GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. -## Configure NFS (optional) - -[Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) -are recommended over NFS wherever possible for improved performance. - -See how to [configure NFS](../nfs.md). - -WARNING: -Engineering support for NFS for Git repositories is deprecated, and [technical support is scheduled to be unavailable](../nfs.md#gitaly-and-nfs-deprecation) -after the release of GitLab 15.6. No further enhancements are planned for this feature. - -Read: - -- [Gitaly and NFS Deprecation](../nfs.md#gitaly-and-nfs-deprecation). -- About the [correct mount options to use](../nfs.md#upgrade-to-gitaly-cluster-or-disable-caching-if-experiencing-data-loss). - ## Configure Advanced Search You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) @@ -2319,12 +2292,10 @@ Refer to [epic 6127](https://gitlab.com/groups/gitlab-org/-/epics/6127) for more The following tables and diagram detail the hybrid environment using the same formats as the normal environment above. -First are the components that run in Kubernetes. The recommendation at this time is to -use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EKS) and associated machine types, but the memory -and CPU requirements should translate to most other providers. We hope to update this in the -future with further specific cloud provider details. +First are the components that run in Kubernetes. These run across several node groups, although you can change +the overall makeup as desired as long as the minimum CPU and Memory requirements are observed. -| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | +| Service Node Group | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | |---------------------|-------|-------------------------|-----------------|--------------|---------------------------------| | Webservice | 4 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | 127.5 vCPU, 118 GB memory | | Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 15.5 vCPU, 50 GB memory | @@ -2333,7 +2304,7 @@ future with further specific cloud provider details. - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. - Nodes configuration is shown as it is forced to ensure pod vCPU / memory ratios and avoid scaling during **performance testing**. - - In production deployments, there is no need to assign pods to nodes. A minimum of three nodes in three different availability zones is strongly recommended to align with resilient cloud architecture practices. + - In production deployments, there is no need to assign pods to specific nodes. A minimum of three nodes per node group in three different availability zones is strongly recommended to align with resilient cloud architecture practices. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS services where applicable): @@ -2355,7 +2326,8 @@ services where applicable): <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. - [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. - - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB clusters](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - Note that [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is **incompatible** with load balancing enabled by default in [14.4.0](../../update/index.md#1440). - Consul is primarily used for Omnibus PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However, Consul is also used optionally by Prometheus for Omnibus auto host discovery. 2. Can be optionally run on reputable third-party external PaaS Redis solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 84eba01fe11..02739904f5e 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -35,13 +35,13 @@ full list of reference architectures, see | GitLab Rails | 5 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | | Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Object storage<sup>4</sup> | - | - | - | - | -| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. - [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. - - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB clusters](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - Note that [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is **incompatible** with load balancing enabled by default in [14.4.0](../../update/index.md#1440). - Consul is primarily used for Omnibus PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However, Consul is also used optionally by Prometheus for Omnibus auto host discovery. 2. Can be optionally run on reputable third-party external PaaS Redis solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. @@ -189,7 +189,7 @@ CI pipelines alike. As such, large repositories come with notable cost and typically will require more resources to handle, significantly so in some cases. It's therefore **strongly** recommended then to review large repositories -to ensure they maintain good repo health and reduce their size wherever possible. +to ensure they maintain good health and reduce their size wherever possible. NOTE: If best practices aren't followed and large repositories are present on the environment, @@ -227,9 +227,6 @@ To set up GitLab and its components to accommodate up to 25,000 users: environment. 1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) - to have shared disk storage service as an alternative to Gitaly or object - storage. 1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, more advanced code search across your entire GitLab instance. @@ -1295,7 +1292,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - postgresql['max_connections'] = 200 + postgresql['max_connections'] = 500 # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1777,9 +1774,8 @@ To configure Praefect with TLS: Sidekiq requires connection to the [Redis](#configure-redis), [PostgreSQL](#configure-postgresql) and [Gitaly](#configure-gitaly) instances. -Since it's recommended to use [Object storage](#configure-the-object-storage) -over [NFS](#configure-nfs-optional) for data objects, the following examples -include the Object storage configuration. +Because you must use [Object storage](#configure-the-object-storage) instead of NFS for data objects, the following +examples include the Object storage configuration. - `10.6.0.101`: Sidekiq 1 - `10.6.0.102`: Sidekiq 2 @@ -1877,8 +1873,8 @@ Updates to example must be made at: # Set number of Sidekiq queue processes to the same number as available CPUs sidekiq['queue_groups'] = ['*'] * 4 - # Set number of Sidekiq threads per queue process to the recommend number of 10 - sidekiq['max_concurrency'] = 10 + # Set number of Sidekiq threads per queue process to the recommend number of 20 + sidekiq['max_concurrency'] = 20 # Monitoring consul['enable'] = true @@ -1936,7 +1932,7 @@ Updates to example must be made at: NOTE: If you find that the environment's Sidekiq job processing is slow with long queues, more nodes can be added as required. You can also tune your Sidekiq nodes to -run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). +run [multiple Sidekiq processes](../sidekiq/extra_sidekiq_processes.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1947,9 +1943,8 @@ run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). ## Configure GitLab Rails This section describes how to configure the GitLab application (Rails) component. -Since it's recommended to use [Object storage](#configure-the-object-storage) -over [NFS](#configure-nfs-optional) for data objects, the following examples -include the Object storage configuration. +Because you must use [Object storage](#configure-the-object-storage) instead of NFS for data objects, the following +examples include the Object storage configuration. The following IPs will be used as an example: @@ -2090,7 +2085,6 @@ On each node perform the following: 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. - 1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: ```shell @@ -2101,9 +2095,7 @@ On each node perform the following: [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -1. [Enable incremental logging](#enable-incremental-logging), unless you are using [NFS](#configure-nfs-optional). - +1. [Enable incremental logging](#enable-incremental-logging). 1. Confirm the node can connect to Gitaly: ```shell @@ -2228,9 +2220,6 @@ To configure the Monitoring node: ## Configure the object storage GitLab supports using an object storage service for holding numerous types of data. -It's recommended over [NFS](#configure-nfs-optional) and in general it's better -in larger setups as object storage is typically much more performant, reliable, -and scalable. GitLab has been tested on a number of object storage providers: @@ -2254,7 +2243,7 @@ NOTE: When using the [storage-specific form](../object_storage.md#storage-specific-configuration) in GitLab 14.x and earlier, you should enable [direct upload mode](../../development/uploads/index.md#direct-upload). The previous [background upload](../../development/uploads/index.md#direct-upload) mode, -which was deprecated in 14.9, requires shared storage such as [NFS](#configure-nfs-optional). +which was deprecated in 14.9, requires shared storage such as NFS. Using separate buckets for each data type is the recommended approach for GitLab. This ensures there are no collisions across the various types of data GitLab stores. @@ -2273,22 +2262,6 @@ GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. -## Configure NFS (optional) - -[Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) -are recommended over NFS wherever possible for improved performance. - -See how to [configure NFS](../nfs.md). - -WARNING: -Engineering support for NFS for Git repositories is deprecated, and [technical support is scheduled to be unavailable](../nfs.md#gitaly-and-nfs-deprecation) -after the release of GitLab 15.6. No further enhancements are planned for this feature. - -Read: - -- [Gitaly and NFS Deprecation](../nfs.md#gitaly-and-nfs-deprecation). -- About the [correct mount options to use](../nfs.md#upgrade-to-gitaly-cluster-or-disable-caching-if-experiencing-data-loss). - ## Configure Advanced Search You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) @@ -2338,12 +2311,10 @@ Refer to [epic 6127](https://gitlab.com/groups/gitlab-org/-/epics/6127) for more The following tables and diagram detail the hybrid environment using the same formats as the normal environment above. -First are the components that run in Kubernetes. The recommendation at this time is to -use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EKS) and associated machine types, but the memory -and CPU requirements should translate to most other providers. We hope to update this in the -future with further specific cloud provider details. +First are the components that run in Kubernetes. These run across several node groups, although you can change +the overall makeup as desired as long as the minimum CPU and Memory requirements are observed. -| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | +| Service Node Group | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | |---------------------|-------|-------------------------|-----------------|--------------|---------------------------------| | Webservice | 7 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | 223 vCPU, 206.5 GB memory | | Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 15.5 vCPU, 50 GB memory | @@ -2352,7 +2323,7 @@ future with further specific cloud provider details. - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. - Nodes configuration is shown as it is forced to ensure pod vCPU / memory ratios and avoid scaling during **performance testing**. - - In production deployments, there is no need to assign pods to nodes. A minimum of three nodes in three different availability zones is strongly recommended to align with resilient cloud architecture practices. + - In production deployments, there is no need to assign pods to specific nodes. A minimum of three nodes per node group in three different availability zones is strongly recommended to align with resilient cloud architecture practices. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS services where applicable): @@ -2362,7 +2333,7 @@ services where applicable): | Consul<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | PostgreSQL<sup>1</sup> | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | | PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Internal load balancing node<sup>3</sup> | 1 | 4 vCPU, 3.6GB memory | `n1-highcpu-4` | `c5.xlarge` | +| Internal load balancing node<sup>3</sup> | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | | Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | | Gitaly<sup>5 6</sup> | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | @@ -2374,7 +2345,8 @@ services where applicable): <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. - [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. - - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB clusters](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - Note that [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is **incompatible** with load balancing enabled by default in [14.4.0](../../update/index.md#1440). - Consul is primarily used for Omnibus PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However, Consul is also used optionally by Prometheus for Omnibus auto host discovery. 2. Can be optionally run on reputable third-party external PaaS Redis solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index 1acae93f764..f41c8e9cb24 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -29,12 +29,12 @@ For a full list of reference architectures, see | GitLab Rails | 2 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | | Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | | Object storage<sup>4</sup> | - | - | - | - | - | -| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. - [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. - - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB clusters](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - Note that [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is **incompatible** with load balancing enabled by default in [14.4.0](../../update/index.md#1440), and [Azure Database for PostgreSQL](https://azure.microsoft.com/en-gb/products/postgresql/#overview) is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). - Consul is primarily used for Omnibus PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However, Consul is also used optionally by Prometheus for Omnibus auto host discovery. 2. Can be optionally run on reputable third-party external PaaS Redis solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. @@ -125,7 +125,7 @@ CI pipelines alike. As such, large repositories come with notable cost and typically will require more resources to handle, significantly so in some cases. It's therefore **strongly** recommended then to review large repositories -to ensure they maintain good repo health and reduce their size wherever possible. +to ensure they maintain good health and reduce their size wherever possible. NOTE: If best practices aren't followed and large repositories are present on the environment, @@ -151,9 +151,6 @@ To set up GitLab and its components to accommodate up to 2,000 users: environment. 1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) - to have shared disk storage service as an alternative to Gitaly or object - storage. 1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, more advanced code search across your entire GitLab instance. @@ -700,8 +697,8 @@ On each node perform the following: puma['listen'] = '0.0.0.0' sidekiq['listen_address'] = "0.0.0.0" - # Configure Sidekiq with 2 workers and 10 max concurrency - sidekiq['max_concurrency'] = 10 + # Configure Sidekiq with 2 workers and 20 max concurrency + sidekiq['max_concurrency'] = 20 sidekiq['queue_groups'] = ['*'] * 2 # Add the monitoring node's IP address to the monitoring whitelist and allow it to @@ -780,9 +777,7 @@ On each node perform the following: [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -1. [Enable incremental logging](#enable-incremental-logging), unless you are using [NFS](#configure-nfs-optional). - +1. [Enable incremental logging](#enable-incremental-logging). 1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the node can connect to Gitaly. 1. Tail the logs to see the requests: @@ -930,9 +925,6 @@ running [Prometheus](../monitoring/prometheus/index.md) and ## Configure the object storage GitLab supports using an object storage service for holding numerous types of data. -It's recommended over [NFS](#configure-nfs-optional) and in general it's better -in larger setups as object storage is typically much more performant, reliable, -and scalable. GitLab has been tested on a number of object storage providers: @@ -957,7 +949,7 @@ NOTE: When using the [storage-specific form](../object_storage.md#storage-specific-configuration) in GitLab 14.x and earlier, you should enable [direct upload mode](../../development/uploads/index.md#direct-upload). The previous [background upload](../../development/uploads/index.md#direct-upload) mode, -which was deprecated in 14.9, requires shared storage such as [NFS](#configure-nfs-optional). +which was deprecated in 14.9, requires shared storage such as NFS. Using separate buckets for each data type is the recommended approach for GitLab. This ensures there are no collisions across the various types of data GitLab stores. @@ -976,23 +968,6 @@ GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. -## Configure NFS (optional) - -For improved performance, [object storage](#configure-the-object-storage), -along with [Gitaly](#configure-gitaly), are recommended over using NFS whenever -possible. - -See how to [configure NFS](../nfs.md). - -WARNING: -Engineering support for NFS for Git repositories is deprecated, and [technical support is scheduled to be unavailable](../nfs.md#gitaly-and-nfs-deprecation) -after the release of GitLab 15.6. No further enhancements are planned for this feature. - -Read: - -- [Gitaly and NFS Deprecation](../nfs.md#gitaly-and-nfs-deprecation). -- About the [correct mount options to use](../nfs.md#upgrade-to-gitaly-cluster-or-disable-caching-if-experiencing-data-loss). - ## Configure Advanced Search **(PREMIUM SELF)** You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) @@ -1046,12 +1021,10 @@ Refer to [epic 6127](https://gitlab.com/groups/gitlab-org/-/epics/6127) for more The following tables and diagram detail the hybrid environment using the same formats as the normal environment above. -First are the components that run in Kubernetes. The recommendation at this time is to -use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EKS) and associated machine types, but the memory -and CPU requirements should translate to most other providers. We hope to update this in the -future with further specific cloud provider details. +First are the components that run in Kubernetes. These run across several node groups, although you can change +the overall makeup as desired as long as the minimum CPU and Memory requirements are observed. -| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | +| Service Node Group | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | |---------------------|-------|------------------------|-----------------|--------------|---------------------------------| | Webservice | 3 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | 23.7 vCPU, 16.9 GB memory | | Sidekiq | 2 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 7.8 vCPU, 25.9 GB memory | @@ -1060,7 +1033,7 @@ future with further specific cloud provider details. - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. - Nodes configuration is shown as it is forced to ensure pod vCPU / memory ratios and avoid scaling during **performance testing**. - - In production deployments, there is no need to assign pods to nodes. A minimum of three nodes in three different availability zones is strongly recommended to align with resilient cloud architecture practices. + - In production deployments, there is no need to assign pods to specific nodes. A minimum of three nodes per node group in three different availability zones is strongly recommended to align with resilient cloud architecture practices. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS services where applicable): @@ -1076,7 +1049,8 @@ services where applicable): <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. - [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. - - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB clusters](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - Note that [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is **incompatible** with load balancing enabled by default in [14.4.0](../../update/index.md#1440), and [Azure Database for PostgreSQL](https://azure.microsoft.com/en-gb/products/postgresql/#overview) is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). - Consul is primarily used for Omnibus PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However, Consul is also used optionally by Prometheus for Omnibus auto host discovery. 2. Can be optionally run on reputable third-party external PaaS Redis solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 4fc6af3f72e..008b5ffcc0e 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -44,13 +44,13 @@ For a full list of reference architectures, see | GitLab Rails | 3 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | | Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Object storage<sup>4</sup> | - | - | - | - | -| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. - [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. - - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB clusters](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - Note that [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is **incompatible** with load balancing enabled by default in [14.4.0](../../update/index.md#1440). - Consul is primarily used for Omnibus PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However, Consul is also used optionally by Prometheus for Omnibus auto host discovery. 2. Can be optionally run on reputable third-party external PaaS Redis solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. @@ -195,7 +195,7 @@ CI pipelines alike. As such, large repositories come with notable cost and typically will require more resources to handle, significantly so in some cases. It's therefore **strongly** recommended then to review large repositories -to ensure they maintain good repo health and reduce their size wherever possible. +to ensure they maintain good health and reduce their size wherever possible. NOTE: If best practices aren't followed and large repositories are present on the environment, @@ -233,9 +233,6 @@ To set up GitLab and its components to accommodate up to 3,000 users: environment. 1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) - to have shared disk storage service as an alternative to Gitaly or object - storage. 1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, more advanced code search across your entire GitLab instance. @@ -1230,7 +1227,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - postgresql['max_connections'] = 200 + postgresql['max_connections'] = 500 # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1711,9 +1708,8 @@ To configure Praefect with TLS: Sidekiq requires connection to the [Redis](#configure-redis), [PostgreSQL](#configure-postgresql) and [Gitaly](#configure-gitaly) instances. -Since it's recommended to use [Object storage](#configure-the-object-storage) -over [NFS](#configure-nfs-optional) for data objects, the following examples -include the Object storage configuration. +Because you must use [Object storage](#configure-the-object-storage) instead of NFS for data objects, the following +examples include the Object storage configuration. The following IPs will be used as an example: @@ -1794,8 +1790,8 @@ Updates to example must be made at: ## Set number of Sidekiq queue processes to the same number as available CPUs sidekiq['queue_groups'] = ['*'] * 2 - ## Set number of Sidekiq threads per queue process to the recommend number of 10 - sidekiq['max_concurrency'] = 10 + ## Set number of Sidekiq threads per queue process to the recommend number of 20 + sidekiq['max_concurrency'] = 20 # Monitoring consul['enable'] = true @@ -1869,7 +1865,7 @@ Updates to example must be made at: NOTE: If you find that the environment's Sidekiq job processing is slow with long queues, more nodes can be added as required. You can also tune your Sidekiq nodes to -run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). +run [multiple Sidekiq processes](../sidekiq/extra_sidekiq_processes.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1880,9 +1876,8 @@ run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). ## Configure GitLab Rails This section describes how to configure the GitLab application (Rails) component. -Since it's recommended to use [Object storage](#configure-the-object-storage) -over [NFS](#configure-nfs-optional) for data objects, the following examples -include the Object storage configuration. +Because you must use [Object storage](#configure-the-object-storage) instead of NFS for data objects, the following +examples include the Object storage configuration. On each node perform the following: @@ -2021,7 +2016,6 @@ On each node perform the following: 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. - 1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: ```shell @@ -2032,11 +2026,8 @@ On each node perform the following: [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -1. [Enable incremental logging](#enable-incremental-logging), unless you are using [NFS](#configure-nfs-optional). - +1. [Enable incremental logging](#enable-incremental-logging). 1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the node can connect to Gitaly. - 1. Tail the logs to see the requests: ```shell @@ -2175,9 +2166,6 @@ running [Prometheus](../monitoring/prometheus/index.md) and ## Configure the object storage GitLab supports using an object storage service for holding numerous types of data. -It's recommended over [NFS](#configure-nfs-optional) and in general it's better -in larger setups as object storage is typically much more performant, reliable, -and scalable. GitLab has been tested on a number of object storage providers: @@ -2201,7 +2189,7 @@ NOTE: When using the [storage-specific form](../object_storage.md#storage-specific-configuration) in GitLab 14.x and earlier, you should enable [direct upload mode](../../development/uploads/index.md#direct-upload). The previous [background upload](../../development/uploads/index.md#direct-upload) mode, -which was deprecated in 14.9, requires shared storage such as [NFS](#configure-nfs-optional). +which was deprecated in 14.9, requires shared storage such as NFS. Using separate buckets for each data type is the recommended approach for GitLab. This ensures there are no collisions across the various types of data GitLab stores. @@ -2220,22 +2208,6 @@ GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. -## Configure NFS (optional) - -[Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) -are recommended over NFS wherever possible for improved performance. - -See how to [configure NFS](../nfs.md). - -WARNING: -Engineering support for NFS for Git repositories is deprecated, and [technical support is scheduled to be unavailable](../nfs.md#gitaly-and-nfs-deprecation) -after the release of GitLab 15.6. No further enhancements are planned for this feature. - -Read: - -- [Gitaly and NFS Deprecation](../nfs.md#gitaly-and-nfs-deprecation). -- About the [correct mount options to use](../nfs.md#upgrade-to-gitaly-cluster-or-disable-caching-if-experiencing-data-loss). - ## Configure Advanced Search You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) @@ -2309,12 +2281,10 @@ Refer to [epic 6127](https://gitlab.com/groups/gitlab-org/-/epics/6127) for more The following tables and diagram detail the hybrid environment using the same formats as the normal environment above. -First are the components that run in Kubernetes. The recommendation at this time is to -use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EKS) and associated machine types, but the memory -and CPU requirements should translate to most other providers. We hope to update this in the -future with further specific cloud provider details. +First are the components that run in Kubernetes. These run across several node groups, although you can change +the overall makeup as desired as long as the minimum CPU and Memory requirements are observed. -| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | +| Service Node Group | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | |---------------------|-------|-------------------------|-----------------|--------------|---------------------------------| | Webservice | 2 | 16 vCPU, 14.4 GB memory | `n1-highcpu-16` | `c5.4xlarge` | 31.8 vCPU, 24.8 GB memory | | Sidekiq | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 11.8 vCPU, 38.9 GB memory | @@ -2323,7 +2293,7 @@ future with further specific cloud provider details. - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. - Nodes configuration is shown as it is forced to ensure pod vCPU / memory ratios and avoid scaling during **performance testing**. - - In production deployments, there is no need to assign pods to nodes. A minimum of three nodes in three different availability zones is strongly recommended to align with resilient cloud architecture practices. + - In production deployments, there is no need to assign pods to specific nodes. A minimum of three nodes per node group in three different availability zones is strongly recommended to align with resilient cloud architecture practices. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS services where applicable): @@ -2344,7 +2314,8 @@ services where applicable): <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. - [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. - - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB clusters](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - Note that [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is **incompatible** with load balancing enabled by default in [14.4.0](../../update/index.md#1440). - Consul is primarily used for Omnibus PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However, Consul is also used optionally by Prometheus for Omnibus auto host discovery. 2. Can be optionally run on reputable third-party external PaaS Redis solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index ca159d62f1f..87d1408b568 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -35,13 +35,13 @@ full list of reference architectures, see | GitLab Rails | 12 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | | Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Object storage<sup>4</sup> | - | - | - | - | -| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. - [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. - - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB clusters](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - Note that [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is **incompatible** with load balancing enabled by default in [14.4.0](../../update/index.md#1440). - Consul is primarily used for Omnibus PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However, Consul is also used optionally by Prometheus for Omnibus auto host discovery. 2. Can be optionally run on reputable third-party external PaaS Redis solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. @@ -189,7 +189,7 @@ CI pipelines alike. As such, large repositories come with notable cost and typically will require more resources to handle, significantly so in some cases. It's therefore **strongly** recommended then to review large repositories -to ensure they maintain good repo health and reduce their size wherever possible. +to ensure they maintain good health and reduce their size wherever possible. NOTE: If best practices aren't followed and large repositories are present on the environment, @@ -227,9 +227,6 @@ To set up GitLab and its components to accommodate up to 50,000 users: environment. 1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) - to have shared disk storage service as an alternative to Gitaly or object - storage. 1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, more advanced code search across your entire GitLab instance. @@ -296,8 +293,8 @@ could also be used, those load balancers have not been validated. ### Balancing algorithm -We recommend that a least-connection load balancing algorithm or equivalent -is used wherever possible to ensure equal spread of calls to the nodes and good performance. +You should use a least-connection load balancing algorithm or equivalent +wherever possible to ensure equal spread of calls to the nodes and good performance. We don't recommend the use of round-robin algorithms as they are known to not spread connections equally in practice. @@ -1288,7 +1285,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - postgresql['max_connections'] = 200 + postgresql['max_connections'] = 500 # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1772,9 +1769,8 @@ To configure Praefect with TLS: Sidekiq requires connection to the [Redis](#configure-redis), [PostgreSQL](#configure-postgresql) and [Gitaly](#configure-gitaly) instances. -Since it's recommended to use [Object storage](#configure-the-object-storage) -over [NFS](#configure-nfs-optional) for data objects, the following examples -include the Object storage configuration. +Because you must use [Object storage](#configure-the-object-storage) instead of NFS for data objects, the following +examples include the Object storage configuration. - `10.6.0.101`: Sidekiq 1 - `10.6.0.102`: Sidekiq 2 @@ -1872,8 +1868,8 @@ Updates to example must be made at: ## Set number of Sidekiq queue processes to the same number as available CPUs sidekiq['queue_groups'] = ['*'] * 4 - ## Set number of Sidekiq threads per queue process to the recommend number of 10 - sidekiq['max_concurrency'] = 10 + ## Set number of Sidekiq threads per queue process to the recommend number of 20 + sidekiq['max_concurrency'] = 20 # Monitoring consul['enable'] = true @@ -1931,7 +1927,7 @@ Updates to example must be made at: NOTE: If you find that the environment's Sidekiq job processing is slow with long queues, more nodes can be added as required. You can also tune your Sidekiq nodes to -run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). +run [multiple Sidekiq processes](../sidekiq/extra_sidekiq_processes.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1942,9 +1938,8 @@ run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). ## Configure GitLab Rails This section describes how to configure the GitLab application (Rails) component. -Since it's recommended to use [Object storage](#configure-the-object-storage) -over [NFS](#configure-nfs-optional) for data objects, the following examples -include the Object storage configuration. +Because you must use [Object storage](#configure-the-object-storage) instead of NFS for data objects, the following +examples include the Object storage configuration. The following IPs will be used as an example: @@ -2092,7 +2087,6 @@ On each node perform the following: 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. - 1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: ```shell @@ -2103,9 +2097,7 @@ On each node perform the following: [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -1. [Enable incremental logging](#enable-incremental-logging), unless you are using [NFS](#configure-nfs-optional). - +1. [Enable incremental logging](#enable-incremental-logging). 1. Confirm the node can connect to Gitaly: ```shell @@ -2230,9 +2222,6 @@ To configure the Monitoring node: ## Configure the object storage GitLab supports using an object storage service for holding numerous types of data. -It's recommended over [NFS](#configure-nfs-optional) and in general it's better -in larger setups as object storage is typically much more performant, reliable, -and scalable. GitLab has been tested on a number of object storage providers: @@ -2256,7 +2245,7 @@ NOTE: When using the [storage-specific form](../object_storage.md#storage-specific-configuration) in GitLab 14.x and earlier, you should enable [direct upload mode](../../development/uploads/index.md#direct-upload). The previous [background upload](../../development/uploads/index.md#direct-upload) mode, -which was deprecated in 14.9, requires shared storage such as [NFS](#configure-nfs-optional). +which was deprecated in 14.9, requires shared storage such as NFS. Using separate buckets for each data type is the recommended approach for GitLab. This ensures there are no collisions across the various types of data GitLab stores. @@ -2275,22 +2264,6 @@ GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. -## Configure NFS (optional) - -[Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) -are recommended over NFS wherever possible for improved performance. - -See how to [configure NFS](../nfs.md). - -WARNING: -Engineering support for NFS for Git repositories is deprecated, and [technical support is scheduled to be unavailable](../nfs.md#gitaly-and-nfs-deprecation) -after the release of GitLab 15.6. No further enhancements are planned for this feature. - -Read: - -- [Gitaly and NFS Deprecation](../nfs.md#gitaly-and-nfs-deprecation). -- About the [correct mount options to use](../nfs.md#upgrade-to-gitaly-cluster-or-disable-caching-if-experiencing-data-loss). - ## Configure Advanced Search You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) @@ -2340,12 +2313,10 @@ Refer to [epic 6127](https://gitlab.com/groups/gitlab-org/-/epics/6127) for more The following tables and diagram detail the hybrid environment using the same formats as the normal environment above. -First are the components that run in Kubernetes. The recommendation at this time is to -use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EKS) and associated machine types, but the memory -and CPU requirements should translate to most other providers. We hope to update this in the -future with further specific cloud provider details. +First are the components that run in Kubernetes. These run across several node groups, although you can change +the overall makeup as desired as long as the minimum CPU and Memory requirements are observed. -| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | +| Service Node Group | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | |---------------------|-------|-------------------------|-----------------|--------------|---------------------------------| | Webservice | 16 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `m5.8xlarge` | 510 vCPU, 472 GB memory | | Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 15.5 vCPU, 50 GB memory | @@ -2354,7 +2325,7 @@ future with further specific cloud provider details. - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. - Nodes configuration is shown as it is forced to ensure pod vCPU / memory ratios and avoid scaling during **performance testing**. - - In production deployments, there is no need to assign pods to nodes. A minimum of three nodes in three different availability zones is strongly recommended to align with resilient cloud architecture practices. + - In production deployments, there is no need to assign pods to specific nodes. A minimum of three nodes per node group in three different availability zones is strongly recommended to align with resilient cloud architecture practices. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS services where applicable): @@ -2376,7 +2347,8 @@ services where applicable): <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. - [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. - - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB clusters](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - Note that [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is **incompatible** with load balancing enabled by default in [14.4.0](../../update/index.md#1440). - Consul is primarily used for Omnibus PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However, Consul is also used optionally by Prometheus for Omnibus auto host discovery. 2. Can be optionally run on reputable third-party external PaaS Redis solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index a2b92f9c300..182edb82b5f 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -41,13 +41,13 @@ costly-to-operate environment by using the | GitLab Rails | 3 | 16 vCPU, 14.4 GB memory | `n1-highcpu-16` | `c5.4xlarge` | | Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Object storage<sup>4</sup> | - | - | - | - | -| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. - [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. - - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB clusters](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - Note that [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is **incompatible** with load balancing enabled by default in [14.4.0](../../update/index.md#1440). - Consul is primarily used for Omnibus PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However, Consul is also used optionally by Prometheus for Omnibus auto host discovery. 2. Can be optionally run on reputable third-party external PaaS Redis solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. @@ -192,7 +192,7 @@ CI pipelines alike. As such, large repositories come with notable cost and typically will require more resources to handle, significantly so in some cases. It's therefore **strongly** recommended then to review large repositories -to ensure they maintain good repo health and reduce their size wherever possible. +to ensure they maintain good health and reduce their size wherever possible. NOTE: If best practices aren't followed and large repositories are present on the environment, @@ -230,9 +230,6 @@ To set up GitLab and its components to accommodate up to 5,000 users: environment. 1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure NFS](#configure-nfs-optional) (optional, and not recommended) - to have shared disk storage service as an alternative to Gitaly or object - storage. 1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, more advanced code search across your entire GitLab instance. @@ -1226,7 +1223,7 @@ in the second step, do not supply the `EXTERNAL_URL` value. # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' - postgresql['max_connections'] = 200 + postgresql['max_connections'] = 500 # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1708,9 +1705,8 @@ To configure Praefect with TLS: Sidekiq requires connection to the [Redis](#configure-redis), [PostgreSQL](#configure-postgresql) and [Gitaly](#configure-gitaly) instances. -Since it's recommended to use [Object storage](#configure-the-object-storage) -over [NFS](#configure-nfs-optional) for data objects, the following examples -include the Object storage configuration. +Because you must use [Object storage](#configure-the-object-storage) instead of NFS for data objects, the following +examples include the Object storage configuration. - `10.6.0.71`: Sidekiq 1 - `10.6.0.72`: Sidekiq 2 @@ -1790,8 +1786,8 @@ Updates to example must be made at: ## Set number of Sidekiq queue processes to the same number as available CPUs sidekiq['queue_groups'] = ['*'] * 4 - ## Set number of Sidekiq threads per queue process to the recommend number of 10 - sidekiq['max_concurrency'] = 10 + ## Set number of Sidekiq threads per queue process to the recommend number of 20 + sidekiq['max_concurrency'] = 20 # Monitoring consul['enable'] = true @@ -1865,7 +1861,7 @@ Updates to example must be made at: NOTE: If you find that the environment's Sidekiq job processing is slow with long queues, more nodes can be added as required. You can also tune your Sidekiq nodes to -run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). +run [multiple Sidekiq processes](../sidekiq/extra_sidekiq_processes.md). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1876,9 +1872,8 @@ run [multiple Sidekiq processes](../operations/extra_sidekiq_processes.md). ## Configure GitLab Rails This section describes how to configure the GitLab application (Rails) component. -Since it's recommended to use [Object storage](#configure-the-object-storage) -over [NFS](#configure-nfs-optional) for data objects, the following examples -include the Object storage configuration. +Because you must use [Object storage](#configure-the-object-storage) instead of NFS for data objects, the following +examples include the Object storage configuration. On each node perform the following: @@ -2020,7 +2015,6 @@ On each node perform the following: 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step. - 1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run: ```shell @@ -2031,11 +2025,8 @@ On each node perform the following: [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -1. [Enable incremental logging](#enable-incremental-logging), unless you are using [NFS](#configure-nfs-optional). - +1. [Enable incremental logging](#enable-incremental-logging). 1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the node can connect to Gitaly. - 1. Tail the logs to see the requests: ```shell @@ -2174,9 +2165,6 @@ running [Prometheus](../monitoring/prometheus/index.md) and ## Configure the object storage GitLab supports using an object storage service for holding numerous types of data. -It's recommended over [NFS](#configure-nfs-optional) and in general it's better -in larger setups as object storage is typically much more performant, reliable, -and scalable. GitLab has been tested on a number of object storage providers: @@ -2200,7 +2188,7 @@ NOTE: When using the [storage-specific form](../object_storage.md#storage-specific-configuration) in GitLab 14.x and earlier, you should enable [direct upload mode](../../development/uploads/index.md#direct-upload). The previous [background upload](../../development/uploads/index.md#direct-upload) mode, -which was deprecated in 14.9, requires shared storage such as [NFS](#configure-nfs-optional). +which was deprecated in 14.9, requires shared storage such as NFS. Using separate buckets for each data type is the recommended approach for GitLab. This ensures there are no collisions across the various types of data GitLab stores. @@ -2219,22 +2207,6 @@ GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. -## Configure NFS (optional) - -[Object storage](#configure-the-object-storage), along with [Gitaly](#configure-gitaly) -are recommended over NFS wherever possible for improved performance. - -See how to [configure NFS](../nfs.md). - -WARNING: -Engineering support for NFS for Git repositories is deprecated, and [technical support is scheduled to be unavailable](../nfs.md#gitaly-and-nfs-deprecation) -after the release of GitLab 15.6. No further enhancements are planned for this feature. - -Read: - -- [Gitaly and NFS Deprecation](../nfs.md#gitaly-and-nfs-deprecation). -- About the [correct mount options to use](../nfs.md#upgrade-to-gitaly-cluster-or-disable-caching-if-experiencing-data-loss). - ## Configure Advanced Search You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) @@ -2284,16 +2256,14 @@ Refer to [epic 6127](https://gitlab.com/groups/gitlab-org/-/epics/6127) for more The following tables and diagram detail the hybrid environment using the same formats as the normal environment above. -First are the components that run in Kubernetes. The recommendation at this time is to -use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EKS) and associated machine types, but the memory -and CPU requirements should translate to most other providers. We hope to update this in the -future with further specific cloud provider details. +First are the components that run in Kubernetes. These run across several node groups, although you can change +the overall makeup as desired as long as the minimum CPU and Memory requirements are observed. -| Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | -|-----------------------------------------------|-------|-------------------------|-----------------|--------------|---------------------------------| -| Webservice | 5 | 16 vCPU, 14.4 GB memory | `n1-highcpu-16` | `c5.4xlarge` | 79.5 vCPU, 62 GB memory | -| Sidekiq | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 11.8 vCPU, 38.9 GB memory | -| Supporting services such as NGINX, Prometheus | 2 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | 3.9 vCPU, 11.8 GB memory | +| Service Node Group | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | +|-------------------- |-------|-------------------------|-----------------|--------------|---------------------------------| +| Webservice | 5 | 16 vCPU, 14.4 GB memory | `n1-highcpu-16` | `c5.4xlarge` | 79.5 vCPU, 62 GB memory | +| Sidekiq | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 11.8 vCPU, 38.9 GB memory | +| Supporting services | 2 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | 3.9 vCPU, 11.8 GB memory | - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. @@ -2319,7 +2289,8 @@ services where applicable): <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. - [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work. - - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB clusters](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - Note that [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is **incompatible** with load balancing enabled by default in [14.4.0](../../update/index.md#1440). - Consul is primarily used for Omnibus PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However, Consul is also used optionally by Prometheus for Omnibus auto host discovery. 2. Can be optionally run on reputable third-party external PaaS Redis solutions. See [Recommended cloud providers and services](index.md#recommended-cloud-providers-and-services) for more information. diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index 467cc332e25..60258fb5a09 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -207,7 +207,8 @@ Several cloud provider services are known not to support the above or have been - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is incompatible and not supported. See [14.4.0](../../update/index.md#1440) for more details. - [Azure Database for PostgreSQL Single Server](https://azure.microsoft.com/en-gb/products/postgresql/#overview) (Single / Flexible) is **strongly not recommended** for use due to notable performance / stability issues or missing functionality. See [Recommendation Notes for Azure](#recommendation-notes-for-azure) for more details. -- [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB clusters](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. +- [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. + - Note that [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. ### Recommendation notes for Azure @@ -216,7 +217,7 @@ Due to performance issues that we found with several key Azure services, we only In addition to the above, you should be aware of the additional specific guidance for Azure: - **We outright strongly do not recommend [Azure Database for PostgreSQL Single Server](https://learn.microsoft.com/en-us/azure/postgresql/single-server/overview-single-server)** specifically due to significant performance and stability issues found. **For GitLab 14.0 and higher the service is not supported** due to it only supporting up to PostgreSQL 11. - - A new service, [Azure Database for Postgres Flexible Server](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/) has been released but due to it missing some functionality we don't recommend it at this time. + - A new service, [Azure Database for PostgreSQL Flexible Server](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/) has been released but due to it missing some functionality we don't recommend it at this time. - [Azure Blob Storage](https://azure.microsoft.com/en-gb/products/storage/blobs/) has been found to have performance limits that can impact production use at certain times. However, this has only been seen in larger architectures. ## Validation and test results @@ -240,11 +241,11 @@ Testing occurs against all reference architectures and cloud providers in an aut - The [GitLab Environment Toolkit](https://gitlab.com/gitlab-org/gitlab-environment-toolkit) for building the environments. - The [GitLab Performance Tool](https://gitlab.com/gitlab-org/quality/performance) for performance testing. -Network latency on the test environments between components on all Cloud Providers were measured at <5ms. Note that this is shared as an observation and not as an implicit recommendation. +Network latency on the test environments between components on all Cloud Providers were measured at <5 ms. Note that this is shared as an observation and not as an implicit recommendation. We aim to have a "test smart" approach where architectures tested have a good range that can also apply to others. Testing focuses on 10k Omnibus on GCP as the testing has shown this is a good bellwether for the other architectures and cloud providers as well as Cloud Native Hybrids. -The Standard Reference Architectures are designed to be platform-agnostic, with everything being run on VMs via [Omnibus GitLab](https://docs.gitlab.com/omnibus/). While testing occurs primarily on GCP, ad-hoc testing has shown that they perform similarly on equivalently specced hardware on other Cloud Providers or if run on premises (bare-metal). +The Standard Reference Architectures are designed to be platform-agnostic, with everything being run on VMs via [Omnibus GitLab](https://docs.gitlab.com/omnibus/). While testing occurs primarily on GCP, ad-hoc testing has shown that they perform similarly on hardware with equivalent specs on other Cloud Providers or if run on premises (bare-metal). Testing on these reference architectures is performed with the [GitLab Performance Tool](https://gitlab.com/gitlab-org/quality/performance) diff --git a/doc/administration/restart_gitlab.md b/doc/administration/restart_gitlab.md index 1a1194e16a9..7996db3d1e1 100644 --- a/doc/administration/restart_gitlab.md +++ b/doc/administration/restart_gitlab.md @@ -14,7 +14,7 @@ A short downtime is expected for all methods. ## Omnibus installations -If you have used the [Omnibus packages](https://about.gitlab.com/install/) to install GitLab, then +If you have used the [Omnibus packages](https://about.gitlab.com/install/) to install GitLab, you should already have `gitlab-ctl` in your `PATH`. `gitlab-ctl` interacts with the Omnibus packages and can be used to restart the @@ -88,16 +88,14 @@ sudo gitlab-ctl reconfigure Reconfiguring GitLab should occur in the event that something in its configuration (`/etc/gitlab/gitlab.rb`) has changed. -When you run this command, [Chef](https://www.chef.io/products/chef-infra), the underlying configuration management -application that powers Omnibus GitLab, makes sure that all things like directories, -permissions, and services are in place and in the same shape that they were -initially shipped. +When you run `gitlab-ctl reconfigure`, [Chef](https://www.chef.io/products/chef-infra), +the underlying configuration management application that powers Omnibus GitLab, runs some checks. +Chef ensures directories, permissions, and services are in place and working. -It also [restarts GitLab components](#how-to-restart-gitlab) -where needed, if any of their configuration files have changed. +Chef also [restarts GitLab components](#how-to-restart-gitlab) if any of their configuration files have changed. If you manually edit any files in `/var/opt/gitlab` that are managed by Chef, -running reconfigure reverts the changes and restarts the services that +running `reconfigure` reverts the changes and restarts the services that depend on those files. ## Installations from source @@ -118,7 +116,7 @@ This should restart Puma, Sidekiq, GitLab Workhorse, and [Mailroom](reply_by_ema ## Helm chart installations -There is no single command to restart the entire GitLab application installed via +There is no single command to restart the entire GitLab application installed through the [cloud-native Helm chart](https://docs.gitlab.com/charts/). Usually, it should be enough to restart a specific component separately (for example, `gitaly`, `puma`, `workhorse`, or `gitlab-shell`) by deleting all the pods related to it: diff --git a/doc/administration/server_hooks.md b/doc/administration/server_hooks.md index 448becb32dc..3d4f39b5ff0 100644 --- a/doc/administration/server_hooks.md +++ b/doc/administration/server_hooks.md @@ -47,15 +47,30 @@ To create server hooks for a repository: `pre-receive` server hook, the filename should be `pre-receive` with no extension. - To create many server hooks, create a directory for the hooks that matches the hook type. For example, for a `pre-receive` server hook, the directory name should be `pre-receive.d`. Put the files for the hook in that directory. -1. Make the server hook files executable and ensure that they are owned by the Git user. +1. **Make the server hook files executable** and ensure that they are owned by the Git user. 1. Write the code to make the server hook function as expected. Git server hooks can be in any programming language. Ensure the [shebang](https://en.wikipedia.org/wiki/Shebang_(Unix)) at the top reflects the language type. For example, if the script is in Ruby the shebang is probably `#!/usr/bin/env ruby`. -1. Make the hook file executable, ensure that it's owned by the Git user, and ensure it does not match the backup file +1. Ensure the hook file does not match the backup file pattern (`*~`). If the server hook code is properly implemented, it should execute when the Git hook is next triggered. +### Gitaly Cluster + +If you use [Gitaly Cluster](gitaly/index.md), the scripts must be copied to every Gitaly node that has a replica of the repository. Every Gitaly node +needs a copy because any node can be made a primary at any time. Server hooks only run on primary nodes. + +The location to copy the scripts to depends on where repositories are stored: + +- In GitLab 15.2 and earlier, Gitaly Cluster uses the [hashed storage path](repository_storage_types.md#hashed-storage) + reported by the GitLab application. +- In GitLab 15.3 and later, new repositories are created using + [Praefect-generated replica paths](gitaly/index.md#praefect-generated-replica-paths-gitlab-150-and-later), + which are not the hashed storage path. The replica path can be identified by + [querying the Praefect repository metadata](../administration/gitaly/troubleshooting.md#view-repository-metadata) + using `-relative-path` to specify the expected GitLab hashed storage path. + ## Create global server hooks for all repositories To create a Git hook that applies to all repositories, set a global server hook. Global server hooks also apply to: diff --git a/doc/administration/sidekiq.md b/doc/administration/sidekiq.md deleted file mode 100644 index 01f83f98607..00000000000 --- a/doc/administration/sidekiq.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'sidekiq/index.md' -remove_date: '2022-11-11' ---- - -This document was moved to [another location](sidekiq/index.md). - -<!-- This redirect file can be deleted after <2022-11-11>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/sidekiq/extra_sidekiq_processes.md b/doc/administration/sidekiq/extra_sidekiq_processes.md index feaaa55aa59..d5007e9a3e9 100644 --- a/doc/administration/sidekiq/extra_sidekiq_processes.md +++ b/doc/administration/sidekiq/extra_sidekiq_processes.md @@ -6,91 +6,41 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Run multiple Sidekiq processes **(FREE SELF)** -GitLab allows you to start multiple Sidekiq processes. -These processes can be used to consume a dedicated set -of queues. This can be used to ensure certain queues always have dedicated -workers, no matter the number of jobs to be processed. +GitLab allows you to start multiple Sidekiq processes to process background jobs +at a higher rate on a single instance. By default, Sidekiq starts one worker +process and only uses a single core. NOTE: The information in this page applies only to Omnibus GitLab. -## Available Sidekiq queues - -For a list of the existing Sidekiq queues, check the following files: - -- [Queues for both GitLab Community and Enterprise Editions](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/all_queues.yml) -- [Queues for GitLab Enterprise Editions only](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/all_queues.yml) - -Each entry in the above files represents a queue on which Sidekiq processes -can be started. - ## Start multiple processes > - [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4006) in GitLab 12.10, starting multiple processes with Sidekiq cluster. > - [Sidekiq cluster moved](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/181) to GitLab Free in 12.10. > - [Sidekiq cluster became default](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4140) in GitLab 13.0. -When starting multiple processes, the number of processes should -equal (and **not** exceed) the number of CPU cores you want to -dedicate to Sidekiq. Each Sidekiq process can use only 1 CPU -core, subject to the available workload and concurrency settings. +When starting multiple processes, the number of processes should at most equal +(and **not** exceed) the number of CPU cores you want to dedicate to Sidekiq. +The Sidekiq worker process uses no more than one CPU core. -To start multiple processes: +To start multiple processes, use the `sidekiq['queue_groups']` array setting to +specify how many processes to create using `sidekiq-cluster` and which queues +they should handle. Each item in the array equates to one additional Sidekiq +process, and values in each item determine the queues it works on. In the vast +majority of cases, all processes should listen to all queues (see +[processing specific job classes](processing_specific_job_classes.md) for more +details). -1. Using the `sidekiq['queue_groups']` array setting, specify how many processes to - create using `sidekiq-cluster` and which queue they should handle. - Each item in the array equates to one additional Sidekiq - process, and values in each item determine the queues it works on. +For example, to create four Sidekiq processes, each listening +to all available queues: - For example, the following setting creates three Sidekiq processes, one to run on - `elastic_commit_indexer`, one to run on `mailers`, and one process running on all queues: +1. Edit `/etc/gitlab/gitlab.rb`: ```ruby - sidekiq['queue_groups'] = [ - "elastic_commit_indexer", - "mailers", - "*" - ] + sidekiq['queue_groups'] = ['*'] * 4 ``` - To have an additional Sidekiq process handle multiple queues, add multiple - queue names to its item delimited by commas. For example: - - ```ruby - sidekiq['queue_groups'] = [ - "elastic_commit_indexer, elastic_association_indexer", - "mailers", - "*" - ] - ``` - - [In GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26594) and - later, the special queue name `*` means all queues. This starts two - processes, each handling all queues: - - ```ruby - sidekiq['queue_groups'] = [ - "*", - "*" - ] - ``` - - `*` which matches all workers. - As a result, the wildcard query must stay at the end of the list or the rules after it are ignored. - - `*` cannot be combined with concrete queue names - `*, mailers` - just handles the `mailers` queue. - - When `sidekiq-cluster` is only running on a single node, make sure that at least - one process is running on all queues using `*`. This ensures a process - automatically picks up jobs in queues created in the future, - including queues that have dedicated processes. - - If `sidekiq-cluster` is running on more than one node, you can also use - [`--negate`](#negate-settings) and list all the queues that are already being - processed. - -1. Save the file and reconfigure GitLab for the changes to take effect: +1. Save the file and reconfigure GitLab: ```shell sudo gitlab-ctl reconfigure @@ -101,125 +51,38 @@ To view the Sidekiq processes in GitLab: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Monitoring > Background Jobs**. -## Negate settings +## Concurrency -To have the Sidekiq process work on every queue **except** the ones -you list. In this example, we exclude all import-related jobs from a Sidekiq node: +By default each process defined under `sidekiq` starts with a number of threads +that equals the number of queues, plus one spare thread, up to a maximum of 50. +For example, a process that handles all queues will use 50 threads by default. -1. Edit `/etc/gitlab/gitlab.rb` and add: - - ```ruby - sidekiq['negate'] = true - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category=importers" - ] - ``` - -1. Save the file and reconfigure GitLab for the changes to take effect: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -## Queue selector - -> - [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/45) in GitLab 12.8. -> - [Sidekiq cluster, including queue selector, moved](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/181) to GitLab Free in 12.10. -> - [Renamed from `experimental_queue_selector` to `queue_selector`](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/147) in GitLab 13.6. - -In addition to selecting queues by name, as above, the `queue_selector` option -allows queue groups to be selected in a more general way using a -[worker matching query](extra_sidekiq_routing.md#worker-matching-query). After `queue_selector` -is set, all `queue_groups` must follow the aforementioned syntax. - -In `/etc/gitlab/gitlab.rb`: - -```ruby -sidekiq['enable'] = true -sidekiq['queue_selector'] = true -sidekiq['queue_groups'] = [ - # Run all non-CPU-bound queues that are high urgency - 'resource_boundary!=cpu&urgency=high', - # Run all continuous integration and pages queues that are not high urgency - 'feature_category=continuous_integration,pages&urgency!=high', - # Run all queues - '*' -] -``` - -## Ignore all import queues - -When [importing from GitHub](../../user/project/import/github.md) or -other sources, Sidekiq might use all of its resources to perform those -operations. To set up two separate `sidekiq-cluster` processes, where -one only processes imports and the other processes all other queues: - -1. Edit `/etc/gitlab/gitlab.rb` and add: - - ```ruby - sidekiq['enable'] = true - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category=importers", - "feature_category!=importers" - ] - ``` - -1. Save the file and reconfigure GitLab for the changes to take effect: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -## Number of threads - -By default each process defined under `sidekiq` starts with a -number of threads that equals the number of queues, plus one spare thread. -For example, a process that handles the `process_commit` and `post_receive` -queues uses three threads in total. - -These thread run inside a single Ruby process, and each process -can only use a single CPU core. The usefulness of threading depends -on the work having some external dependencies to wait on, like database queries or -HTTP requests. Most Sidekiq deployments benefit from this threading, and when -running fewer queues in a process, increasing the thread count might be -even more desirable to make the most effective use of CPU resources. +These threads run inside a single Ruby process, and each process can only use a +single CPU core. The usefulness of threading depends on the work having some +external dependencies to wait on, like database queries or HTTP requests. Most +Sidekiq deployments benefit from this threading. ### Manage thread counts explicitly -The correct maximum thread count (also called concurrency) depends on the workload. -Typical values range from `1` for highly CPU-bound tasks to `15` or higher for mixed -low-priority work. A reasonable starting range is `15` to `25` for a non-specialized -deployment. +The correct maximum thread count (also called concurrency) depends on the +workload. Typical values range from `5` for highly CPU-bound tasks to `15` or +higher for mixed low-priority work. A reasonable starting range is `15` to `25` +for a non-specialized deployment. -You can find example values used by GitLab.com by searching for `concurrency:` in -[the Helm charts](https://gitlab.com/gitlab-com/gl-infra/k8s-workloads/gitlab-com/-/blob/master/releases/gitlab/values/gprd.yaml.gotmpl). -The values vary according to the work each specific deployment of Sidekiq does. -Any other specialized deployments with processes dedicated to specific queues should -have the concurrency tuned according to: -have the concurrency tuned according to: +We only recommend setting explicit concurrency by setting `min_concurrency` and +`max_concurrency` to the same value. The two values are kept for backwards +compatibility reasons, but for more predictable results, use the same value. -- The CPU usage of each type of process. -- The throughput achieved. - -Each thread requires a Redis connection, so adding threads may increase Redis -latency and potentially cause client timeouts. See the -[Sidekiq documentation about Redis](https://github.com/mperham/sidekiq/wiki/Using-Redis) for more -details. - -#### When running Sidekiq cluster (default) +For example, to set the concurrency to `20`: -Running Sidekiq cluster is the default in GitLab 13.0 and later. - -1. Edit `/etc/gitlab/gitlab.rb` and add: +1. Edit `/etc/gitlab/gitlab.rb`: ```ruby - sidekiq['min_concurrency'] = 15 - sidekiq['max_concurrency'] = 25 + sidekiq['min_concurrency'] = 20 + sidekiq['max_concurrency'] = 20 ``` -1. Save the file and reconfigure GitLab for the changes to take effect: +1. Save the file and reconfigure GitLab: ```shell sudo gitlab-ctl reconfigure @@ -231,50 +94,45 @@ the other. Setting `min_concurrency` to `0` disables the limit. For each queue group, let `N` be one more than the number of queues. The concurrency is set to: +1. `min_concurrency`, if it's equal to `max_concurrency`. 1. `N`, if it's between `min_concurrency` and `max_concurrency`. 1. `max_concurrency`, if `N` exceeds this value. 1. `min_concurrency`, if `N` is less than this value. -If `min_concurrency` is equal to `max_concurrency`, then this value is used -regardless of the number of queues. - When `min_concurrency` is greater than `max_concurrency`, it is treated as being equal to `max_concurrency`. -#### When running a single Sidekiq process - -Running a single Sidekiq process is the default in GitLab 12.10 and earlier. - -WARNING: -Running Sidekiq directly was removed in GitLab -[14.0](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/240). - -1. Edit `/etc/gitlab/gitlab.rb` and add: - - ```ruby - sidekiq['cluster'] = false - sidekiq['concurrency'] = 25 - ``` - -1. Save the file and reconfigure GitLab for the changes to take effect: +You can find example values used by GitLab.com by searching for `concurrency:` +in [the Helm charts](https://gitlab.com/gitlab-com/gl-infra/k8s-workloads/gitlab-com/-/blob/master/releases/gitlab/values/gprd.yaml.gotmpl). +The values vary according to the work each specific deployment of Sidekiq does. +Any other specialized deployments with processes dedicated to specific queues +should have the concurrency tuned according to: - ```shell - sudo gitlab-ctl reconfigure - ``` +- The CPU usage of each type of process. +- The throughput achieved. -This sets the concurrency (number of threads) for the Sidekiq process. +Each thread requires a Redis connection, so adding threads may increase Redis +latency and potentially cause client timeouts. See the [Sidekiq documentation about Redis](https://github.com/mperham/sidekiq/wiki/Using-Redis) +for more details. ## Modify the check interval -To modify `sidekiq-cluster`'s health check interval for the additional Sidekiq processes: +To modify Sidekiq's health check interval for the additional Sidekiq +processes: -1. Edit `/etc/gitlab/gitlab.rb` and add (the value can be any integer number of seconds): +1. Edit `/etc/gitlab/gitlab.rb`: ```ruby sidekiq['interval'] = 5 ``` -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + The value can be any integer number of seconds. + +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` ## Troubleshoot using the CLI @@ -291,6 +149,9 @@ takes arguments using the following syntax: /opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster [QUEUE,QUEUE,...] [QUEUE, ...] ``` +The `--dryrun` argument allows viewing the command to be executed without +actually starting it. + Each separate argument denotes a group of queues that have to be processed by a Sidekiq process. Multiple queues can be processed by the same process by separating them with a comma instead of a space. @@ -301,29 +162,6 @@ explicitly list all the queue names. For more information about queue namespaces see the relevant section in the [Sidekiq development documentation](../../development/sidekiq/index.md#queue-namespaces). -For example, say you want to start 2 extra processes: one to process the -`process_commit` queue, and one to process the `post_receive` queue. This can be -done as follows: - -```shell -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit post_receive -``` - -If you instead want to start one process processing both queues, you'd use the -following syntax: - -```shell -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit,post_receive -``` - -If you want to have one Sidekiq process dealing with the `process_commit` and -`post_receive` queues, and one process to process the `gitlab_shell` queue, -you'd use the following: - -```shell -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit,post_receive gitlab_shell -``` - ### Monitor the `sidekiq-cluster` command The `sidekiq-cluster` command does not terminate once it has started the desired diff --git a/doc/administration/sidekiq/extra_sidekiq_routing.md b/doc/administration/sidekiq/extra_sidekiq_routing.md index 56c51beb758..d1d65498fcc 100644 --- a/doc/administration/sidekiq/extra_sidekiq_routing.md +++ b/doc/administration/sidekiq/extra_sidekiq_routing.md @@ -1,163 +1,11 @@ --- -stage: Systems -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: 'processing_specific_job_classes.md#routing-rules' +remove_date: '2023-02-01' --- -# Queue routing rules **(FREE SELF)** +This document was moved to [another location](processing_specific_job_classes.md#routing-rules). -When the number of Sidekiq jobs increases to a certain scale, the system faces -some scalability issues. One of them is that the length of the queue tends to get -longer. High-urgency jobs have to wait longer until other less urgent jobs -finish. This head-of-line blocking situation may eventually affect the -responsiveness of the system, especially critical actions. In another scenario, -the performance of some jobs is degraded due to other long running or CPU-intensive jobs -(computing or rendering ones) in the same machine. - -To counter the aforementioned issues, one effective solution is to split -Sidekiq jobs into different queues and assign machines handling each queue -exclusively. For example, all CPU-intensive jobs could be routed to the -`cpu-bound` queue and handled by a fleet of CPU optimized instances. The queue -topology differs between companies depending on the workloads and usage -patterns. Therefore, GitLab supports a flexible mechanism for the -administrator to route the jobs based on their characteristics. - -As an alternative to [Queue selector](extra_sidekiq_processes.md#queue-selector), which -configures Sidekiq cluster to listen to a specific set of workers or queues, -GitLab also supports routing a job from a worker to the desired queue when it -is scheduled. Sidekiq clients try to match a job against a configured list of -routing rules. Rules are evaluated from first to last, and as soon as we find a -match for a given worker we stop processing for that worker (first match wins). -If the worker doesn't match any rule, it falls back to the queue name generated -from the worker name. - -By default, if the routing rules are not configured (or denoted with an empty -array), all the jobs are routed to the queue generated from the worker name. - -## Example configuration - -In `/etc/gitlab/gitlab.rb`: - -```ruby -sidekiq['routing_rules'] = [ - # Do not re-route workers that require their own queue - ['tags=needs_own_queue', nil], - # Route all non-CPU-bound workers that are high urgency to `high-urgency` queue - ['resource_boundary!=cpu&urgency=high', 'high-urgency'], - # Route all database, gitaly and global search workers that are throttled to `throttled` queue - ['feature_category=database,gitaly,global_search&urgency=throttled', 'throttled'], - # Route all workers having contact with outside work to a `network-intenstive` queue - ['has_external_dependencies=true|feature_category=hooks|tags=network', 'network-intensive'], - # Route all import workers to the queues generated by the worker name, for - # example, JiraImportWorker to `jira_import`, SVNWorker to `svn_worker` - ['feature_category=import', nil], - # Wildcard matching, route the rest to `default` queue - ['*', 'default'] -] -``` - -The routing rules list is an order-matter array of tuples of query and -corresponding queue: - -- The query is following a [worker matching query](#worker-matching-query) syntax. -- The `<queue_name>` must be a valid Sidekiq queue name. If the queue name - is `nil`, or an empty string, the worker is routed to the queue generated - by the name of the worker instead. - -The query supports wildcard matching `*`, which matches all workers. As a -result, the wildcard query must stay at the end of the list or the rules after it -are ignored. - -NOTE: -Mixing queue routing rules and queue selectors requires care to -ensure all jobs that are scheduled and picked up by appropriate Sidekiq -workers. - -## Worker matching query - -GitLab provides a query syntax to match a worker based on its -attributes. This query syntax is employed by both -[Queue routing rules](#queue-routing-rules) and -[Queue selector](extra_sidekiq_processes.md#queue-selector). A query includes two -components: - -- Attributes that can be selected. -- Operators used to construct a query. - -### Available attributes - -> [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/261) in GitLab 13.1 (`tags`). - -Queue matching query works upon the worker attributes, described in -[Sidekiq style guide](../../development/sidekiq/index.md). We support querying -based on a subset of worker attributes: - -- `feature_category` - the - [GitLab feature category](https://about.gitlab.com/direction/maturity/#category-maturity) the - queue belongs to. For example, the `merge` queue belongs to the - `source_code_management` category. -- `has_external_dependencies` - whether or not the queue connects to external - services. For example, all importers have this set to `true`. -- `urgency` - how important it is that this queue's jobs run - quickly. Can be `high`, `low`, or `throttled`. For example, the - `authorized_projects` queue is used to refresh user permissions, and - is `high` urgency. -- `worker_name` - the worker name. Use this attribute to select a specific worker. -- `name` - the queue name generated from the worker name. Use this attribute to select a specific queue. Because this is generated from - the worker name, it does not change based on the result of other routing - rules. -- `resource_boundary` - if the queue is bound by `cpu`, `memory`, or - `unknown`. For example, the `ProjectExportWorker` is memory bound as it has - to load data in memory before saving it for export. -- `tags` - short-lived annotations for queues. These are expected to frequently - change from release to release, and may be removed entirely. - -`has_external_dependencies` is a boolean attribute: only the exact -string `true` is considered true, and everything else is considered -false. - -`tags` is a set, which means that `=` checks for intersecting sets, and -`!=` checks for disjoint sets. For example, `tags=a,b` selects queues -that have tags `a`, `b`, or both. `tags!=a,b` selects queues that have -neither of those tags. - -The attributes of each worker are hard-coded in the source code. For -convenience, we generate a -[list of all available attributes in GitLab Community Edition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/all_queues.yml) -and a -[list of all available attributes in GitLab Enterprise Edition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/all_queues.yml). - -### Available operators - -`queue_selector` supports the following operators, listed from highest -to lowest precedence: - -- `|` - the logical `OR` operator. For example, `query_a|query_b` (where `query_a` - and `query_b` are queries made up of the other operators here) includes - queues that match either query. -- `&` - the logical `AND` operator. For example, `query_a&query_b` (where - `query_a` and `query_b` are queries made up of the other operators here) will - only include queues that match both queries. -- `!=` - the `NOT IN` operator. For example, `feature_category!=issue_tracking` - excludes all queues from the `issue_tracking` feature category. -- `=` - the `IN` operator. For example, `resource_boundary=cpu` includes all - queues that are CPU bound. -- `,` - the concatenate set operator. For example, - `feature_category=continuous_integration,pages` includes all queues from - either the `continuous_integration` category or the `pages` category. This - example is also possible using the OR operator, but allows greater brevity, as - well as being lower precedence. - -The operator precedence for this syntax is fixed: it's not possible to make `AND` -have higher precedence than `OR`. - -[In GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26594) and -later, as with the standard queue group syntax above, a single `*` as the -entire queue group selects all queues. - -### Migration - -After the Sidekiq routing rules are changed, administrators must take care -with the migration to avoid losing jobs entirely, especially in a system with -long queues of jobs. The migration can be done by following the migration steps -mentioned in [Sidekiq job migration](sidekiq_job_migration.md) +<!-- This redirect file can be deleted after <2023-02-01>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/sidekiq/index.md b/doc/administration/sidekiq/index.md index f17c248e60e..7b3ecdd0890 100644 --- a/doc/administration/sidekiq/index.md +++ b/doc/administration/sidekiq/index.md @@ -379,11 +379,11 @@ To enable LDAP with the synchronization worker for Sidekiq: ## Configure SAML Groups for SAML Group Sync -If you use [SAML Group Sync](../../user/group/saml_sso/group_sync.md), you must configure [SAML Groups](../../integration/saml.md#saml-groups) on all your Sidekiq nodes. +If you use [SAML Group Sync](../../user/group/saml_sso/group_sync.md), you must configure [SAML Groups](../../integration/saml.md#configure-users-based-on-saml-group-membership) on all your Sidekiq nodes. ## Disable Rugged -Calls into Rugged, Ruby bindings for `libgit2`, [lock the Sidekiq processes's GVL](https://silverhammermba.github.io/emberb/c/#c-in-ruby-threads), +Calls into Rugged, Ruby bindings for `libgit2`, [lock the Sidekiq processes (GVL)](https://silverhammermba.github.io/emberb/c/#c-in-ruby-threads), blocking all jobs on that worker from proceeding. If Rugged calls performed by Sidekiq are slow, this can cause significant delays in background task processing. @@ -398,7 +398,7 @@ sudo gitlab-rake gitlab:features:disable_rugged ## Related topics - [Extra Sidekiq processes](extra_sidekiq_processes.md) -- [Extra Sidekiq routing](extra_sidekiq_routing.md) +- [Processing specific job classes](processing_specific_job_classes.md) - [Sidekiq health checks](sidekiq_health_check.md) - [Using the GitLab-Sidekiq chart](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/) diff --git a/doc/administration/sidekiq/processing_specific_job_classes.md b/doc/administration/sidekiq/processing_specific_job_classes.md new file mode 100644 index 00000000000..080ad7d4eae --- /dev/null +++ b/doc/administration/sidekiq/processing_specific_job_classes.md @@ -0,0 +1,337 @@ +--- +stage: Systems +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Processing specific job classes + +WARNING: +These are advanced settings. While they are used on GitLab.com, most GitLab +instances should add more processes that all listen to all queues. This is the +same approach we take in our [Reference Architectures](../reference_architectures/index.md). + +GitLab has two options for creating Sidekiq processes that only handle specific +job classes: + +1. [Routing rules](#routing-rules) are used on GitLab.com. They direct jobs + inside the application to queue names configured by administrators. This + lowers the load on Redis, which is important on very large-scale deployments. +1. [Queue selectors](#queue-selectors) perform the job selection outside the + application, when starting the Sidekiq process. This was used on GitLab.com + until September 2021, and is retained for compatibility reasons. + +Both of these use the same [worker matching query](#worker-matching-query) +syntax. While they can technically be used together, most deployments should +choose one or the other; there is no particular benefit in combining them. + +Routing rules must be the same across all GitLab nodes as they are part of the +application configuration. Queue selectors can be different across GitLab nodes +because they only change the arguments to the launched Sidekiq process. + +## Routing rules + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/59604) in GitLab 13.12. +> - [Default routing rule value](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/97908) added in GitLab 15.4. + +NOTE: +Mailer jobs cannot be routed by routing rules, and always go to the +`mailers` queue. When using routing rules, ensure that at least one process is +listening to the `mailers` queue. Typically this can be placed alongside the +`default` queue. + +We recommend most GitLab instances using routing rules to manage their Sidekiq +queues. This allows administrators to choose single queue names for groups of +job classes based on their attributes. The syntax is an ordered array of pairs of `[query, queue]`: + +1. The query is a [worker matching query](#worker-matching-query). +1. The queue name must be a valid Sidekiq queue name. If the queue name + is `nil`, or an empty string, the worker is routed to the queue generated + by the name of the worker instead. (See [list of available job classes](#list-of-available-job-classes) + for more information). + The queue name does not have to match any existing queue name in the + list of available job classes. +1. The first query matching a worker is chosen for that worker; later rules are + ignored. + +### Routing rules migration + +After the Sidekiq routing rules are changed, administrators must take care with +the migration to avoid losing jobs entirely, especially in a system with long +queues of jobs. The migration can be done by following the migration steps +mentioned in [Sidekiq job migration](sidekiq_job_migration.md). + +### Detailed example + +This is a comprehensive example intended to show different possibilities. It is +not a recommendation. + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + sidekiq['routing_rules'] = [ + # Route all non-CPU-bound workers that are high urgency to `high-urgency` queue + ['resource_boundary!=cpu&urgency=high', 'high-urgency'], + # Route all database, gitaly and global search workers that are throttled to `throttled` queue + ['feature_category=database,gitaly,global_search&urgency=throttled', 'throttled'], + # Route all workers having contact with outside world to a `network-intenstive` queue + ['has_external_dependencies=true|feature_category=hooks|tags=network', 'network-intensive'], + # Route all import workers to the queues generated by the worker name, for + # example, JiraImportWorker to `jira_import`, SVNWorker to `svn_worker` + ['feature_category=import', 'import'], + # Wildcard matching, route the rest to `default` queue + ['*', 'default'] + ] + ``` + + The `queue_groups` can then be set to match these generated queue names. For + instance: + + ```ruby + sidekiq['queue_selector'] = false + sidekiq['queue_groups'] = [ + # Run two high-urgency processes + 'high-urgency', + 'high-urgency', + # Run one process for throttled, network-intensive, import + 'throttled,network-intensive,import', + # Run one 'catchall' process on the default and mailers queues + 'default,mailers' + ] + ``` + +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +## Queue selectors + +> - [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/45) in GitLab 12.8. +> - [Sidekiq cluster, including queue selector, moved](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/181) to GitLab Free in 12.10. +> - [Renamed from `experimental_queue_selector` to `queue_selector`](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/147) in GitLab 13.6. + +The `queue_selector` option allows queue groups to be selected in a more general +way using a [worker matching query](#worker-matching-query). After +`queue_selector` is set, all `queue_groups` must follow the aforementioned +syntax. + +### Using queue selectors + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + sidekiq['enable'] = true + sidekiq['routing_rules'] = [['*', nil]] + sidekiq['queue_selector'] = true + sidekiq['queue_groups'] = [ + # Run all non-CPU-bound queues that are high urgency + 'resource_boundary!=cpu&urgency=high', + # Run all continuous integration and pages queues that are not high urgency + 'feature_category=continuous_integration,pages&urgency!=high', + # Run all queues + '*' + ] + ``` + +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +### Negate settings + +This allows you to have the Sidekiq process work on every queue **except** the +ones you list. This is generally only used when there are multiple Sidekiq +nodes. In this example, we exclude all import-related jobs from a Sidekiq node. + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + sidekiq['routing_rules'] = [['*', nil]] + sidekiq['negate'] = true + sidekiq['queue_selector'] = true + sidekiq['queue_groups'] = [ + "feature_category=importers" + ] + ``` + +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +### Migrating from queue selectors to routing rules + +We recommend GitLab deployments add more Sidekiq processes listening to all queues, as in the +[Reference Architectures](../reference_architectures/index.md). For very large-scale deployments, we recommend +[routing rules](#routing-rules) instead of [queue selectors](#queue-selectors). We use routing rules on GitLab.com as +it helps to lower the load on Redis. + +To migrate from queue selectors to routing rules: + +1. Open `/etc/gitlab/gitlab.rb`. +1. Set `sidekiq['queue_selector']` to `false`. +1. Take all queue `selector`s in the `sidekiq['queue_groups']`. +1. Give each `selector` a `queue_name` and put them in `[selector, queue_name]` format. +1. Replace `sidekiq['routing_rules']` with an array of `[selector, queue_name]` entries. +1. Add a wildcard match of `['*', 'default']` as the last entry in `sidekiq['routing_rules']`. This "catchall" queue has + to be named as `default`. +1. Replace `sidekiq['queue_groups']` with `queue_name`s. +1. Add at least one `default` queue and at least one `mailers` queue to the `sidekiq['queue_groups']`. +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +1. Run the Rake task to [migrate existing jobs](sidekiq_job_migration.md): + + ```shell + sudo gitlab-rake gitlab:sidekiq:migrate_jobs:retry gitlab:sidekiq:migrate_jobs:schedule gitlab:sidekiq:migrate_jobs:queued + ``` + +NOTE: +It is important to run the Rake task immediately after reconfiguring GitLab. +After reconfiguring GitLab, existing jobs are not processed until the Rake task starts to migrate the jobs. + +The following example better illustrates the migration process above: + +1. Check the following content of `/etc/gitlab/gitlab.rb`: + + ```ruby + sidekiq['routing_rules'] = [] + sidekiq['queue_selector'] = true + sidekiq['queue_groups'] = [ + 'urgency=high', + 'urgency=low', + 'urgency=throttled', + '*' + ] + ``` + +1. Update `/etc/gitlab/gitlab.rb` to use routing rules: + + ```ruby + sidekiq['min_concurrency'] = 20 + sidekiq['max_concurrency'] = 20 + + sidekiq['routing_rules'] = [ + ['urgency=high', 'high_urgency'], + ['urgency=low', 'low_urgency'], + ['urgency=throttled', 'throttled_urgency'], + # Wildcard matching, route the rest to `default` queue + ['*', 'default'] + ] + + sidekiq['queue_selector'] = false + sidekiq['queue_groups'] = [ + 'high_urgency', + 'low_urgency', + 'throttled_urgency', + 'default,mailers' + ] + ``` + +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +1. Run the Rake task to [migrate existing jobs](sidekiq_job_migration.md): + + ```shell + sudo gitlab-rake gitlab:sidekiq:migrate_jobs:retry gitlab:sidekiq:migrate_jobs:schedule gitlab:sidekiq:migrate_jobs:queued + ``` + +WARNING: +As described in [the concurrency section](extra_sidekiq_processes.md#manage-thread-counts-explicitly), we +recommend setting `min_concurrency` and `max_concurrency` to the same value. For example, if the number of queues +in a queue group entry is 1, while `min_concurrency` is set to `0`, and `max_concurrency` is set to `20`, the resulting +concurrency will be set to `2` instead. A concurrency of `2` might be too low in most cases, except for very highly-CPU +bound tasks. + +## Worker matching query + +GitLab provides a query syntax to match a worker based on its attributes. This +query syntax is employed by both [routing rules](#routing-rules) and +[queue selectors](#queue-selectors). A query includes two components: + +- Attributes that can be selected. +- Operators used to construct a query. + +### Available attributes + +> [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/261) in GitLab 13.1 (`tags`). + +Queue matching query works upon the worker attributes, described in +[Sidekiq style guide](../../development/sidekiq/index.md). We support querying +based on a subset of worker attributes: + +- `feature_category` - the + [GitLab feature category](https://about.gitlab.com/direction/maturity/#category-maturity) the + queue belongs to. For example, the `merge` queue belongs to the + `source_code_management` category. +- `has_external_dependencies` - whether or not the queue connects to external + services. For example, all importers have this set to `true`. +- `urgency` - how important it is that this queue's jobs run + quickly. Can be `high`, `low`, or `throttled`. For example, the + `authorized_projects` queue is used to refresh user permissions, and + is `high` urgency. +- `worker_name` - the worker name. Use this attribute to select a specific worker. Find all available names in [the job classes lists](#list-of-available-job-classes) below. +- `name` - the queue name generated from the worker name. Use this attribute to select a specific queue. Because this is generated from + the worker name, it does not change based on the result of other routing + rules. +- `resource_boundary` - if the queue is bound by `cpu`, `memory`, or + `unknown`. For example, the `ProjectExportWorker` is memory bound as it has + to load data in memory before saving it for export. +- `tags` - short-lived annotations for queues. These are expected to frequently + change from release to release, and may be removed entirely. + +`has_external_dependencies` is a boolean attribute: only the exact +string `true` is considered true, and everything else is considered +false. + +`tags` is a set, which means that `=` checks for intersecting sets, and +`!=` checks for disjoint sets. For example, `tags=a,b` selects queues +that have tags `a`, `b`, or both. `tags!=a,b` selects queues that have +neither of those tags. + +### Available operators + +Routing rules and queue selectors support the following operators, listed from +highest to lowest precedence: + +- `|` - the logical `OR` operator. For example, `query_a|query_b` (where `query_a` + and `query_b` are queries made up of the other operators here) includes + queues that match either query. +- `&` - the logical `AND` operator. For example, `query_a&query_b` (where + `query_a` and `query_b` are queries made up of the other operators here) will + only include queues that match both queries. +- `!=` - the `NOT IN` operator. For example, `feature_category!=issue_tracking` + excludes all queues from the `issue_tracking` feature category. +- `=` - the `IN` operator. For example, `resource_boundary=cpu` includes all + queues that are CPU bound. +- `,` - the concatenate set operator. For example, + `feature_category=continuous_integration,pages` includes all queues from + either the `continuous_integration` category or the `pages` category. This + example is also possible using the OR operator, but allows greater brevity, as + well as being lower precedence. + +The operator precedence for this syntax is fixed: it's not possible to make `AND` +have higher precedence than `OR`. + +As with the standard queue group syntax above, a single `*` as the +entire queue group selects all queues. + +### List of available job classes + +For a list of the existing Sidekiq job classes and queues, check the following +files: + +- [Queues for all GitLab editions](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/all_queues.yml) +- [Queues for GitLab Enterprise Editions only](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/all_queues.yml) diff --git a/doc/administration/sidekiq/sidekiq_job_migration.md b/doc/administration/sidekiq/sidekiq_job_migration.md index f61021ad4e7..b93d86d4c86 100644 --- a/doc/administration/sidekiq/sidekiq_job_migration.md +++ b/doc/administration/sidekiq/sidekiq_job_migration.md @@ -4,7 +4,7 @@ group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Sidekiq job migration **(FREE SELF)** +# Sidekiq job migration Rake tasks **(FREE SELF)** WARNING: This operation should be very uncommon. We do not recommend it for the vast majority of GitLab instances. @@ -17,24 +17,27 @@ If the Sidekiq routing rules are changed, administrators need to take care with 1. Listen to both the old and new queues. 1. Update the routing rules. -1. Wait until there are no publishers dispatching jobs to the old queues. -1. Run the [Rake tasks for future jobs](#future-jobs). -1. Wait for the old queues to be empty. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Run the [Rake tasks for migrating queued and future jobs](#migrate-queued-and-future-jobs). 1. Stop listening to the old queues. -## Future jobs +## Migrate queued and future jobs Step 4 involves rewriting some Sidekiq job data for jobs that are already stored in Redis, but due to run in future. There are two sets of jobs to run in future: scheduled jobs and jobs to be retried. We provide a separate Rake task to migrate each set: - `gitlab:sidekiq:migrate_jobs:retry` for jobs to be retried. - `gitlab:sidekiq:migrate_jobs:scheduled` for scheduled jobs. -Most of the time, running both at the same time is the correct choice. There are two separate tasks to allow for more fine-grained control where needed. To run both at once: +Queued jobs that are yet to be run can also be migrated with a Rake task: + +- `gitlab:sidekiq:migrate_jobs:queued` for queued jobs to be performed asynchronously. + +Most of the time, running all three at the same time is the correct choice. There are three separate tasks to allow for more fine-grained control where needed. To run all three at once: ```shell # omnibus-gitlab -sudo gitlab-rake gitlab:sidekiq:migrate_jobs:retry gitlab:sidekiq:migrate_jobs:schedule +sudo gitlab-rake gitlab:sidekiq:migrate_jobs:retry gitlab:sidekiq:migrate_jobs:schedule gitlab:sidekiq:migrate_jobs:queued # source installations -bundle exec rake gitlab:sidekiq:migrate_jobs:retry gitlab:sidekiq:migrate_jobs:schedule RAILS_ENV=production +bundle exec rake gitlab:sidekiq:migrate_jobs:retry gitlab:sidekiq:migrate_jobs:schedule gitlab:sidekiq:migrate_jobs:queued RAILS_ENV=production ``` diff --git a/doc/administration/sidekiq/sidekiq_troubleshooting.md b/doc/administration/sidekiq/sidekiq_troubleshooting.md index d2afe171e9c..b261e385949 100644 --- a/doc/administration/sidekiq/sidekiq_troubleshooting.md +++ b/doc/administration/sidekiq/sidekiq_troubleshooting.md @@ -56,6 +56,120 @@ gitlab_rails['env'] = {"SIDEKIQ_LOG_ARGUMENTS" => "0"} In GitLab 13.5 and earlier, set `SIDEKIQ_LOG_ARGUMENTS` to `1` to start logging arguments passed to Sidekiq. +## Investigating Sidekiq queue backlogs or slow performance + +Symptoms of slow Sidekiq performance include problems with merge request status updates, +and delays before CI pipelines start running. + +Potential causes include: + +- The GitLab instance may need more Sidekiq workers. By default, a single-node Omnibus GitLab + runs one worker, restricting the execution of Sidekiq jobs to a maximum of one CPU core. + [Read more about running multiple Sidekiq workers](extra_sidekiq_processes.md). + +- The instance is configured with more Sidekiq workers, but most of the extra workers are + not configured to run any job that is queued. This can result in a backlog of jobs + when the instance is busy, if the workload has changed in the months or years since + the workers were configured, or as a result of GitLab product changes. + +Gather data on the state of the Sidekiq workers with the following Ruby script. + +1. Create the script: + + ```ruby + cat > /var/opt/gitlab/sidekiqcheck.rb <<EOF + require 'sidekiq/monitor' + Sidekiq::Monitor::Status.new.display('overview') + Sidekiq::Monitor::Status.new.display('processes'); nil + Sidekiq::Monitor::Status.new.display('queues'); nil + puts "----------- workers ----------- " + workers = Sidekiq::Workers.new + workers.each do |_process_id, _thread_id, work| + pp work + end + puts "----------- Queued Jobs ----------- " + Sidekiq::Queue.all.each do |queue| + queue.each do |job| + pp job + end + end ;nil + puts "----------- done! ----------- " + EOF + ``` + +1. Execute and capture the output: + + ```shell + sudo gitlab-rails runner /var/opt/gitlab/sidekiqcheck.rb > /tmp/sidekiqcheck_$(date '+%Y%m%d-%H:%M').out + ``` + + If the performance issue is intermittent: + + - Run this in a cron job every five minutes. Write the files to a location with enough space: allow for 500KB per file. + - Refer back to the data to see what went wrong. + +1. Analyze the output. The following commands assume that you have a directory of output files. + + 1. `grep 'Busy: ' *` shows how many jobs were being run. `grep 'Enqueued: ' *` + shows the backlog of work at that time. + + 1. Look at the number of busy threads across the workers in samples where Sidekiq is under load: + + ```shell + ls | while read f ; do if grep -q 'Enqueued: 0' $f; then : + else echo $f; egrep 'Busy:|Enqueued:|---- Processes' $f + grep 'Threads:' $f ; fi + done | more + ``` + + Example output: + + ```plaintext + sidekiqcheck_20221024-14:00.out + Busy: 47 + Enqueued: 363 + ---- Processes (13) ---- + Threads: 30 (0 busy) + Threads: 30 (0 busy) + Threads: 30 (0 busy) + Threads: 30 (0 busy) + Threads: 23 (0 busy) + Threads: 30 (0 busy) + Threads: 30 (0 busy) + Threads: 30 (0 busy) + Threads: 30 (0 busy) + Threads: 30 (0 busy) + Threads: 30 (0 busy) + Threads: 30 (24 busy) + Threads: 30 (23 busy) + ``` + + - In this output file, 47 threads were busy, and there was a backlog of 363 jobs. + - Of the 13 worker processes, only two were busy. + - This indicates that the other workers are configured too specifically. + - Look at the full output to work out which workers were busy. + Correlate with your `sidekiq_queues` configuration in `gitlab.rb`. + - An overloaded single-worker environment might look like this: + + ```plaintext + sidekiqcheck_20221024-14:00.out + Busy: 25 + Enqueued: 363 + ---- Processes (1) ---- + Threads: 25 (25 busy) + ``` + + 1. Look at the `---- Queues (xxx) ----` section of the output file to + determine what jobs were queued up at the time. + + 1. The files also include low level details about the state of Sidekiq at the time. + This could be useful for identifying where spikes in workload are coming from. + + - The `----------- workers -----------` section details the jobs that make up the + `Busy` count in the summary. + - The `----------- Queued Jobs -----------` section provides details on + jobs that are `Enqueued`. + ## Thread dump Send the Sidekiq process ID the `TTIN` signal to output thread @@ -379,3 +493,60 @@ has number of drawbacks, as mentioned in [Why Ruby's Timeout is dangerous (and T > - in any of your code, regardless of whether it could have possibly raised an exception before > > Nobody writes code to defend against an exception being raised on literally any line. That's not even possible. So Thread.raise is basically like a sneak attack on your code that could result in almost anything. It would probably be okay if it were pure-functional code that did not modify any state. But this is Ruby, so that's unlikely :) + +## Omnibus GitLab 14.0 and later: remove the `sidekiq-cluster` service + +Omnibus GitLab instances that were configured to run `sidekiq-cluster` prior to GitLab 14.0 +might still have this service running along side `sidekiq` in later releases. + +The code to manage `sidekiq-cluster` was removed in GitLab 14.0. +The configuration files remain on disk so the `sidekiq-cluster` process continues +to be started by the GitLab systemd service . + +The extra service can be identified as running by: + +- `gitlab-ctl status` showing both services: + + ```plaintext + run: sidekiq: (pid 1386) 445s; run: log: (pid 1385) 445s + run: sidekiq-cluster: (pid 1388) 445s; run: log: (pid 1381) 445s + ``` + +- `ps -ef | grep 'runsv sidekiq'` showing two processes: + + ```plaintext + root 31047 31045 0 13:54 ? 00:00:00 runsv sidekiq-cluster + root 31054 31045 0 13:54 ? 00:00:00 runsv sidekiq + ``` + +To remove the `sidekiq-cluster` service from servers running GitLab 14.0 and later: + +1. Stop GitLab and the systemd service: + + ```shell + sudo gitlab-ctl stop + sudo systemctl stop gitlab-runsvdir.service + ``` + +1. Remove the `runsv` service definition: + + ```shell + sudo rm -rf /opt/gitlab/sv/sidekiq-cluster + ``` + +1. Restart GitLab: + + ```shell + sudo systemctl start gitlab-runsvdir.service + ``` + +1. Check that all services are up, and the `sidekiq-cluster` service is not listed: + + ```shell + sudo gitlab-ctl status + ``` + +This change might reduce the amount of work Sidekiq can do. Symptoms like delays creating pipelines +indicate that additional Sidekiq processes would be beneficial. +Consider [adding additional Sidekiq processes](extra_sidekiq_processes.md) +to compensate for removing the `sidekiq-cluster` service. diff --git a/doc/administration/sidekiq_health_check.md b/doc/administration/sidekiq_health_check.md deleted file mode 100644 index 3294eb663f2..00000000000 --- a/doc/administration/sidekiq_health_check.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'sidekiq/sidekiq_health_check.md' -remove_date: '2022-11-11' ---- - -This document was moved to [another location](sidekiq/sidekiq_health_check.md). - -<!-- This redirect file can be deleted after <2022-11-11>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/snippets/index.md b/doc/administration/snippets/index.md index 7bf828afedd..4bd03aeb8c8 100644 --- a/doc/administration/snippets/index.md +++ b/doc/administration/snippets/index.md @@ -72,3 +72,7 @@ You can also use the API to [retrieve the current value](../../api/settings.md#g ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/application/settings" ``` + +## Related topics + +- [User snippets](../../user/snippets.md) diff --git a/doc/administration/terraform_state.md b/doc/administration/terraform_state.md index 02a95e28747..d3b941bd129 100644 --- a/doc/administration/terraform_state.md +++ b/doc/administration/terraform_state.md @@ -160,6 +160,10 @@ sudo find /var/opt/gitlab/gitlab-rails/shared/terraform_state -type f | grep -v ### S3-compatible connection settings +In GitLab 13.2 and later, you should use the +[consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). +This section describes the earlier configuration format. + See [the available connection settings for different providers](object_storage.md#connection-settings). **In Omnibus installations:** diff --git a/doc/administration/troubleshooting/elasticsearch.md b/doc/administration/troubleshooting/elasticsearch.md deleted file mode 100644 index 7390f4bc816..00000000000 --- a/doc/administration/troubleshooting/elasticsearch.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../../integration/advanced_search/elasticsearch_troubleshooting.md' -remove_date: '2022-11-02' ---- - -This document was moved to [another location](../../integration/advanced_search/elasticsearch_troubleshooting.md). - -<!-- This redirect file can be deleted after <2022-11-02>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index 6993d48b450..7c5feb24e15 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -1,11 +1,95 @@ --- -redirect_to: 'index.md' -remove_date: '2023-02-01' +stage: Systems +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -This document was moved to [another location](index.md). +# GitLab Rails Console Cheat Sheet **(FREE SELF)** -<!-- This redirect file can be deleted after 2023-02-01. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> +This was the GitLab Support Team's collection of information regarding the GitLab Rails +console, for use while troubleshooting. It is listed here for posterity, +as most content has been moved to feature-specific troubleshooting pages and sections, +see epic [&8147](https://gitlab.com/groups/gitlab-org/-/epics/8147#tree). +Please update your bookmarks accordingly. + +If you are currently having an issue with GitLab, +it is highly recommended that you first check +our guide on [the Rails console](../operations/rails_console.md), +and your [support options](https://about.gitlab.com/support/), +before attempting the information pointed to from here. + +WARNING: +Some of these scripts could be damaging if not run correctly, +or under the right conditions. We highly recommend running them under the +guidance of a Support Engineer, or running them in a test environment with a +backup of the instance ready to be restored, just in case. + +WARNING: +As GitLab changes, changes to the code are inevitable, +and so some scripts may not work as they once used to. These are not kept +up-to-date as these scripts/commands were added as they were found/needed. As +mentioned above, we recommend running these scripts under the supervision of a +Support Engineer, who can also verify that they continue to work as they +should and, if needed, update the script for the latest version of GitLab. + +## Mirrors + +### Find mirrors with "bad decrypt" errors + +This content has been converted to a Rake task, see [verify database values can be decrypted using the current secrets](../raketasks/check.md#verify-database-values-can-be-decrypted-using-the-current-secrets). + +### Transfer mirror users and tokens to a single service account + +This content has been moved to [Troubleshooting Repository mirroring](../../user/project/repository/mirror/index.md#transfer-mirror-users-and-tokens-to-a-single-service-account-in-rails-console). + +## Merge requests + +## CI + +This content has been moved to [Troubleshooting CI/CD](../../ci/troubleshooting.md). + +## License + +This content has been moved to [Activate GitLab EE with a license file or key](../../user/admin_area/license_file.md). + +## Registry + +### Registry Disk Space Usage by Project + +Find this content in the [Container Registry troubleshooting documentation](../packages/container_registry.md#registry-disk-space-usage-by-project). + +### Run the Cleanup policy now + +Find this content in the [Container Registry troubleshooting documentation](../packages/container_registry.md#run-the-cleanup-policy-now). + +## Sidekiq + +This content has been moved to [Troubleshooting Sidekiq](../sidekiq/sidekiq_troubleshooting.md). + +## Geo + +### Reverify all uploads (or any SSF data type which is verified) + +Moved to [Geo replication troubleshooting](../geo/replication/troubleshooting.md#reverify-all-uploads-or-any-ssf-data-type-which-is-verified). + +### Artifacts + +Moved to [Geo replication troubleshooting](../geo/replication/troubleshooting.md#find-failed-artifacts). + +### Repository verification failures + +Moved to [Geo replication troubleshooting](../geo/replication/troubleshooting.md#find-repository-verification-failures). + +### Resync repositories + +Moved to [Geo replication troubleshooting - Resync repository types except for project or project wiki repositories](../geo/replication/troubleshooting.md#repository-types-except-for-project-or-project-wiki-repositories). + +Moved to [Geo replication troubleshooting - Resync project and project wiki repositories](../geo/replication/troubleshooting.md#resync-project-and-project-wiki-repositories). + +### Blob types + +Moved to [Geo replication troubleshooting](../geo/replication/troubleshooting.md#blob-types). + +## Generate Service Ping + +This content has been moved to [Service Ping Troubleshooting](../../development/service_ping/troubleshooting.md). diff --git a/doc/administration/troubleshooting/linux_cheat_sheet.md b/doc/administration/troubleshooting/linux_cheat_sheet.md index 90cd1e24c79..ae0ef44f0b1 100644 --- a/doc/administration/troubleshooting/linux_cheat_sheet.md +++ b/doc/administration/troubleshooting/linux_cheat_sheet.md @@ -294,8 +294,8 @@ small differences should not be considered significant. |Setup | access times | |:--------------|:--------------| -| EFS | 10 - 30ms | -| Local Storage | 0.01 - 1ms | +| EFS | 10 - 30 ms | +| Local Storage | 0.01 - 1 ms | ## Networking diff --git a/doc/administration/troubleshooting/postgresql.md b/doc/administration/troubleshooting/postgresql.md index 829fed38060..d5288bfead8 100644 --- a/doc/administration/troubleshooting/postgresql.md +++ b/doc/administration/troubleshooting/postgresql.md @@ -44,10 +44,6 @@ This section is for links to information elsewhere in the GitLab documentation. - Consuming PostgreSQL from [within CI runners](../../ci/services/postgres.md). -- [Using Slony to update PostgreSQL](../../update/upgrading_postgresql_using_slony.md). - - Uses replication to handle PostgreSQL upgrades if the schemas are the same. - - Reduces downtime to a short window for switching to the newer version. - - Managing Omnibus PostgreSQL versions [from the development docs](https://docs.gitlab.com/omnibus/development/managing-postgresql-versions.html). - [PostgreSQL scaling](../postgresql/replication_and_failover.md) @@ -104,14 +100,14 @@ or `statement_timeout`, but to leave the third setting at 60 seconds. Setting `idle_in_transaction` protects the database from sessions potentially hanging for days. There's more discussion in [the issue relating to introducing this timeout on GitLab.com](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/1053). -PostgresSQL defaults: +PostgreSQL defaults: - `statement_timeout = 0` (never) - `idle_in_transaction_session_timeout = 0` (never) Comments in issue [#30528](https://gitlab.com/gitlab-org/gitlab/-/issues/30528) indicate that these should both be set to at least a number of minutes for all -Omnibus GitLab installations (so they don't hang indefinitely). However, 15s +Omnibus GitLab installations (so they don't hang indefinitely). However, 15 s for `statement_timeout` is very short, and is only effective if the underlying infrastructure is very performant. @@ -188,7 +184,7 @@ To temporarily change the statement timeout: ### Database is not accepting commands to avoid wraparound data loss -This error likely means that AUTOVACUUM is failing to complete its run: +This error likely means that `autovacuum` is failing to complete its run: ```plaintext ERROR: database is not accepting commands to avoid wraparound data loss in database "gitlabhq_production" @@ -211,7 +207,7 @@ To resolve the error, run `VACUUM` manually: The [database requirements](../../install/requirements.md#database) for GitLab include: -- Support for MySQL was removed in GitLab 12.1; [migrate to PostgreSQL](../../update/mysql_to_postgresql.md). +- Support for MySQL was removed in [GitLab 12.1](../../update/index.md#1210). - Review and install the [required extension list](../../install/postgresql_extensions.md). ### Serialization errors in the `production/sidekiq` log diff --git a/doc/administration/troubleshooting/sidekiq.md b/doc/administration/troubleshooting/sidekiq.md deleted file mode 100644 index e49e0ed4f1c..00000000000 --- a/doc/administration/troubleshooting/sidekiq.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../sidekiq/sidekiq_troubleshooting.md' -remove_date: '2022-11-11' ---- - -This document was moved to [another location](../sidekiq/sidekiq_troubleshooting.md). - -<!-- This redirect file can be deleted after <2022-11-11>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/troubleshooting/tracing_correlation_id.md b/doc/administration/troubleshooting/tracing_correlation_id.md deleted file mode 100644 index 917e27bab70..00000000000 --- a/doc/administration/troubleshooting/tracing_correlation_id.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../logs/tracing_correlation_id.md' -remove_date: '2022-11-12' ---- - -This document was moved to [another location](../logs/tracing_correlation_id.md). - -<!-- This redirect file can be deleted after 2022-11-12. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
\ No newline at end of file diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md index f0def7320cc..ff0b8ecf178 100644 --- a/doc/administration/uploads.md +++ b/doc/administration/uploads.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Uploads administration **(FREE SELF)** -Uploads represent all user data that may be sent to GitLab as a single file. As an example, avatars and notes' attachments are uploads. Uploads are integral to GitLab functionality, and therefore cannot be disabled. +Uploads represent all user data that may be sent to GitLab as a single file. For example, avatars and note attachments are uploads. Uploads are integral to GitLab functionality and therefore cannot be disabled. ## Using local storage @@ -14,15 +14,15 @@ This is the default configuration. To change the location where the uploads are stored locally, use the steps in this section based on your installation method: NOTE: -For historical reasons, instance level uploads (for example the [favicon](../user/admin_area/appearance.md#favicon)) are stored into a base directory, -which by default is `uploads/-/system`. It is strongly discouraged to change the base -directory on an existing GitLab installation. +For historical reasons, uploads for the whole instance (for example the [favicon](../user/admin_area/appearance.md#favicon)) are stored in a base directory, +which by default is `uploads/-/system`. Changing the base +directory on an existing GitLab installation is strongly discouraged. **In Omnibus GitLab installations:** _The uploads are stored by default in `/var/opt/gitlab/gitlab-rails/uploads`._ -1. To change the storage path for example to `/mnt/storage/uploads`, edit +1. To change the storage path, for example to `/mnt/storage/uploads`, edit `/etc/gitlab/gitlab.rb` and add the following line: ```ruby @@ -38,7 +38,7 @@ _The uploads are stored by default in `/var/opt/gitlab/gitlab-rails/uploads`._ _The uploads are stored by default in `/home/git/gitlab/public/uploads`._ -1. To change the storage path for example to `/mnt/storage/uploads`, edit +1. To change the storage path, for example to `/mnt/storage/uploads`, edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following lines: ```yaml @@ -57,10 +57,12 @@ This configuration relies on valid AWS credentials to be configured already. [Read more about using object storage with GitLab](object_storage.md). -We recommend using the [consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). The following instructions apply to the original configuration format. - ### Object Storage Settings +In GitLab 13.2 and later, you should use the +[consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). +This section describes the earlier configuration format. + For source installations the following settings are nested under `uploads:` and then `object_store:`. On Omnibus GitLab installs they are prefixed by `uploads_object_store_`. | Setting | Description | Default | @@ -104,7 +106,7 @@ _The uploads are stored by default in ``` 1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. Migrate any existing local uploads to the object storage using [`gitlab:uploads:migrate:all` Rake task](raketasks/uploads/migrate.md). +1. Migrate any existing local uploads to the object storage with the [`gitlab:uploads:migrate:all` Rake task](raketasks/uploads/migrate.md). **In installations from source:** @@ -127,4 +129,4 @@ _The uploads are stored by default in ``` 1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. -1. Migrate any existing local uploads to the object storage using [`gitlab:uploads:migrate:all` Rake task](raketasks/uploads/migrate.md). +1. Migrate any existing local uploads to the object storage with the [`gitlab:uploads:migrate:all` Rake task](raketasks/uploads/migrate.md). diff --git a/doc/administration/user_settings.md b/doc/administration/user_settings.md index c96a6311208..b02e1c7244b 100644 --- a/doc/administration/user_settings.md +++ b/doc/administration/user_settings.md @@ -18,7 +18,7 @@ ability to create top-level groups (does not affect existing users' setting), Gi - The [application setting API](../api/settings.md#change-application-settings). - In GitLab 15.4 and earlier, in a configuration file by following the steps in this section. -To disable new users' ability to create top-level groups using the configuation file: +To disable new users' ability to create top-level groups using the configuration file: **Omnibus GitLab installations** diff --git a/doc/api/alert_management_alerts.md b/doc/api/alert_management_alerts.md index bf3d6287341..702d453e140 100644 --- a/doc/api/alert_management_alerts.md +++ b/doc/api/alert_management_alerts.md @@ -33,7 +33,7 @@ Example response: "created_at": "2020-11-12T20:07:58.156Z", "filename": "sample_2054", "file_path": "/uploads/-/system/alert_metric_image/file/17/sample_2054.png", - "url": "example.com/metric", + "url": "https://example.com/metric", "url_text": "An example metric" } ``` @@ -62,7 +62,7 @@ Example response: "created_at": "2020-11-12T20:07:58.156Z", "filename": "sample_2054", "file_path": "/uploads/-/system/alert_metric_image/file/17/sample_2054.png", - "url": "example.com/metric", + "url": "https://example.com/metric", "url_text": "An example metric" }, { @@ -70,7 +70,7 @@ Example response: "created_at": "2020-11-12T20:14:26.441Z", "filename": "sample_2054", "file_path": "/uploads/-/system/alert_metric_image/file/18/sample_2054.png", - "url": "example.com/metric", + "url": "https://example.com/metric", "url_text": "An example metric" } ] @@ -102,8 +102,8 @@ Example response: "created_at": "2020-11-13T00:06:18.084Z", "filename": "file.png", "file_path": "/uploads/-/system/alert_metric_image/file/23/file.png", - "url": "http://example.com", - "url_text": "Example website" + "url": "https://example.com/metric", + "url_text": "An example metric" } ``` diff --git a/doc/api/appearance.md b/doc/api/appearance.md index f8926f8a91d..622239e7283 100644 --- a/doc/api/appearance.md +++ b/doc/api/appearance.md @@ -29,6 +29,7 @@ Example response: ```json { "title": "GitLab Test Instance", + "short_title": "GitLab", "description": "gitlab-test.example.com", "logo": "/uploads/-/system/appearance/logo/1/logo.png", "header_logo": "/uploads/-/system/appearance/header_logo/1/header.png", @@ -54,6 +55,7 @@ PUT /application/appearance | Attribute | Type | Required | Description | | --------------------------------- | ------- | -------- | ----------- | | `title` | string | no | Instance title on the sign in / sign up page +| `short_title` | string | no | Short title for progressive web app | `description` | string | no | Markdown text shown on the sign in / sign up page | `logo` | mixed | no | Instance image used on the sign in / sign up page. See [Change logo](#change-logo) | `header_logo` | mixed | no | Instance image used for the main navigation bar @@ -75,6 +77,7 @@ Example response: ```json { "title": "GitLab Test Instance", + "short_title": "GitLab", "description": "gitlab-test.example.com", "logo": "/uploads/-/system/appearance/logo/1/logo.png", "header_logo": "/uploads/-/system/appearance/header_logo/1/header.png", diff --git a/doc/api/applications.md b/doc/api/applications.md index f7b646d2351..53ea2f51d1a 100644 --- a/doc/api/applications.md +++ b/doc/api/applications.md @@ -103,7 +103,7 @@ Parameters: | Attribute | Type | Required | Description | |:----------|:--------|:---------|:----------------------------------------------------| -| `id` | integer | yes | The ID of the application (not the application_id). | +| `id` | integer | yes | The ID of the application (not the `application_id`). | Example request: diff --git a/doc/api/bulk_imports.md b/doc/api/bulk_imports.md index e18a77df6df..a438bc13818 100644 --- a/doc/api/bulk_imports.md +++ b/doc/api/bulk_imports.md @@ -29,10 +29,11 @@ POST /bulk_imports | `entities[source_full_path]` | String | yes | Source full path of the entity to import. | | `entities[destination_name]` | String | yes | Deprecated: Use :destination_slug instead. Destination slug for the entity. | | `entities[destination_slug]` | String | yes | Destination slug for the entity. | -| `entities[destination_namespace]` | String | no | Destination namespace for the entity. | +| `entities[destination_namespace]` | String | yes | Destination namespace for the entity. | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/bulk_imports" \ + --header "Content-Type: application/json" \ --data '{ "configuration": { "url": "http://gitlab.example/", diff --git a/doc/api/commits.md b/doc/api/commits.md index 3fe77dd5f43..83fa54231ee 100644 --- a/doc/api/commits.md +++ b/doc/api/commits.md @@ -100,15 +100,15 @@ POST /projects/:id/repository/commits | `stats` | boolean | no | Include commit stats. Default is true | | `force` | boolean | no | When `true` overwrites the target branch with a new commit based on the `start_branch` or `start_sha` | -| `actions[]` Attribute | Type | Required | Description | -| --------------------- | ---- | -------- | ----------- | -| `action` | string | yes | The action to perform, `create`, `delete`, `move`, `update`, `chmod`| -| `file_path` | string | yes | Full path to the file. Ex. `lib/class.rb` | -| `previous_path` | string | no | Original full path to the file being moved. Ex. `lib/class1.rb`. Only considered for `move` action. | -| `content` | string | no | File content, required for all except `delete`, `chmod`, and `move`. Move actions that do not specify `content` preserve the existing file content, and any other value of `content` overwrites the file content. | -| `encoding` | string | no | `text` or `base64`. `text` is default. | -| `last_commit_id` | string | no | Last known file commit ID. Only considered in update, move, and delete actions. | -| `execute_filemode` | boolean | no | When `true/false` enables/disables the execute flag on the file. Only considered for `chmod` action. | +| `actions[]` Attribute | Type | Required | Description | +|-----------------------|---------|----------|-------------| +| `action` | string | yes | The action to perform: `create`, `delete`, `move`, `update`, or `chmod`. | +| `file_path` | string | yes | Full path to the file. For example: `lib/class.rb`. | +| `previous_path` | string | no | Original full path to the file being moved. For example `lib/class1.rb`. Only considered for `move` action. | +| `content` | string | no | File content, required for all except `delete`, `chmod`, and `move`. Move actions that do not specify `content` preserve the existing file content, and any other value of `content` overwrites the file content. | +| `encoding` | string | no | `text` or `base64`. `text` is default. | +| `last_commit_id` | string | no | Last known file commit ID. Only considered in update, move, and delete actions. | +| `execute_filemode` | boolean | no | When `true/false` enables/disables the execute flag on the file. Only considered for `chmod` action. | ```shell PAYLOAD=$(cat << 'JSON' @@ -695,9 +695,10 @@ Example response: ] ``` -### Post the build status to a commit +### Set the pipeline status of a commit -Adds or updates a build status of a commit. +Add or update the pipeline status of a commit. If the commit is associated with a merge request, +the API call must target the commit in the merge request's source branch. ```plaintext POST /projects/:id/statuses/:sha diff --git a/doc/api/container_registry.md b/doc/api/container_registry.md index 5b11c324802..a2e4d9f37f5 100644 --- a/doc/api/container_registry.md +++ b/doc/api/container_registry.md @@ -419,18 +419,51 @@ To query those, follow the Registry's built-in mechanism to obtain and use an NOTE: These are different from project or personal access tokens in the GitLab application. +### Obtain token from GitLab + +```plaintext +GET ${CI_SERVER_URL}/jwt/auth?service=container_registry&scope=* +``` + +You must specify the correct [scopes and actions](https://docs.docker.com/registry/spec/auth/scope/) to retrieve a valid token: + +```shell +$ SCOPE="repository:${CI_REGISTRY_IMAGE}:delete" #or push,pull + +$ curl --request GET --user "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" \ + "https://gitlab.example.com/jwt/auth?service=container_registry&scope=${SCOPE}" +{"token":" ... "} +``` + +### Delete image tags by reference + +```plaintext +DELETE http(s)://${CI_REGISTRY}/v2/${CI_REGISTRY_IMAGE}/tags/reference/${CI_COMMIT_SHORT_SHA} +``` + +You can use the token retrieved with the predefined `CI_REGISTRY_USER` and `CI_REGISTRY_PASSWORD` variables to delete container image tags by reference on your GitLab instance. +The `tag_delete` [Container-Registry-Feature](https://gitlab.com/gitlab-org/container-registry/-/tree/v3.61.0-gitlab/docs-gitlab#api) must be enabled. + +```shell +$ curl --request DELETE --header "Authorization: Bearer <token_from_above>" \ + --header "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + "https://gitlab.example.com:5050/v2/${CI_REGISTRY_IMAGE}/tags/reference/${CI_COMMIT_SHORT_SHA}" +``` + ### Listing all container repositories ```plaintext -GET /v2/_catalog +GET http(s)://${CI_REGISTRY}/v2/_catalog ``` To list all container repositories on your GitLab instance, administrator credentials are required: ```shell -$ curl --request GET --user "<admin-username>:<admin-password>" "https://gitlab.example.com/jwt/auth?service=container_registry&scope=registry:catalog:*" +$ SCOPE="registry:catalog:*" + +$ curl --request GET --user "<admin-username>:<admin-password>" \ + "https://gitlab.example.com/jwt/auth?service=container_registry&scope=${SCOPE}" {"token":" ... "} $ curl --header "Authorization: Bearer <token_from_above>" https://gitlab.example.com:5050/v2/_catalog -{"repositories":["user/project1", "group/subgroup/project2", ... ]} ``` diff --git a/doc/api/deployments.md b/doc/api/deployments.md index daf2b635855..688806e9b22 100644 --- a/doc/api/deployments.md +++ b/doc/api/deployments.md @@ -15,20 +15,25 @@ Get a list of deployments in a project. GET /projects/:id/deployments ``` -| Attribute | Type | Required | Description | -|------------------|----------------|----------|-----------------------------------------------------------------------------------------------------------------| -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | -| `order_by` | string | no | Return deployments ordered by either one of `id`, `iid`, `created_at`, `updated_at` or `ref` fields. Default is `id`. | -| `sort` | string | no | Return deployments sorted in `asc` or `desc` order. Default is `asc`. | -| `updated_after` | datetime | no | Return deployments updated after the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | -| `updated_before` | datetime | no | Return deployments updated before the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | -| `environment` | string | no | The [name of the environment](../ci/environments/index.md) to filter deployments by. | -| `status` | string | no | The status to filter deployments by. One of `created`, `running`, `success`, `failed`, `canceled`, or `blocked`. +| Attribute | Type | Required | Description | +|-------------------|----------------|----------|-----------------------------------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | +| `order_by` | string | no | Return deployments ordered by either one of `id`, `iid`, `created_at`, `updated_at`, `finished_at` or `ref` fields. Default is `id`. | +| `sort` | string | no | Return deployments sorted in `asc` or `desc` order. Default is `asc`. | +| `updated_after` | datetime | no | Return deployments updated after the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `updated_before` | datetime | no | Return deployments updated before the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `finished_after` | datetime | no | Return deployments finished after the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `finished_before` | datetime | no | Return deployments finished before the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `environment` | string | no | The [name of the environment](../ci/environments/index.md) to filter deployments by. | +| `status` | string | no | The status to filter deployments by. One of `created`, `running`, `success`, `failed`, `canceled`, or `blocked`. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/deployments" ``` +NOTE: +When using `finished_before` or `finished_after`, you should specify the `order_by` to be `finished_at` and `status` should be `success`. + Example response: ```json diff --git a/doc/api/discussions.md b/doc/api/discussions.md index e82f6927e0b..67fca84e449 100644 --- a/doc/api/discussions.md +++ b/doc/api/discussions.md @@ -911,7 +911,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>"\ To create a new thread: -1. [Get the latest merge request version](merge_requests.md#get-mr-diff-versions): +1. [Get the latest merge request version](merge_requests.md#get-merge-request-diff-versions): ```shell curl --header "PRIVATE-TOKEN: <your_access_token>"\ diff --git a/doc/api/dora/metrics.md b/doc/api/dora/metrics.md index d902f5eb91f..4374d1dde95 100644 --- a/doc/api/dora/metrics.md +++ b/doc/api/dora/metrics.md @@ -103,7 +103,7 @@ parameter: |:---------------------------|:-----------------------------------| | `change_failure_rate` | The number of incidents divided by the number of deployments during the time period. Available only for production environment. | | `deployment_frequency` | The number of successful deployments during the time period. | -| `lead_time_for_changes` | The median number of seconds between the merge of the merge request (MR) and the deployment of the MR's commits for all MRs deployed during the time period. | +| `lead_time_for_changes` | The median number of seconds between the merge of the merge request (MR) and the deployment of the MR commits for all MRs deployed during the time period. | | `time_to_restore_service` | The median number of seconds an incident was open during the time period. Available only for production environment. | NOTE: diff --git a/doc/api/epic_issues.md b/doc/api/epic_issues.md index 4bbd0b21136..28c74a0a418 100644 --- a/doc/api/epic_issues.md +++ b/doc/api/epic_issues.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Epic Issues API **(PREMIUM)** -Every API call to epic_issues must be authenticated. +Every API call to the epic issues API endpoint must be authenticated. If a user is not a member of a group and the group is private, a `GET` request on that group results in a `404` status code. diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index 00380e1624b..6b62e82f54d 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -523,6 +523,18 @@ Example response: "container_repositories_registry_count": 5, "container_repositories_synced_in_percentage": "100.00%", "container_repositories_synced_missing_on_primary_count": 0, + "dependency_proxy_manifests_count": 5, + "dependency_proxy_manifests_checksum_total_count": 5, + "dependency_proxy_manifests_checksummed_count": 5, + "dependency_proxy_manifests_checksum_failed_count": 5, + "dependency_proxy_manifests_synced_count": 5, + "dependency_proxy_manifests_failed_count": 0, + "dependency_proxy_manifests_registry_count": 5, + "dependency_proxy_manifests_verification_total_count": 5, + "dependency_proxy_manifests_verified_count": 5, + "dependency_proxy_manifests_verification_failed_count": 5, + "dependency_proxy_manifests_synced_in_percentage": "100.00%", + "dependency_proxy_manifests_verified_in_percentage": "100.00%" }, { "geo_node_id": 2, @@ -707,6 +719,18 @@ Example response: "container_repositories_registry_count": 5, "container_repositories_synced_in_percentage": "100.00%", "container_repositories_synced_missing_on_primary_count": 0, + "dependency_proxy_manifests_count": 5, + "dependency_proxy_manifests_checksum_total_count": 5, + "dependency_proxy_manifests_checksummed_count": 5, + "dependency_proxy_manifests_checksum_failed_count": 5, + "dependency_proxy_manifests_synced_count": 5, + "dependency_proxy_manifests_failed_count": 0, + "dependency_proxy_manifests_registry_count": 5, + "dependency_proxy_manifests_verification_total_count": 5, + "dependency_proxy_manifests_verified_count": 5, + "dependency_proxy_manifests_verification_failed_count": 5, + "dependency_proxy_manifests_synced_in_percentage": "100.00%", + "dependency_proxy_manifests_verified_in_percentage": "100.00%" } ] ``` @@ -901,6 +925,18 @@ Example response: "container_repositories_registry_count": 5, "container_repositories_synced_in_percentage": "100.00%", "container_repositories_synced_missing_on_primary_count": 0, + "dependency_proxy_manifests_count": 5, + "dependency_proxy_manifests_checksum_total_count": 5, + "dependency_proxy_manifests_checksummed_count": 5, + "dependency_proxy_manifests_checksum_failed_count": 5, + "dependency_proxy_manifests_synced_count": 5, + "dependency_proxy_manifests_failed_count": 0, + "dependency_proxy_manifests_registry_count": 5, + "dependency_proxy_manifests_verification_total_count": 5, + "dependency_proxy_manifests_verified_count": 5, + "dependency_proxy_manifests_verification_failed_count": 5, + "dependency_proxy_manifests_synced_in_percentage": "100.00%", + "dependency_proxy_manifests_verified_in_percentage": "100.00%" } ``` diff --git a/doc/api/graphql/audit_report.md b/doc/api/graphql/audit_report.md index 2f96bc5ecdd..5529f0b872a 100644 --- a/doc/api/graphql/audit_report.md +++ b/doc/api/graphql/audit_report.md @@ -20,13 +20,13 @@ The query includes: - [`pageInfo`](#pageinfo) - [`nodes`](#nodes) -## pageInfo +## `pageInfo` This contains the data needed to implement pagination. GitLab uses cursor-based [pagination](getting_started.md#pagination). For more information, see [Pagination](https://graphql.org/learn/pagination/) in the GraphQL documentation. -## nodes +## `nodes` In a GraphQL query, `nodes` is used to represent a collection of [`nodes` on a graph](https://en.wikipedia.org/wiki/Vertex_(graph_theory)). In this case, the collection of nodes is a collection of `User` objects. For each one, diff --git a/doc/api/graphql/index.md b/doc/api/graphql/index.md index 4cf296ac1f3..0ae6013df80 100644 --- a/doc/api/graphql/index.md +++ b/doc/api/graphql/index.md @@ -172,6 +172,7 @@ Limit | Default Max page size | 100 records (nodes) per page. Applies to most connections in the API. Particular connections may have different max page size limits that are higher or lower. [Max query complexity](#max-query-complexity) | `200` for unauthenticated requests and `250` for authenticated requests. Request timeout | 30 seconds. +Max query size | 10,000 characters per query. If this limit is reached, use [variables](https://graphql.org/learn/queries/#variables) and [fragments](https://graphql.org/learn/queries/#fragments) to reduce the query size. Remove white spaces as last resort. ### Max query complexity diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index b1f9d6ceae1..a3d4458bb6c 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -347,7 +347,7 @@ Returns [`Namespace`](#namespace). ### `Query.package` -Find a package. This field can only be resolved for one query in any single request. +Find a package. This field can only be resolved for one query in any single request. Returns `null` if a package has no `default` status. Returns [`PackageDetailsType`](#packagedetailstype). @@ -850,6 +850,25 @@ Input type: `AuditEventsStreamingDestinationEventsAddInput` | <a id="mutationauditeventsstreamingdestinationeventsadderrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationauditeventsstreamingdestinationeventsaddeventtypefilters"></a>`eventTypeFilters` | [`[String!]`](#string) | Event type filters present. | +### `Mutation.auditEventsStreamingDestinationEventsRemove` + +Input type: `AuditEventsStreamingDestinationEventsRemoveInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationauditeventsstreamingdestinationeventsremoveclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationauditeventsstreamingdestinationeventsremovedestinationid"></a>`destinationId` | [`AuditEventsExternalAuditEventDestinationID!`](#auditeventsexternalauditeventdestinationid) | Destination URL. | +| <a id="mutationauditeventsstreamingdestinationeventsremoveeventtypefilters"></a>`eventTypeFilters` | [`[String!]!`](#string) | List of event type filters to remove from streaming. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationauditeventsstreamingdestinationeventsremoveclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationauditeventsstreamingdestinationeventsremoveerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.auditEventsStreamingHeadersCreate` Input type: `AuditEventsStreamingHeadersCreateInput` @@ -2079,8 +2098,8 @@ Input type: `DastSiteProfileCreateInput` | <a id="mutationdastsiteprofilecreatefullpath"></a>`fullPath` | [`ID!`](#id) | Project the site profile belongs to. | | <a id="mutationdastsiteprofilecreateprofilename"></a>`profileName` | [`String!`](#string) | Name of the site profile. | | <a id="mutationdastsiteprofilecreaterequestheaders"></a>`requestHeaders` | [`String`](#string) | Comma-separated list of request header names and values to be added to every request made by DAST. | -| <a id="mutationdastsiteprofilecreatescanfilepath"></a>`scanFilePath` | [`String`](#string) | File Path or URL used as input for the scan method. Will not be saved or updated if `dast_api_scanner` feature flag is disabled. | -| <a id="mutationdastsiteprofilecreatescanmethod"></a>`scanMethod` | [`DastScanMethodType`](#dastscanmethodtype) | Scan method by the scanner. Is not saved or updated if `dast_api_scanner` feature flag is disabled. | +| <a id="mutationdastsiteprofilecreatescanfilepath"></a>`scanFilePath` | [`String`](#string) | File Path or URL used as input for the scan method. | +| <a id="mutationdastsiteprofilecreatescanmethod"></a>`scanMethod` | [`DastScanMethodType`](#dastscanmethodtype) | Scan method by the scanner. | | <a id="mutationdastsiteprofilecreatetargettype"></a>`targetType` | [`DastTargetTypeEnum`](#dasttargettypeenum) | Type of target to be scanned. | | <a id="mutationdastsiteprofilecreatetargeturl"></a>`targetUrl` | [`String`](#string) | URL of the target to be scanned. | @@ -2127,8 +2146,8 @@ Input type: `DastSiteProfileUpdateInput` | <a id="mutationdastsiteprofileupdateid"></a>`id` | [`DastSiteProfileID!`](#dastsiteprofileid) | ID of the site profile to be updated. | | <a id="mutationdastsiteprofileupdateprofilename"></a>`profileName` | [`String!`](#string) | Name of the site profile. | | <a id="mutationdastsiteprofileupdaterequestheaders"></a>`requestHeaders` | [`String`](#string) | Comma-separated list of request header names and values to be added to every request made by DAST. | -| <a id="mutationdastsiteprofileupdatescanfilepath"></a>`scanFilePath` | [`String`](#string) | File Path or URL used as input for the scan method. Will not be saved or updated if `dast_api_scanner` feature flag is disabled. | -| <a id="mutationdastsiteprofileupdatescanmethod"></a>`scanMethod` | [`DastScanMethodType`](#dastscanmethodtype) | Scan method by the scanner. Is not saved or updated if `dast_api_scanner` feature flag is disabled. | +| <a id="mutationdastsiteprofileupdatescanfilepath"></a>`scanFilePath` | [`String`](#string) | File Path or URL used as input for the scan method. | +| <a id="mutationdastsiteprofileupdatescanmethod"></a>`scanMethod` | [`DastScanMethodType`](#dastscanmethodtype) | Scan method by the scanner. | | <a id="mutationdastsiteprofileupdatetargettype"></a>`targetType` | [`DastTargetTypeEnum`](#dasttargettypeenum) | Type of target to be scanned. | | <a id="mutationdastsiteprofileupdatetargeturl"></a>`targetUrl` | [`String`](#string) | URL of the target to be scanned. | @@ -3132,6 +3151,27 @@ Input type: `IssuableResourceLinkDestroyInput` | <a id="mutationissuableresourcelinkdestroyerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationissuableresourcelinkdestroyissuableresourcelink"></a>`issuableResourceLink` | [`IssuableResourceLink`](#issuableresourcelink) | Issuable resource link. | +### `Mutation.issueLinkAlerts` + +Input type: `IssueLinkAlertsInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationissuelinkalertsalertreferences"></a>`alertReferences` | [`[String!]!`](#string) | Alerts references to be linked to the incident. | +| <a id="mutationissuelinkalertsclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationissuelinkalertsiid"></a>`iid` | [`String!`](#string) | IID of the issue to mutate. | +| <a id="mutationissuelinkalertsprojectpath"></a>`projectPath` | [`ID!`](#id) | Project the issue to mutate is in. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationissuelinkalertsclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationissuelinkalertserrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationissuelinkalertsissue"></a>`issue` | [`Issue`](#issue) | Issue after mutation. | + ### `Mutation.issueMove` Input type: `IssueMoveInput` @@ -3434,6 +3474,27 @@ Input type: `IssueSetWeightInput` | <a id="mutationissuesetweighterrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationissuesetweightissue"></a>`issue` | [`Issue`](#issue) | Issue after mutation. | +### `Mutation.issueUnlinkAlert` + +Input type: `IssueUnlinkAlertInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationissueunlinkalertalertid"></a>`alertId` | [`AlertManagementAlertID!`](#alertmanagementalertid) | Global ID of the alert to unlink from the incident. | +| <a id="mutationissueunlinkalertclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationissueunlinkalertiid"></a>`iid` | [`String!`](#string) | IID of the issue to mutate. | +| <a id="mutationissueunlinkalertprojectpath"></a>`projectPath` | [`ID!`](#id) | Project the issue to mutate is in. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationissueunlinkalertclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationissueunlinkalerterrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationissueunlinkalertissue"></a>`issue` | [`Issue`](#issue) | Issue after mutation. | + ### `Mutation.iterationCadenceCreate` Input type: `IterationCadenceCreateInput` @@ -4252,6 +4313,31 @@ Input type: `PipelineRetryInput` | <a id="mutationpipelineretryerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationpipelineretrypipeline"></a>`pipeline` | [`Pipeline`](#pipeline) | Pipeline after mutation. | +### `Mutation.pipelineScheduleCreate` + +Input type: `PipelineScheduleCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationpipelineschedulecreateactive"></a>`active` | [`Boolean`](#boolean) | Indicates if the pipeline schedule should be active or not. | +| <a id="mutationpipelineschedulecreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationpipelineschedulecreatecron"></a>`cron` | [`String!`](#string) | Cron expression of the pipeline schedule. | +| <a id="mutationpipelineschedulecreatecrontimezone"></a>`cronTimezone` | [`String`](#string) | Cron time zone supported by ActiveSupport::TimeZone. For example: "Pacific Time (US & Canada)" (default: "UTC"). | +| <a id="mutationpipelineschedulecreatedescription"></a>`description` | [`String!`](#string) | Description of the pipeline schedule. | +| <a id="mutationpipelineschedulecreateprojectpath"></a>`projectPath` | [`ID!`](#id) | Full path of the project the pipeline schedule is associated with. | +| <a id="mutationpipelineschedulecreateref"></a>`ref` | [`String!`](#string) | Ref of the pipeline schedule. | +| <a id="mutationpipelineschedulecreatevariables"></a>`variables` | [`[PipelineScheduleVariableInput!]`](#pipelineschedulevariableinput) | Variables for the pipeline schedule. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationpipelineschedulecreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationpipelineschedulecreateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationpipelineschedulecreatepipelineschedule"></a>`pipelineSchedule` | [`PipelineSchedule`](#pipelineschedule) | Created pipeline schedule. | + ### `Mutation.pipelineScheduleDelete` Input type: `PipelineScheduleDeleteInput` @@ -4270,6 +4356,25 @@ Input type: `PipelineScheduleDeleteInput` | <a id="mutationpipelinescheduledeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationpipelinescheduledeleteerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +### `Mutation.pipelineSchedulePlay` + +Input type: `PipelineSchedulePlayInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationpipelinescheduleplayclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationpipelinescheduleplayid"></a>`id` | [`CiPipelineScheduleID!`](#cipipelinescheduleid) | ID of the pipeline schedule to mutate. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationpipelinescheduleplayclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationpipelinescheduleplayerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationpipelinescheduleplaypipelineschedule"></a>`pipelineSchedule` | [`PipelineSchedule`](#pipelineschedule) | Pipeline schedule after mutation. | + ### `Mutation.pipelineScheduleTakeOwnership` Input type: `PipelineScheduleTakeOwnershipInput` @@ -4313,6 +4418,25 @@ Input type: `ProjectCiCdSettingsUpdateInput` | <a id="mutationprojectcicdsettingsupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationprojectcicdsettingsupdateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +### `Mutation.projectInitializeProductAnalytics` + +Input type: `ProjectInitializeProductAnalyticsInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationprojectinitializeproductanalyticsclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationprojectinitializeproductanalyticsprojectpath"></a>`projectPath` | [`ID!`](#id) | Full path of the project to initialize. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationprojectinitializeproductanalyticsclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationprojectinitializeproductanalyticserrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationprojectinitializeproductanalyticsproject"></a>`project` | [`Project`](#project) | Project on which the initialization took place. | + ### `Mutation.projectSetComplianceFramework` Assign (or unset) a compliance framework to a project. @@ -5048,6 +5172,7 @@ Input type: `TimelineEventUpdateInput` | <a id="mutationtimelineeventupdateid"></a>`id` | [`IncidentManagementTimelineEventID!`](#incidentmanagementtimelineeventid) | ID of the timeline event to update. | | <a id="mutationtimelineeventupdatenote"></a>`note` | [`String`](#string) | Text note of the timeline event. | | <a id="mutationtimelineeventupdateoccurredat"></a>`occurredAt` | [`Time`](#time) | Timestamp when the event occurred. | +| <a id="mutationtimelineeventupdatetimelineeventtagnames"></a>`timelineEventTagNames` | [`[String!]`](#string) | Tags for the incident timeline event. | #### Fields @@ -5067,7 +5192,7 @@ Input type: `TimelogCreateInput` | ---- | ---- | ----------- | | <a id="mutationtimelogcreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationtimelogcreateissuableid"></a>`issuableId` | [`IssuableID!`](#issuableid) | Global ID of the issuable (Issue, WorkItem or MergeRequest). | -| <a id="mutationtimelogcreatespentat"></a>`spentAt` | [`Date!`](#date) | When the time was spent. | +| <a id="mutationtimelogcreatespentat"></a>`spentAt` | [`Time!`](#time) | When the time was spent. | | <a id="mutationtimelogcreatesummary"></a>`summary` | [`String!`](#string) | Summary of time spent. | | <a id="mutationtimelogcreatetimespent"></a>`timeSpent` | [`String!`](#string) | Amount of time spent. | @@ -5757,7 +5882,7 @@ Input type: `VulnerabilityDismissInput` | Name | Type | Description | | ---- | ---- | ----------- | | <a id="mutationvulnerabilitydismissclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationvulnerabilitydismisscomment"></a>`comment` | [`String`](#string) | Comment why vulnerability should be dismissed. | +| <a id="mutationvulnerabilitydismisscomment"></a>`comment` | [`String`](#string) | Comment why vulnerability should be dismissed (max. 50 000 characters). | | <a id="mutationvulnerabilitydismissdismissalreason"></a>`dismissalReason` | [`VulnerabilityDismissalReason`](#vulnerabilitydismissalreason) | Reason why vulnerability should be dismissed. | | <a id="mutationvulnerabilitydismissid"></a>`id` | [`VulnerabilityID!`](#vulnerabilityid) | ID of the vulnerability to be dismissed. | @@ -5890,6 +6015,7 @@ Input type: `WorkItemCreateInput` | <a id="mutationworkitemcreateconfidential"></a>`confidential` | [`Boolean`](#boolean) | Sets the work item confidentiality. | | <a id="mutationworkitemcreatedescription"></a>`description` | [`String`](#string) | Description of the work item. | | <a id="mutationworkitemcreatehierarchywidget"></a>`hierarchyWidget` | [`WorkItemWidgetHierarchyCreateInput`](#workitemwidgethierarchycreateinput) | Input for hierarchy widget. | +| <a id="mutationworkitemcreateiterationwidget"></a>`iterationWidget` | [`WorkItemWidgetIterationInput`](#workitemwidgetiterationinput) | Iteration widget of the work item. | | <a id="mutationworkitemcreatemilestonewidget"></a>`milestoneWidget` | [`WorkItemWidgetMilestoneInput`](#workitemwidgetmilestoneinput) | Input for milestone widget. | | <a id="mutationworkitemcreateprojectpath"></a>`projectPath` | [`ID!`](#id) | Full path of the project the work item is associated with. | | <a id="mutationworkitemcreatetitle"></a>`title` | [`String!`](#string) | Title of the work item. | @@ -6000,11 +6126,13 @@ Input type: `WorkItemUpdateInput` | <a id="mutationworkitemupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationworkitemupdateconfidential"></a>`confidential` | [`Boolean`](#boolean) | Sets the work item confidentiality. | | <a id="mutationworkitemupdatedescriptionwidget"></a>`descriptionWidget` | [`WorkItemWidgetDescriptionInput`](#workitemwidgetdescriptioninput) | Input for description widget. | +| <a id="mutationworkitemupdatehealthstatuswidget"></a>`healthStatusWidget` | [`WorkItemWidgetHealthStatusInput`](#workitemwidgethealthstatusinput) | Input for health status widget. | | <a id="mutationworkitemupdatehierarchywidget"></a>`hierarchyWidget` | [`WorkItemWidgetHierarchyUpdateInput`](#workitemwidgethierarchyupdateinput) | Input for hierarchy widget. | | <a id="mutationworkitemupdateid"></a>`id` | [`WorkItemID!`](#workitemid) | Global ID of the work item. | | <a id="mutationworkitemupdateiterationwidget"></a>`iterationWidget` | [`WorkItemWidgetIterationInput`](#workitemwidgetiterationinput) | Input for iteration widget. | | <a id="mutationworkitemupdatelabelswidget"></a>`labelsWidget` | [`WorkItemWidgetLabelsUpdateInput`](#workitemwidgetlabelsupdateinput) | Input for labels widget. | | <a id="mutationworkitemupdatemilestonewidget"></a>`milestoneWidget` | [`WorkItemWidgetMilestoneInput`](#workitemwidgetmilestoneinput) | Input for milestone widget. | +| <a id="mutationworkitemupdateprogresswidget"></a>`progressWidget` | [`WorkItemWidgetProgressInput`](#workitemwidgetprogressinput) | Input for progress widget. | | <a id="mutationworkitemupdatestartandduedatewidget"></a>`startAndDueDateWidget` | [`WorkItemWidgetStartAndDueDateUpdateInput`](#workitemwidgetstartandduedateupdateinput) | Input for start and due date widget. | | <a id="mutationworkitemupdatestateevent"></a>`stateEvent` | [`WorkItemStateEvent`](#workitemstateevent) | Close or reopen a work item. | | <a id="mutationworkitemupdatestatuswidget"></a>`statusWidget` | [`StatusInput`](#statusinput) | Input for status widget. | @@ -7329,6 +7457,29 @@ The edge type for [`DependencyProxyManifest`](#dependencyproxymanifest). | <a id="dependencyproxymanifestedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | | <a id="dependencyproxymanifestedgenode"></a>`node` | [`DependencyProxyManifest`](#dependencyproxymanifest) | The item at the end of the edge. | +#### `DependencyProxyManifestRegistryConnection` + +The connection type for [`DependencyProxyManifestRegistry`](#dependencyproxymanifestregistry). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="dependencyproxymanifestregistryconnectionedges"></a>`edges` | [`[DependencyProxyManifestRegistryEdge]`](#dependencyproxymanifestregistryedge) | A list of edges. | +| <a id="dependencyproxymanifestregistryconnectionnodes"></a>`nodes` | [`[DependencyProxyManifestRegistry]`](#dependencyproxymanifestregistry) | A list of nodes. | +| <a id="dependencyproxymanifestregistryconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `DependencyProxyManifestRegistryEdge` + +The edge type for [`DependencyProxyManifestRegistry`](#dependencyproxymanifestregistry). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="dependencyproxymanifestregistryedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="dependencyproxymanifestregistryedgenode"></a>`node` | [`DependencyProxyManifestRegistry`](#dependencyproxymanifestregistry) | The item at the end of the edge. | + #### `DeploymentConnection` The connection type for [`Deployment`](#deployment). @@ -8326,6 +8477,29 @@ The edge type for [`Namespace`](#namespace). | <a id="namespaceedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | | <a id="namespaceedgenode"></a>`node` | [`Namespace`](#namespace) | The item at the end of the edge. | +#### `NestedEnvironmentConnection` + +The connection type for [`NestedEnvironment`](#nestedenvironment). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="nestedenvironmentconnectionedges"></a>`edges` | [`[NestedEnvironmentEdge]`](#nestedenvironmentedge) | A list of edges. | +| <a id="nestedenvironmentconnectionnodes"></a>`nodes` | [`[NestedEnvironment]`](#nestedenvironment) | A list of nodes. | +| <a id="nestedenvironmentconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `NestedEnvironmentEdge` + +The edge type for [`NestedEnvironment`](#nestedenvironment). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="nestedenvironmentedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="nestedenvironmentedgenode"></a>`node` | [`NestedEnvironment`](#nestedenvironment) | The item at the end of the edge. | + #### `NetworkPolicyConnection` The connection type for [`NetworkPolicy`](#networkpolicy). @@ -8652,6 +8826,29 @@ The edge type for [`PipelineSchedule`](#pipelineschedule). | <a id="pipelinescheduleedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | | <a id="pipelinescheduleedgenode"></a>`node` | [`PipelineSchedule`](#pipelineschedule) | The item at the end of the edge. | +#### `PipelineScheduleVariableConnection` + +The connection type for [`PipelineScheduleVariable`](#pipelineschedulevariable). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="pipelineschedulevariableconnectionedges"></a>`edges` | [`[PipelineScheduleVariableEdge]`](#pipelineschedulevariableedge) | A list of edges. | +| <a id="pipelineschedulevariableconnectionnodes"></a>`nodes` | [`[PipelineScheduleVariable]`](#pipelineschedulevariable) | A list of nodes. | +| <a id="pipelineschedulevariableconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `PipelineScheduleVariableEdge` + +The edge type for [`PipelineScheduleVariable`](#pipelineschedulevariable). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="pipelineschedulevariableedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="pipelineschedulevariableedgenode"></a>`node` | [`PipelineScheduleVariable`](#pipelineschedulevariable) | The item at the end of the edge. | + #### `PipelineSecurityReportFindingConnection` The connection type for [`PipelineSecurityReportFinding`](#pipelinesecurityreportfinding). @@ -10087,6 +10284,7 @@ Describes an alert from the project's Alert Management. | <a id="alertmanagementalertenvironment"></a>`environment` | [`Environment`](#environment) | Environment for the alert. | | <a id="alertmanagementalerteventcount"></a>`eventCount` | [`Int`](#int) | Number of events of this alert. | | <a id="alertmanagementalerthosts"></a>`hosts` | [`[String!]`](#string) | List of hosts the alert came from. | +| <a id="alertmanagementalertid"></a>`id` | [`ID!`](#id) | ID of the alert. | | <a id="alertmanagementalertiid"></a>`iid` | [`ID!`](#id) | Internal ID of the alert. | | <a id="alertmanagementalertissue"></a>`issue` | [`Issue`](#issue) | Issue attached to the alert. | | <a id="alertmanagementalertissueiid"></a>`issueIid` **{warning-solid}** | [`ID`](#id) | **Deprecated** in 13.10. Use issue field. | @@ -10787,6 +10985,21 @@ CI/CD config variables. | <a id="ciconfigvariablevalue"></a>`value` | [`String`](#string) | Value of the variable. | | <a id="ciconfigvariablevalueoptions"></a>`valueOptions` | [`[String!]`](#string) | Value options for the variable. | +### `CiFreezePeriod` + +Represents a deployment freeze window of a project. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="cifreezeperiodcrontimezone"></a>`cronTimezone` | [`String`](#string) | Time zone for the cron fields, defaults to UTC if not provided. | +| <a id="cifreezeperiodendcron"></a>`endCron` | [`String!`](#string) | End of the freeze period in cron format. | +| <a id="cifreezeperiodendtime"></a>`endTime` | [`Time`](#time) | Timestamp (UTC) of when the current/next active period ends. | +| <a id="cifreezeperiodstartcron"></a>`startCron` | [`String!`](#string) | Start of the freeze period in cron format. | +| <a id="cifreezeperiodstarttime"></a>`startTime` | [`Time`](#time) | Timestamp (UTC) of when the current/next active period starts. | +| <a id="cifreezeperiodstatus"></a>`status` | [`CiFreezePeriodStatus!`](#cifreezeperiodstatus) | Freeze period status. | + ### `CiGroup` #### Fields @@ -10970,10 +11183,11 @@ CI/CD variables for a project. | <a id="cirunnerdescription"></a>`description` | [`String`](#string) | Description of the runner. | | <a id="cirunnereditadminurl"></a>`editAdminUrl` | [`String`](#string) | Admin form URL of the runner. Only available for administrators. | | <a id="cirunnerexecutorname"></a>`executorName` | [`String`](#string) | Executor last advertised by the runner. | -| <a id="cirunnergroups"></a>`groups` | [`GroupConnection`](#groupconnection) | Groups the runner is associated with. For group runners only. (see [Connections](#connections)) | +| <a id="cirunnergroups"></a>`groups` | [`GroupConnection`](#groupconnection) | Types::GroupConnection. (see [Connections](#connections)) | | <a id="cirunnerid"></a>`id` | [`CiRunnerID!`](#cirunnerid) | ID of the runner. | | <a id="cirunneripaddress"></a>`ipAddress` | [`String`](#string) | IP address of the runner. | | <a id="cirunnerjobcount"></a>`jobCount` | [`Int`](#int) | Number of jobs processed by the runner (limited to 1000, plus one to indicate that more items exist). | +| <a id="cirunnerjobexecutionstatus"></a>`jobExecutionStatus` **{warning-solid}** | [`CiRunnerJobExecutionStatus`](#cirunnerjobexecutionstatus) | **Introduced** in 15.7. This feature is in Alpha. It can be changed or removed at any time. Job execution status of the runner. | | <a id="cirunnerlocked"></a>`locked` | [`Boolean`](#boolean) | Indicates the runner is locked. | | <a id="cirunnermaintenancenote"></a>`maintenanceNote` | [`String`](#string) | Runner's maintenance notes. | | <a id="cirunnermaintenancenotehtml"></a>`maintenanceNoteHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `maintenance_note`. | @@ -11644,8 +11858,8 @@ Represents a DAST Site Profile. | <a id="dastsiteprofileprofilename"></a>`profileName` | [`String`](#string) | Name of the site profile. | | <a id="dastsiteprofilereferencedinsecuritypolicies"></a>`referencedInSecurityPolicies` | [`[String!]`](#string) | List of security policy names that are referencing given project. | | <a id="dastsiteprofilerequestheaders"></a>`requestHeaders` | [`String`](#string) | Comma-separated list of request header names and values to be added to every request made by DAST. | -| <a id="dastsiteprofilescanfilepath"></a>`scanFilePath` | [`String`](#string) | Scan File Path used as input for the scanner. Will always return `null` if `dast_api_scanner` feature flag is disabled. | -| <a id="dastsiteprofilescanmethod"></a>`scanMethod` | [`DastScanMethodType`](#dastscanmethodtype) | Scan method used by the scanner. Always returns `null` if `dast_api_scanner` feature flag is disabled. | +| <a id="dastsiteprofilescanfilepath"></a>`scanFilePath` | [`String`](#string) | Scan File Path used as input for the scanner. | +| <a id="dastsiteprofilescanmethod"></a>`scanMethod` | [`DastScanMethodType`](#dastscanmethodtype) | Scan method used by the scanner. | | <a id="dastsiteprofiletargettype"></a>`targetType` | [`DastTargetTypeEnum`](#dasttargettypeenum) | Type of target to be scanned. | | <a id="dastsiteprofiletargeturl"></a>`targetUrl` | [`String`](#string) | URL of the target to be scanned. | | <a id="dastsiteprofileuserpermissions"></a>`userPermissions` | [`DastSiteProfilePermissions!`](#dastsiteprofilepermissions) | Permissions for the current user on the resource. | @@ -11765,6 +11979,25 @@ Dependency proxy manifest. | <a id="dependencyproxymanifeststatus"></a>`status` | [`DependencyProxyManifestStatus!`](#dependencyproxymanifeststatus) | Status of the manifest (default, pending_destruction, processing, error). | | <a id="dependencyproxymanifestupdatedat"></a>`updatedAt` | [`Time!`](#time) | Date of most recent update. | +### `DependencyProxyManifestRegistry` + +Represents the Geo replication and verification state of a dependency_proxy_manifest. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="dependencyproxymanifestregistrycreatedat"></a>`createdAt` | [`Time`](#time) | Timestamp when the DependencyProxyManifestRegistry was created. | +| <a id="dependencyproxymanifestregistrydependencyproxymanifestid"></a>`dependencyProxyManifestId` | [`ID!`](#id) | ID of the Dependency Proxy Manifest. | +| <a id="dependencyproxymanifestregistryid"></a>`id` | [`ID!`](#id) | ID of the DependencyProxyManifestRegistry. | +| <a id="dependencyproxymanifestregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the DependencyProxyManifestRegistry. | +| <a id="dependencyproxymanifestregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the DependencyProxyManifestRegistry. | +| <a id="dependencyproxymanifestregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the DependencyProxyManifestRegistry is resynced. | +| <a id="dependencyproxymanifestregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the DependencyProxyManifestRegistry. | +| <a id="dependencyproxymanifestregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the DependencyProxyManifestRegistry. | +| <a id="dependencyproxymanifestregistryverificationretryat"></a>`verificationRetryAt` | [`Time`](#time) | Timestamp after which the DependencyProxyManifestRegistry is reverified. | +| <a id="dependencyproxymanifestregistryverifiedat"></a>`verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the DependencyProxyManifestRegistry. | + ### `DependencyProxySetting` Group-level Dependency Proxy settings. @@ -11783,6 +12016,7 @@ The deployment of an environment. | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="deploymentapprovalsummary"></a>`approvalSummary` | [`DeploymentApprovalSummary`](#deploymentapprovalsummary) | Approval summary of the deployment.This field can only be resolved for one deployment in any single request. | | <a id="deploymentcommit"></a>`commit` | [`Commit`](#commit) | Commit details of the deployment. | | <a id="deploymentcreatedat"></a>`createdAt` | [`Time`](#time) | When the deployment record was created. | | <a id="deploymentfinishedat"></a>`finishedAt` | [`Time`](#time) | When the deployment finished. | @@ -11793,8 +12027,10 @@ The deployment of an environment. | <a id="deploymentsha"></a>`sha` | [`String`](#string) | Git-SHA that the deployment ran on. | | <a id="deploymentstatus"></a>`status` | [`DeploymentStatus`](#deploymentstatus) | Status of the deployment. | | <a id="deploymenttag"></a>`tag` | [`Boolean`](#boolean) | True or false if the deployment ran on a Git-tag. | +| <a id="deploymenttags"></a>`tags` | [`[DeploymentTag!]`](#deploymenttag) | Git tags that contain this deployment. This field can only be resolved for one deployment in any single request. | | <a id="deploymenttriggerer"></a>`triggerer` | [`UserCore`](#usercore) | User who executed the deployment. | | <a id="deploymentupdatedat"></a>`updatedAt` | [`Time`](#time) | When the deployment record was updated. | +| <a id="deploymentuserpermissions"></a>`userPermissions` | [`DeploymentPermissions!`](#deploymentpermissions) | Permissions for the current user on the resource. | ### `DeploymentApproval` @@ -11823,28 +12059,15 @@ Approval summary of the deployment. | <a id="deploymentapprovalsummarytotalpendingapprovalcount"></a>`totalPendingApprovalCount` | [`Int`](#int) | Total pending approval count. | | <a id="deploymentapprovalsummarytotalrequiredapprovals"></a>`totalRequiredApprovals` | [`Int`](#int) | Total number of required approvals. | -### `DeploymentDetails` - -The details of the deployment. +### `DeploymentPermissions` #### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="deploymentdetailsapprovalsummary"></a>`approvalSummary` | [`DeploymentApprovalSummary`](#deploymentapprovalsummary) | Approval summary of the deployment. | -| <a id="deploymentdetailscommit"></a>`commit` | [`Commit`](#commit) | Commit details of the deployment. | -| <a id="deploymentdetailscreatedat"></a>`createdAt` | [`Time`](#time) | When the deployment record was created. | -| <a id="deploymentdetailsfinishedat"></a>`finishedAt` | [`Time`](#time) | When the deployment finished. | -| <a id="deploymentdetailsid"></a>`id` | [`ID`](#id) | Global ID of the deployment. | -| <a id="deploymentdetailsiid"></a>`iid` | [`ID`](#id) | Project-level internal ID of the deployment. | -| <a id="deploymentdetailsjob"></a>`job` | [`CiJob`](#cijob) | Pipeline job of the deployment. | -| <a id="deploymentdetailsref"></a>`ref` | [`String`](#string) | Git-Ref that the deployment ran on. | -| <a id="deploymentdetailssha"></a>`sha` | [`String`](#string) | Git-SHA that the deployment ran on. | -| <a id="deploymentdetailsstatus"></a>`status` | [`DeploymentStatus`](#deploymentstatus) | Status of the deployment. | -| <a id="deploymentdetailstag"></a>`tag` | [`Boolean`](#boolean) | True or false if the deployment ran on a Git-tag. | -| <a id="deploymentdetailstags"></a>`tags` | [`[DeploymentTag!]`](#deploymenttag) | Git tags that contain this deployment. | -| <a id="deploymentdetailstriggerer"></a>`triggerer` | [`UserCore`](#usercore) | User who executed the deployment. | -| <a id="deploymentdetailsupdatedat"></a>`updatedAt` | [`Time`](#time) | When the deployment record was updated. | +| <a id="deploymentpermissionsapprovedeployment"></a>`approveDeployment` | [`Boolean!`](#boolean) | Indicates the user can perform `approve_deployment` on this resource. This field can only be resolved for one environment in any single request. | +| <a id="deploymentpermissionsdestroydeployment"></a>`destroyDeployment` | [`Boolean!`](#boolean) | Indicates the user can perform `destroy_deployment` on this resource. | +| <a id="deploymentpermissionsupdatedeployment"></a>`updateDeployment` | [`Boolean!`](#boolean) | Indicates the user can perform `update_deployment` on this resource. | ### `DeploymentTag` @@ -12283,6 +12506,7 @@ Describes where code is deployed for a project. | <a id="environmentautodeleteat"></a>`autoDeleteAt` | [`Time`](#time) | When the environment is going to be deleted automatically. | | <a id="environmentautostopat"></a>`autoStopAt` | [`Time`](#time) | When the environment is going to be stopped automatically. | | <a id="environmentcreatedat"></a>`createdAt` | [`Time`](#time) | When the environment was created. | +| <a id="environmentdeployfreezes"></a>`deployFreezes` | [`[CiFreezePeriod!]`](#cifreezeperiod) | Deployment freeze periods of the environment. | | <a id="environmentenvironmenttype"></a>`environmentType` | [`String`](#string) | Folder name of the environment. | | <a id="environmentexternalurl"></a>`externalUrl` | [`String`](#string) | External URL of the environment. | | <a id="environmentid"></a>`id` | [`ID!`](#id) | ID of the environment. | @@ -12294,6 +12518,7 @@ Describes where code is deployed for a project. | <a id="environmentstate"></a>`state` | [`String!`](#string) | State of the environment, for example: available/stopped. | | <a id="environmenttier"></a>`tier` | [`DeploymentTier`](#deploymenttier) | Deployment tier of the environment. | | <a id="environmentupdatedat"></a>`updatedAt` | [`Time`](#time) | When the environment was updated. | +| <a id="environmentuserpermissions"></a>`userPermissions` | [`EnvironmentPermissions!`](#environmentpermissions) | Permissions for the current user on the resource. This field can only be resolved for one environment in any single request. | #### Fields with arguments @@ -12338,6 +12563,16 @@ Returns [`MetricsDashboard`](#metricsdashboard). | ---- | ---- | ----------- | | <a id="environmentmetricsdashboardpath"></a>`path` | [`String!`](#string) | Path to a file which defines a metrics dashboard eg: `"config/prometheus/common_metrics.yml"`. | +### `EnvironmentPermissions` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="environmentpermissionsdestroyenvironment"></a>`destroyEnvironment` | [`Boolean!`](#boolean) | Indicates the user can perform `destroy_environment` on this resource. | +| <a id="environmentpermissionsstopenvironment"></a>`stopEnvironment` | [`Boolean!`](#boolean) | Indicates the user can perform `stop_environment` on this resource. | +| <a id="environmentpermissionsupdateenvironment"></a>`updateEnvironment` | [`Boolean!`](#boolean) | Indicates the user can perform `update_environment` on this resource. | + ### `Epic` Represents an epic. @@ -12628,6 +12863,7 @@ Relationship between an epic and an issue. | <a id="epicissuenotes"></a>`notes` | [`NoteConnection!`](#noteconnection) | All notes on this noteable. (see [Connections](#connections)) | | <a id="epicissueparticipants"></a>`participants` | [`UserCoreConnection`](#usercoreconnection) | List of participants in the issue. (see [Connections](#connections)) | | <a id="epicissueprojectid"></a>`projectId` | [`Int!`](#int) | ID of the issue project. | +| <a id="epicissuerelatedvulnerabilities"></a>`relatedVulnerabilities` | [`VulnerabilityConnection`](#vulnerabilityconnection) | Related vulnerabilities of the issue. (see [Connections](#connections)) | | <a id="epicissuerelationpath"></a>`relationPath` | [`String`](#string) | URI path of the epic-issue relation. | | <a id="epicissuerelativeposition"></a>`relativePosition` | [`Int`](#int) | Relative position of the issue (used for positioning in epic tree and issue boards). | | <a id="epicissueseverity"></a>`severity` | [`IssuableSeverity`](#issuableseverity) | Severity level of the incident. | @@ -12876,6 +13112,17 @@ Describes an external status check. | <a id="fileuploadpath"></a>`path` | [`String!`](#string) | Path of the upload. | | <a id="fileuploadsize"></a>`size` | [`Int!`](#int) | Size of the upload in bytes. | +### `ForkDetails` + +Details of the fork project compared to its upstream project. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="forkdetailsahead"></a>`ahead` | [`Int`](#int) | Number of commits ahead of upstream. | +| <a id="forkdetailsbehind"></a>`behind` | [`Int`](#int) | Number of commits behind upstream. | + ### `GeoNode` #### Fields @@ -12920,11 +13167,7 @@ four standard [pagination arguments](#connection-pagination-arguments): ##### `GeoNode.containerRepositoryRegistries` -Find Container Repository registries on this Geo node. Ignored if `geo_container_repository_replication` feature flag is disabled. - -WARNING: -**Introduced** in 15.5. -This feature is in Alpha. It can be changed or removed at any time. +Find Container Repository registries on this Geo node. Returns [`ContainerRepositoryRegistryConnection`](#containerrepositoryregistryconnection). @@ -12962,6 +13205,28 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="geonodedependencyproxyblobregistriesreplicationstate"></a>`replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | <a id="geonodedependencyproxyblobregistriesverificationstate"></a>`verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | +##### `GeoNode.dependencyProxyManifestRegistries` + +Find Dependency Proxy Manifest registries on this Geo node. Ignored if `geo_dependency_proxy_manifest_replication` feature flag is disabled. + +WARNING: +**Introduced** in 15.6. +This feature is in Alpha. It can be changed or removed at any time. + +Returns [`DependencyProxyManifestRegistryConnection`](#dependencyproxymanifestregistryconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="geonodedependencyproxymanifestregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| <a id="geonodedependencyproxymanifestregistriesreplicationstate"></a>`replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| <a id="geonodedependencyproxymanifestregistriesverificationstate"></a>`verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | + ##### `GeoNode.groupWikiRepositoryRegistries` Find group wiki repository registries on this Geo node. @@ -13938,6 +14203,7 @@ Returns [`[VulnerableProjectsByGrade!]!`](#vulnerableprojectsbygrade). | Name | Type | Description | | ---- | ---- | ----------- | | <a id="groupvulnerabilitygradesincludesubgroups"></a>`includeSubgroups` | [`Boolean`](#boolean) | Include grades belonging to subgroups. | +| <a id="groupvulnerabilitygradeslettergrade"></a>`letterGrade` | [`VulnerabilityGrade`](#vulnerabilitygrade) | Filter the response by given letter grade. | ##### `Group.vulnerabilitySeveritiesCount` @@ -14165,7 +14431,6 @@ A block of time for which a participant is on-call. | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="instancesecuritydashboardvulnerabilitygrades"></a>`vulnerabilityGrades` | [`[VulnerableProjectsByGrade!]!`](#vulnerableprojectsbygrade) | Represents vulnerable project counts for each grade. | | <a id="instancesecuritydashboardvulnerabilityscanners"></a>`vulnerabilityScanners` | [`VulnerabilityScannerConnection`](#vulnerabilityscannerconnection) | Vulnerability scanners reported on the vulnerabilities from projects selected in Instance Security Dashboard. (see [Connections](#connections)) | #### Fields with arguments @@ -14202,6 +14467,18 @@ four standard [pagination arguments](#connection-pagination-arguments): | ---- | ---- | ----------- | | <a id="instancesecuritydashboardprojectssearch"></a>`search` | [`String`](#string) | Search query, which can be for the project name, a path, or a description. | +##### `InstanceSecurityDashboard.vulnerabilityGrades` + +Represents vulnerable project counts for each grade. + +Returns [`[VulnerableProjectsByGrade!]!`](#vulnerableprojectsbygrade). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="instancesecuritydashboardvulnerabilitygradeslettergrade"></a>`letterGrade` | [`VulnerabilityGrade`](#vulnerabilitygrade) | Filter the response by given letter grade. | + ##### `InstanceSecurityDashboard.vulnerabilitySeveritiesCount` Counts for each vulnerability severity from projects selected in Instance Security Dashboard. @@ -14284,6 +14561,7 @@ Describes an issuable resource link for incident issues. | <a id="issuenotes"></a>`notes` | [`NoteConnection!`](#noteconnection) | All notes on this noteable. (see [Connections](#connections)) | | <a id="issueparticipants"></a>`participants` | [`UserCoreConnection`](#usercoreconnection) | List of participants in the issue. (see [Connections](#connections)) | | <a id="issueprojectid"></a>`projectId` | [`Int!`](#int) | ID of the issue project. | +| <a id="issuerelatedvulnerabilities"></a>`relatedVulnerabilities` | [`VulnerabilityConnection`](#vulnerabilityconnection) | Related vulnerabilities of the issue. (see [Connections](#connections)) | | <a id="issuerelativeposition"></a>`relativePosition` | [`Int`](#int) | Relative position of the issue (used for positioning in epic tree and issue boards). | | <a id="issueseverity"></a>`severity` | [`IssuableSeverity`](#issuableseverity) | Severity level of the incident. | | <a id="issuesladueat"></a>`slaDueAt` | [`Time`](#time) | Timestamp of when the issue SLA expires. | @@ -14564,6 +14842,20 @@ Represents the Geo replication and verification state of a job_artifact. | <a id="kasexternalurl"></a>`externalUrl` | [`String`](#string) | URL used by the Agents to communicate with KAS. | | <a id="kasversion"></a>`version` | [`String`](#string) | KAS version. | +### `Key` + +Represents an SSH key. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="keycreatedat"></a>`createdAt` | [`Time!`](#time) | Timestamp of when the key was created. | +| <a id="keyexpiresat"></a>`expiresAt` | [`Time!`](#time) | Timestamp of when the key expires. It's null if it never expires. | +| <a id="keyid"></a>`id` | [`ID!`](#id) | ID of the key. | +| <a id="keykey"></a>`key` | [`String!`](#string) | Public key of the key pair. | +| <a id="keytitle"></a>`title` | [`String!`](#string) | Title of the key. | + ### `Label` #### Fields @@ -16014,6 +16306,18 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="namespacecicdsettingallowstalerunnerpruning"></a>`allowStaleRunnerPruning` | [`Boolean`](#boolean) | Indicates if stale runners directly belonging to this namespace should be periodically pruned. | | <a id="namespacecicdsettingnamespace"></a>`namespace` | [`Namespace`](#namespace) | Namespace the CI/CD settings belong to. | +### `NestedEnvironment` + +Describes where code is deployed for a project organized by folder. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="nestedenvironmentenvironment"></a>`environment` | [`Environment`](#environment) | Latest environment in the folder. | +| <a id="nestedenvironmentname"></a>`name` | [`String!`](#string) | Human-readable name of the environment. | +| <a id="nestedenvironmentsize"></a>`size` | [`Int!`](#int) | Number of environments nested in the folder. | + ### `NetworkPolicy` Represents the network policy. @@ -16643,23 +16947,31 @@ Represents pipeline counts for the project. ### `PipelineSchedule` +Represents a pipeline schedule. + #### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="pipelinescheduleactive"></a>`active` | [`Boolean!`](#boolean) | Indicates if a pipeline schedule is active. | +| <a id="pipelinescheduleactive"></a>`active` | [`Boolean!`](#boolean) | Indicates if the pipeline schedule is active. | +| <a id="pipelineschedulecreatedat"></a>`createdAt` | [`Time!`](#time) | Timestamp of when the pipeline schedule was created. | | <a id="pipelineschedulecron"></a>`cron` | [`String!`](#string) | Cron notation for the schedule. | | <a id="pipelineschedulecrontimezone"></a>`cronTimezone` | [`String!`](#string) | Timezone for the pipeline schedule. | | <a id="pipelinescheduledescription"></a>`description` | [`String`](#string) | Description of the pipeline schedule. | +| <a id="pipelinescheduleeditpath"></a>`editPath` | [`String`](#string) | Edit path of the pipeline schedule. | | <a id="pipelineschedulefortag"></a>`forTag` | [`Boolean!`](#boolean) | Indicates if a pipelines schedule belongs to a tag. | | <a id="pipelinescheduleid"></a>`id` | [`ID!`](#id) | ID of the pipeline schedule. | | <a id="pipelineschedulelastpipeline"></a>`lastPipeline` | [`Pipeline`](#pipeline) | Last pipeline object. | | <a id="pipelineschedulenextrunat"></a>`nextRunAt` | [`Time!`](#time) | Time when the next pipeline will run. | | <a id="pipelinescheduleowner"></a>`owner` | [`UserCore!`](#usercore) | Owner of the pipeline schedule. | +| <a id="pipelinescheduleproject"></a>`project` | [`Project`](#project) | Project of the pipeline schedule. | | <a id="pipelineschedulerealnextrun"></a>`realNextRun` | [`Time!`](#time) | Time when the next pipeline will run. | +| <a id="pipelinescheduleref"></a>`ref` | [`String`](#string) | Ref of the pipeline schedule. | | <a id="pipelineschedulereffordisplay"></a>`refForDisplay` | [`String`](#string) | Git ref for the pipeline schedule. | | <a id="pipelineschedulerefpath"></a>`refPath` | [`String`](#string) | Path to the ref that triggered the pipeline. | +| <a id="pipelinescheduleupdatedat"></a>`updatedAt` | [`Time!`](#time) | Timestamp of when the pipeline schedule was last updated. | | <a id="pipelinescheduleuserpermissions"></a>`userPermissions` | [`PipelineSchedulePermissions!`](#pipelineschedulepermissions) | Permissions for the current user on the resource. | +| <a id="pipelineschedulevariables"></a>`variables` | [`PipelineScheduleVariableConnection`](#pipelineschedulevariableconnection) | Pipeline schedule variables. (see [Connections](#connections)) | ### `PipelineSchedulePermissions` @@ -16672,6 +16984,18 @@ Represents pipeline counts for the project. | <a id="pipelineschedulepermissionstakeownershippipelineschedule"></a>`takeOwnershipPipelineSchedule` | [`Boolean!`](#boolean) | Indicates the user can perform `take_ownership_pipeline_schedule` on this resource. | | <a id="pipelineschedulepermissionsupdatepipelineschedule"></a>`updatePipelineSchedule` | [`Boolean!`](#boolean) | Indicates the user can perform `update_pipeline_schedule` on this resource. | +### `PipelineScheduleVariable` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="pipelineschedulevariableid"></a>`id` | [`ID!`](#id) | ID of the variable. | +| <a id="pipelineschedulevariablekey"></a>`key` | [`String`](#string) | Name of the variable. | +| <a id="pipelineschedulevariableraw"></a>`raw` | [`Boolean`](#boolean) | Indicates whether the variable is raw. | +| <a id="pipelineschedulevariablevalue"></a>`value` | [`String`](#string) | Value of the variable. | +| <a id="pipelineschedulevariablevariabletype"></a>`variableType` | [`CiVariableType`](#civariabletype) | Type of the variable. | + ### `PipelineSecurityReportFinding` Represents vulnerability finding of a security report on the pipeline. @@ -16696,10 +17020,11 @@ Represents vulnerability finding of a security report on the pipeline. | <a id="pipelinesecurityreportfindingreporttype"></a>`reportType` | [`VulnerabilityReportType`](#vulnerabilityreporttype) | Type of the security report that found the vulnerability finding. | | <a id="pipelinesecurityreportfindingscanner"></a>`scanner` | [`VulnerabilityScanner`](#vulnerabilityscanner) | Scanner metadata for the vulnerability. | | <a id="pipelinesecurityreportfindingseverity"></a>`severity` | [`VulnerabilitySeverity`](#vulnerabilityseverity) | Severity of the vulnerability finding. | -| <a id="pipelinesecurityreportfindingsolution"></a>`solution` | [`String`](#string) | URL to the vulnerability's details page. | +| <a id="pipelinesecurityreportfindingsolution"></a>`solution` | [`String`](#string) | Solution for resolving the security report finding. | | <a id="pipelinesecurityreportfindingstate"></a>`state` | [`VulnerabilityState`](#vulnerabilitystate) | Finding status. | | <a id="pipelinesecurityreportfindingtitle"></a>`title` | [`String`](#string) | Title of the vulnerability finding. | -| <a id="pipelinesecurityreportfindinguuid"></a>`uuid` | [`String`](#string) | Name of the vulnerability finding. | +| <a id="pipelinesecurityreportfindinguuid"></a>`uuid` | [`String`](#string) | UUIDv5 digest based on the vulnerability's report type, primary identifier, location, fingerprint, project identifier. | +| <a id="pipelinesecurityreportfindingvulnerability"></a>`vulnerability` | [`Vulnerability`](#vulnerability) | Vulnerability related to the security report finding. | ### `PreviewBillableUserChange` @@ -16707,9 +17032,9 @@ Represents vulnerability finding of a security report on the pipeline. | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="previewbillableuserchangehasoverage"></a>`hasOverage` | [`Boolean`](#boolean) | If the group has an overage after change. | | <a id="previewbillableuserchangenewbillableusercount"></a>`newBillableUserCount` | [`Int`](#int) | Total number of billable users after change. | | <a id="previewbillableuserchangeseatsinsubscription"></a>`seatsInSubscription` | [`Int`](#int) | Number of seats in subscription. | +| <a id="previewbillableuserchangewillincreaseoverage"></a>`willIncreaseOverage` | [`Boolean`](#boolean) | If the group will have an increased overage after change. | ### `ProductAnalyticsDashboard` @@ -16723,6 +17048,18 @@ Represents a product analytics dashboard. | <a id="productanalyticsdashboardtitle"></a>`title` | [`String!`](#string) | Title of the dashboard. | | <a id="productanalyticsdashboardwidgets"></a>`widgets` | [`ProductAnalyticsDashboardWidgetConnection!`](#productanalyticsdashboardwidgetconnection) | Widgets shown on the dashboard. (see [Connections](#connections)) | +### `ProductAnalyticsDashboardVisualization` + +Represents a product analytics dashboard visualization. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="productanalyticsdashboardvisualizationdata"></a>`data` | [`JSON!`](#json) | Data of the visualization. | +| <a id="productanalyticsdashboardvisualizationoptions"></a>`options` | [`JSON!`](#json) | Options of the visualization. | +| <a id="productanalyticsdashboardvisualizationtype"></a>`type` | [`String!`](#string) | Type of the visualization. | + ### `ProductAnalyticsDashboardWidget` Represents a product analytics dashboard widget. @@ -16733,6 +17070,7 @@ Represents a product analytics dashboard widget. | ---- | ---- | ----------- | | <a id="productanalyticsdashboardwidgetgridattributes"></a>`gridAttributes` | [`JSON`](#json) | Description of the position and size of the widget. | | <a id="productanalyticsdashboardwidgettitle"></a>`title` | [`String!`](#string) | Title of the widget. | +| <a id="productanalyticsdashboardwidgetvisualization"></a>`visualization` | [`ProductAnalyticsDashboardVisualization!`](#productanalyticsdashboardvisualization) | Visualization of the widget. | ### `Project` @@ -16775,6 +17113,7 @@ Represents a product analytics dashboard widget. | <a id="projectissuesenabled"></a>`issuesEnabled` | [`Boolean`](#boolean) | Indicates if Issues are enabled for the current user. | | <a id="projectjiraimportstatus"></a>`jiraImportStatus` | [`String`](#string) | Status of Jira import background job of the project. | | <a id="projectjiraimports"></a>`jiraImports` | [`JiraImportConnection`](#jiraimportconnection) | Jira imports into the project. (see [Connections](#connections)) | +| <a id="projectjitsukey"></a>`jitsuKey` **{warning-solid}** | [`String`](#string) | **Introduced** in 15.7. This feature is in Alpha. It can be changed or removed at any time. Jitsu key assigned to the project. | | <a id="projectjobsenabled"></a>`jobsEnabled` | [`Boolean`](#boolean) | Indicates if CI/CD pipeline jobs are enabled for the current user. | | <a id="projectlanguages"></a>`languages` | [`[RepositoryLanguage!]`](#repositorylanguage) | Programming languages used in the project. | | <a id="projectlastactivityat"></a>`lastActivityAt` | [`Time`](#time) | Timestamp of the project last activity. | @@ -17088,7 +17427,7 @@ four standard [pagination arguments](#connection-pagination-arguments): Details of the deployment of the project. -Returns [`DeploymentDetails`](#deploymentdetails). +Returns [`Deployment`](#deployment). ###### Arguments @@ -17109,10 +17448,11 @@ Returns [`Environment`](#environment). | <a id="projectenvironmentname"></a>`name` | [`String`](#string) | Name of the environment. | | <a id="projectenvironmentsearch"></a>`search` | [`String`](#string) | Search query for environment name. | | <a id="projectenvironmentstates"></a>`states` | [`[String!]`](#string) | States of environments that should be included in result. | +| <a id="projectenvironmenttype"></a>`type` | [`String`](#string) | Search query for environment type. | ##### `Project.environments` -Environments of the project. +Environments of the project. This field can only be resolved for one project in any single request. Returns [`EnvironmentConnection`](#environmentconnection). @@ -17127,6 +17467,23 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="projectenvironmentsname"></a>`name` | [`String`](#string) | Name of the environment. | | <a id="projectenvironmentssearch"></a>`search` | [`String`](#string) | Search query for environment name. | | <a id="projectenvironmentsstates"></a>`states` | [`[String!]`](#string) | States of environments that should be included in result. | +| <a id="projectenvironmentstype"></a>`type` | [`String`](#string) | Search query for environment type. | + +##### `Project.forkDetails` + +Details of the fork project compared to its upstream project. + +WARNING: +**Introduced** in 15.7. +This feature is in Alpha. It can be changed or removed at any time. + +Returns [`ForkDetails`](#forkdetails). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="projectforkdetailsref"></a>`ref` | [`String`](#string) | Ref of the fork. Default value is HEAD. | ##### `Project.forkTargets` @@ -17447,6 +17804,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="projectjobsstatuses"></a>`statuses` | [`[CiJobStatus!]`](#cijobstatus) | Filter jobs by status. | +| <a id="projectjobswithartifacts"></a>`withArtifacts` | [`Boolean`](#boolean) | Filter by artifacts presence. | ##### `Project.label` @@ -17547,6 +17905,25 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="projectmilestonestimeframe"></a>`timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | | <a id="projectmilestonestitle"></a>`title` | [`String`](#string) | Title of the milestone. | +##### `Project.nestedEnvironments` + +Environments for this project with nested folders, can only be resolved for one project in any single request. + +Returns [`NestedEnvironmentConnection`](#nestedenvironmentconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="projectnestedenvironmentsname"></a>`name` | [`String`](#string) | Name of the environment. | +| <a id="projectnestedenvironmentssearch"></a>`search` | [`String`](#string) | Search query for environment name. | +| <a id="projectnestedenvironmentsstates"></a>`states` | [`[String!]`](#string) | States of environments that should be included in result. | +| <a id="projectnestedenvironmentstype"></a>`type` | [`String`](#string) | Search query for environment type. | + ##### `Project.networkPolicies` Network Policies of the project. @@ -17730,12 +18107,14 @@ Returns [`Requirement`](#requirement). | Name | Type | Description | | ---- | ---- | ----------- | | <a id="projectrequirementauthorusername"></a>`authorUsername` | [`[String!]`](#string) | Filter requirements by author username. | -| <a id="projectrequirementiid"></a>`iid` | [`ID`](#id) | IID of the requirement, e.g., "1". | -| <a id="projectrequirementiids"></a>`iids` | [`[ID!]`](#id) | List of IIDs of requirements, e.g., `[1, 2]`. | +| <a id="projectrequirementiid"></a>`iid` | [`ID`](#id) | IID of the requirement, for example, "1". | +| <a id="projectrequirementiids"></a>`iids` | [`[ID!]`](#id) | List of IIDs of requirements, for example, `[1, 2]`. | | <a id="projectrequirementlasttestreportstate"></a>`lastTestReportState` | [`RequirementStatusFilter`](#requirementstatusfilter) | State of latest requirement test report. | | <a id="projectrequirementsearch"></a>`search` | [`String`](#string) | Search query for requirement title. | | <a id="projectrequirementsort"></a>`sort` | [`Sort`](#sort) | List requirements by sort order. | | <a id="projectrequirementstate"></a>`state` | [`RequirementState`](#requirementstate) | Filter requirements by state. | +| <a id="projectrequirementworkitemiid"></a>`workItemIid` | [`ID`](#id) | IID of the requirement work item, for example, "1". | +| <a id="projectrequirementworkitemiids"></a>`workItemIids` | [`[ID!]`](#id) | List of IIDs of requirement work items, for example, `[1, 2]`. | ##### `Project.requirements` @@ -17752,12 +18131,37 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="projectrequirementsauthorusername"></a>`authorUsername` | [`[String!]`](#string) | Filter requirements by author username. | -| <a id="projectrequirementsiid"></a>`iid` | [`ID`](#id) | IID of the requirement, e.g., "1". | -| <a id="projectrequirementsiids"></a>`iids` | [`[ID!]`](#id) | List of IIDs of requirements, e.g., `[1, 2]`. | +| <a id="projectrequirementsiid"></a>`iid` | [`ID`](#id) | IID of the requirement, for example, "1". | +| <a id="projectrequirementsiids"></a>`iids` | [`[ID!]`](#id) | List of IIDs of requirements, for example, `[1, 2]`. | | <a id="projectrequirementslasttestreportstate"></a>`lastTestReportState` | [`RequirementStatusFilter`](#requirementstatusfilter) | State of latest requirement test report. | | <a id="projectrequirementssearch"></a>`search` | [`String`](#string) | Search query for requirement title. | | <a id="projectrequirementssort"></a>`sort` | [`Sort`](#sort) | List requirements by sort order. | | <a id="projectrequirementsstate"></a>`state` | [`RequirementState`](#requirementstate) | Filter requirements by state. | +| <a id="projectrequirementsworkitemiid"></a>`workItemIid` | [`ID`](#id) | IID of the requirement work item, for example, "1". | +| <a id="projectrequirementsworkitemiids"></a>`workItemIids` | [`[ID!]`](#id) | List of IIDs of requirement work items, for example, `[1, 2]`. | + +##### `Project.runners` + +Find runners visible to the current user. + +Returns [`CiRunnerConnection`](#cirunnerconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="projectrunnersactive"></a>`active` **{warning-solid}** | [`Boolean`](#boolean) | **Deprecated** in 14.8. This was renamed. Use: `paused`. | +| <a id="projectrunnerspaused"></a>`paused` | [`Boolean`](#boolean) | Filter runners by `paused` (true) or `active` (false) status. | +| <a id="projectrunnerssearch"></a>`search` | [`String`](#string) | Filter by full token or partial text in description field. | +| <a id="projectrunnerssort"></a>`sort` | [`CiRunnerSort`](#cirunnersort) | Sort order of results. | +| <a id="projectrunnersstatus"></a>`status` | [`CiRunnerStatus`](#cirunnerstatus) | Filter runners by status. | +| <a id="projectrunnerstaglist"></a>`tagList` | [`[String!]`](#string) | Filter by tags associated with the runner (comma-separated or array). | +| <a id="projectrunnerstype"></a>`type` | [`CiRunnerType`](#cirunnertype) | Filter runners by type. | +| <a id="projectrunnersupgradestatus"></a>`upgradeStatus` | [`CiRunnerUpgradeStatus`](#cirunnerupgradestatus) | Filter by upgrade status. | ##### `Project.scanExecutionPolicies` @@ -18086,6 +18490,7 @@ Returns [`UserMergeRequestInteraction`](#usermergerequestinteraction). | <a id="projectpermissionsreadcommitstatus"></a>`readCommitStatus` | [`Boolean!`](#boolean) | Indicates the user can perform `read_commit_status` on this resource. | | <a id="projectpermissionsreadcycleanalytics"></a>`readCycleAnalytics` | [`Boolean!`](#boolean) | Indicates the user can perform `read_cycle_analytics` on this resource. | | <a id="projectpermissionsreaddesign"></a>`readDesign` | [`Boolean!`](#boolean) | Indicates the user can perform `read_design` on this resource. | +| <a id="projectpermissionsreadenvironment"></a>`readEnvironment` | [`Boolean!`](#boolean) | Indicates the user can perform `read_environment` on this resource. | | <a id="projectpermissionsreadmergerequest"></a>`readMergeRequest` | [`Boolean!`](#boolean) | Indicates the user can perform `read_merge_request` on this resource. | | <a id="projectpermissionsreadpagescontent"></a>`readPagesContent` | [`Boolean!`](#boolean) | Indicates the user can perform `read_pages_content` on this resource. | | <a id="projectpermissionsreadproject"></a>`readProject` | [`Boolean!`](#boolean) | Indicates the user can perform `read_project` on this resource. | @@ -18509,6 +18914,7 @@ Represents a requirement. | <a id="requirementtitlehtml"></a>`titleHtml` | [`String`](#string) | GitLab Flavored Markdown rendering of `title`. | | <a id="requirementupdatedat"></a>`updatedAt` | [`Time!`](#time) | Timestamp of when the requirement was last updated. | | <a id="requirementuserpermissions"></a>`userPermissions` | [`RequirementPermissions!`](#requirementpermissions) | Permissions for the current user on the resource. | +| <a id="requirementworkitemiid"></a>`workItemIid` | [`ID!`](#id) | Work item IID of the requirement, will replace current IID as identifier soon. | #### Fields with arguments @@ -19075,6 +19481,20 @@ Represents the Geo sync and verification state of a snippet repository. | <a id="snippetrepositoryregistryverificationretryat"></a>`verificationRetryAt` | [`Time`](#time) | Timestamp after which the SnippetRepositoryRegistry is reverified. | | <a id="snippetrepositoryregistryverifiedat"></a>`verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the SnippetRepositoryRegistry. | +### `SshSignature` + +SSH signature for a signed commit. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="sshsignaturecommitsha"></a>`commitSha` | [`String`](#string) | SHA of the associated commit. | +| <a id="sshsignaturekey"></a>`key` | [`Key`](#key) | SSH key used for the signature. | +| <a id="sshsignatureproject"></a>`project` | [`Project`](#project) | Project of the associated commit. | +| <a id="sshsignatureuser"></a>`user` | [`UserCore`](#usercore) | User associated with the key. | +| <a id="sshsignatureverificationstatus"></a>`verificationStatus` | [`VerificationStatus`](#verificationstatus) | Indicates verification status of the associated key or certificate. | + ### `StatusAction` #### Fields @@ -20454,6 +20874,17 @@ Represents a description widget. | <a id="workitemwidgetdescriptionlasteditedby"></a>`lastEditedBy` | [`UserCore`](#usercore) | User that made the last edit to the work item's description. | | <a id="workitemwidgetdescriptiontype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | +### `WorkItemWidgetHealthStatus` + +Represents a health status widget. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgethealthstatushealthstatus"></a>`healthStatus` | [`HealthStatus`](#healthstatus) | Health status of the work item. | +| <a id="workitemwidgethealthstatustype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | + ### `WorkItemWidgetHierarchy` Represents a hierarchy widget. @@ -20463,6 +20894,7 @@ Represents a hierarchy widget. | Name | Type | Description | | ---- | ---- | ----------- | | <a id="workitemwidgethierarchychildren"></a>`children` | [`WorkItemConnection`](#workitemconnection) | Child work items. (see [Connections](#connections)) | +| <a id="workitemwidgethierarchyhaschildren"></a>`hasChildren` | [`Boolean!`](#boolean) | Indicates if the work item has children. | | <a id="workitemwidgethierarchyparent"></a>`parent` | [`WorkItem`](#workitem) | Parent work item. | | <a id="workitemwidgethierarchytype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | @@ -20500,6 +20932,45 @@ Represents a milestone widget. | <a id="workitemwidgetmilestonemilestone"></a>`milestone` | [`Milestone`](#milestone) | Milestone of the work item. | | <a id="workitemwidgetmilestonetype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | +### `WorkItemWidgetNotes` + +Represents a notes widget. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgetnotestype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | + +#### Fields with arguments + +##### `WorkItemWidgetNotes.discussions` + +Notes on this work item. + +Returns [`DiscussionConnection`](#discussionconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgetnotesdiscussionsfilter"></a>`filter` | [`NotesFilterType`](#notesfiltertype) | Type of notes collection: ALL_NOTES, ONLY_COMMENTS, ONLY_ACTIVITY. | + +### `WorkItemWidgetProgress` + +Represents a progress widget. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgetprogressprogress"></a>`progress` | [`Int`](#int) | Progress of the work item. | +| <a id="workitemwidgetprogresstype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | + ### `WorkItemWidgetStartAndDueDate` Represents a start and due date widget. @@ -20780,6 +21251,15 @@ Values for YAML processor result. | <a id="ciconfigstatusinvalid"></a>`INVALID` | Configuration file is not valid. | | <a id="ciconfigstatusvalid"></a>`VALID` | Configuration file is valid. | +### `CiFreezePeriodStatus` + +Deploy freeze period status. + +| Value | Description | +| ----- | ----------- | +| <a id="cifreezeperiodstatusactive"></a>`ACTIVE` | Freeze period is active. | +| <a id="cifreezeperiodstatusinactive"></a>`INACTIVE` | Freeze period is inactive. | + ### `CiJobKind` | Value | Description | @@ -20810,6 +21290,13 @@ Values for YAML processor result. | <a id="cirunneraccesslevelnot_protected"></a>`NOT_PROTECTED` | A runner that is not protected. | | <a id="cirunneraccesslevelref_protected"></a>`REF_PROTECTED` | A runner that is ref protected. | +### `CiRunnerJobExecutionStatus` + +| Value | Description | +| ----- | ----------- | +| <a id="cirunnerjobexecutionstatusidle"></a>`IDLE` **{warning-solid}** | **Introduced** in 15.7. This feature is in Alpha. It can be changed or removed at any time. Runner is idle. | +| <a id="cirunnerjobexecutionstatusrunning"></a>`RUNNING` **{warning-solid}** | **Introduced** in 15.7. This feature is in Alpha. It can be changed or removed at any time. Runner is executing jobs. | + ### `CiRunnerMembershipFilter` Values for filtering runners in namespaces. The previous type name `RunnerMembershipFilter` was deprecated in 15.4. @@ -21082,6 +21569,7 @@ Scan method to be used by the scanner. | Value | Description | | ----- | ----------- | +| <a id="dastscanmethodtypegraphql"></a>`GRAPHQL` | GraphQL scan method. | | <a id="dastscanmethodtypehar"></a>`HAR` | HAR scan method. | | <a id="dastscanmethodtypeopenapi"></a>`OPENAPI` | OpenAPI scan method. | | <a id="dastscanmethodtypepostman_collection"></a>`POSTMAN_COLLECTION` | Postman scan method. | @@ -21557,6 +22045,7 @@ Issue type. | ----- | ----------- | | <a id="issuetypeincident"></a>`INCIDENT` | Incident issue type. | | <a id="issuetypeissue"></a>`ISSUE` | Issue issue type. | +| <a id="issuetypekey_result"></a>`KEY_RESULT` **{warning-solid}** | **Introduced** in 15.7. This feature is in Alpha. It can be changed or removed at any time. Key Result issue type. Available only when feature flag `okrs_mvc` is enabled. | | <a id="issuetypeobjective"></a>`OBJECTIVE` **{warning-solid}** | **Introduced** in 15.6. This feature is in Alpha. It can be changed or removed at any time. Objective issue type. Available only when feature flag `okrs_mvc` is enabled. | | <a id="issuetyperequirement"></a>`REQUIREMENT` | Requirement issue type. | | <a id="issuetypetask"></a>`TASK` **{warning-solid}** | **Introduced** in 15.2. This feature is in Alpha. It can be changed or removed at any time. Task issue type. | @@ -21631,6 +22120,7 @@ Iteration ID wildcard values. | <a id="jobartifactfiletypenetwork_referee"></a>`NETWORK_REFEREE` | NETWORK REFEREE job artifact file type. | | <a id="jobartifactfiletypeperformance"></a>`PERFORMANCE` | PERFORMANCE job artifact file type. | | <a id="jobartifactfiletyperequirements"></a>`REQUIREMENTS` | REQUIREMENTS job artifact file type. | +| <a id="jobartifactfiletyperequirements_v2"></a>`REQUIREMENTS_V2` | REQUIREMENTS V2 job artifact file type. | | <a id="jobartifactfiletypesast"></a>`SAST` | SAST job artifact file type. | | <a id="jobartifactfiletypesecret_detection"></a>`SECRET_DETECTION` | SECRET DETECTION job artifact file type. | | <a id="jobartifactfiletypeterraform"></a>`TERRAFORM` | TERRAFORM job artifact file type. | @@ -21852,6 +22342,16 @@ Kind of the network policy. | <a id="networkpolicykindciliumnetworkpolicy"></a>`CiliumNetworkPolicy` | Policy kind of Cilium Network Policy. | | <a id="networkpolicykindnetworkpolicy"></a>`NetworkPolicy` | Policy kind of Network Policy. | +### `NotesFilterType` + +Work item notes collection type. + +| Value | Description | +| ----- | ----------- | +| <a id="notesfiltertypeall_notes"></a>`ALL_NOTES` | Show all activity. | +| <a id="notesfiltertypeonly_activity"></a>`ONLY_ACTIVITY` | Show history only. | +| <a id="notesfiltertypeonly_comments"></a>`ONLY_COMMENTS` | Show comments only. | + ### `OncallRotationUnitEnum` Rotation length unit of an on-call rotation. @@ -22193,7 +22693,6 @@ State of a Sentry error. | <a id="servicetypeemails_on_push_service"></a>`EMAILS_ON_PUSH_SERVICE` | EmailsOnPushService type. | | <a id="servicetypeewm_service"></a>`EWM_SERVICE` | EwmService type. | | <a id="servicetypeexternal_wiki_service"></a>`EXTERNAL_WIKI_SERVICE` | ExternalWikiService type. | -| <a id="servicetypeflowdock_service"></a>`FLOWDOCK_SERVICE` | FlowdockService type. | | <a id="servicetypegithub_service"></a>`GITHUB_SERVICE` | GithubService type. | | <a id="servicetypegitlab_slack_application_service"></a>`GITLAB_SLACK_APPLICATION_SERVICE` | GitlabSlackApplicationService type (Gitlab.com only). | | <a id="servicetypehangouts_chat_service"></a>`HANGOUTS_CHAT_SERVICE` | HangoutsChatService type. | @@ -22322,7 +22821,8 @@ Category of error. | <a id="todoactionenumassigned"></a>`assigned` | User was assigned. | | <a id="todoactionenumbuild_failed"></a>`build_failed` | Build triggered by the user failed. | | <a id="todoactionenumdirectly_addressed"></a>`directly_addressed` | User was directly addressed. | -| <a id="todoactionenummarked"></a>`marked` | User added a TODO. | +| <a id="todoactionenummarked"></a>`marked` | User added a to-do item. | +| <a id="todoactionenummember_access_requested"></a>`member_access_requested` | Group access requested from the user. | | <a id="todoactionenummentioned"></a>`mentioned` | User was mentioned. | | <a id="todoactionenummerge_train_removed"></a>`merge_train_removed` | Merge request authored by the user was removed from the merge train. | | <a id="todoactionenumreview_requested"></a>`review_requested` | Review was requested from the user. | @@ -22420,6 +22920,8 @@ Name of the feature that the callout is for. | <a id="usercalloutfeaturenameenumunfinished_tag_cleanup_callout"></a>`UNFINISHED_TAG_CLEANUP_CALLOUT` | Callout feature name for unfinished_tag_cleanup_callout. | | <a id="usercalloutfeaturenameenumuser_reached_limit_free_plan_alert"></a>`USER_REACHED_LIMIT_FREE_PLAN_ALERT` | Callout feature name for user_reached_limit_free_plan_alert. | | <a id="usercalloutfeaturenameenumverification_reminder"></a>`VERIFICATION_REMINDER` | Callout feature name for verification_reminder. | +| <a id="usercalloutfeaturenameenumvscode_web_ide"></a>`VSCODE_WEB_IDE` | Callout feature name for vscode_web_ide. | +| <a id="usercalloutfeaturenameenumvscode_web_ide_callout"></a>`VSCODE_WEB_IDE_CALLOUT` | Callout feature name for vscode_web_ide_callout. | | <a id="usercalloutfeaturenameenumweb_ide_alert_dismissed"></a>`WEB_IDE_ALERT_DISMISSED` | Callout feature name for web_ide_alert_dismissed. | | <a id="usercalloutfeaturenameenumweb_ide_ci_environments_guidance"></a>`WEB_IDE_CI_ENVIRONMENTS_GUIDANCE` | Callout feature name for web_ide_ci_environments_guidance. | @@ -22639,10 +23141,13 @@ Type of a work item widget. | ----- | ----------- | | <a id="workitemwidgettypeassignees"></a>`ASSIGNEES` | Assignees widget. | | <a id="workitemwidgettypedescription"></a>`DESCRIPTION` | Description widget. | +| <a id="workitemwidgettypehealth_status"></a>`HEALTH_STATUS` | Health Status widget. | | <a id="workitemwidgettypehierarchy"></a>`HIERARCHY` | Hierarchy widget. | | <a id="workitemwidgettypeiteration"></a>`ITERATION` | Iteration widget. | | <a id="workitemwidgettypelabels"></a>`LABELS` | Labels widget. | | <a id="workitemwidgettypemilestone"></a>`MILESTONE` | Milestone widget. | +| <a id="workitemwidgettypenotes"></a>`NOTES` | Notes widget. | +| <a id="workitemwidgettypeprogress"></a>`PROGRESS` | Progress widget. | | <a id="workitemwidgettypestart_and_due_date"></a>`START_AND_DUE_DATE` | Start And Due Date widget. | | <a id="workitemwidgettypestatus"></a>`STATUS` | Status widget. | | <a id="workitemwidgettypeweight"></a>`WEIGHT` | Weight widget. | @@ -22658,6 +23163,12 @@ each kind of object. For more information, read about [Scalar Types](https://graphql.org/learn/schema/#scalar-types) on `graphql.org`. +### `AlertManagementAlertID` + +A `AlertManagementAlertID` is a global ID. It is encoded as a string. + +An example `AlertManagementAlertID` is: `"gid://gitlab/AlertManagement::Alert/1"`. + ### `AlertManagementHttpIntegrationID` A `AlertManagementHttpIntegrationID` is a global ID. It is encoded as a string. @@ -23420,6 +23931,7 @@ Implementations: - [`CiInstanceVariable`](#ciinstancevariable) - [`CiManualVariable`](#cimanualvariable) - [`CiProjectVariable`](#ciprojectvariable) +- [`PipelineScheduleVariable`](#pipelineschedulevariable) ##### Fields @@ -23438,6 +23950,7 @@ Represents signing information for a commit. Implementations: - [`GpgSignature`](#gpgsignature) +- [`SshSignature`](#sshsignature) - [`X509Signature`](#x509signature) ##### Fields @@ -23933,10 +24446,13 @@ Implementations: - [`WorkItemWidgetAssignees`](#workitemwidgetassignees) - [`WorkItemWidgetDescription`](#workitemwidgetdescription) +- [`WorkItemWidgetHealthStatus`](#workitemwidgethealthstatus) - [`WorkItemWidgetHierarchy`](#workitemwidgethierarchy) - [`WorkItemWidgetIteration`](#workitemwidgetiteration) - [`WorkItemWidgetLabels`](#workitemwidgetlabels) - [`WorkItemWidgetMilestone`](#workitemwidgetmilestone) +- [`WorkItemWidgetNotes`](#workitemwidgetnotes) +- [`WorkItemWidgetProgress`](#workitemwidgetprogress) - [`WorkItemWidgetStartAndDueDate`](#workitemwidgetstartandduedate) - [`WorkItemWidgetStatus`](#workitemwidgetstatus) - [`WorkItemWidgetWeight`](#workitemwidgetweight) @@ -24199,6 +24715,7 @@ Represents an escalation rule. | <a id="negatedboardissueinputassigneeusername"></a>`assigneeUsername` | [`[String]`](#string) | Filter by assignee username. | | <a id="negatedboardissueinputauthorusername"></a>`authorUsername` | [`String`](#string) | Filter by author username. | | <a id="negatedboardissueinputepicid"></a>`epicId` | [`EpicID`](#epicid) | Filter by epic ID. Incompatible with epicWildcardId. | +| <a id="negatedboardissueinputhealthstatusfilter"></a>`healthStatusFilter` | [`HealthStatus`](#healthstatus) | Health status not applied to the issue. Includes issues where health status is not set. | | <a id="negatedboardissueinputiids"></a>`iids` | [`[String!]`](#string) | List of IIDs of issues. For example `["1", "2"]`. | | <a id="negatedboardissueinputiterationid"></a>`iterationId` | [`[IterationID!]`](#iterationid) | Filter by a list of iteration IDs. Incompatible with iterationWildcardId. | | <a id="negatedboardissueinputiterationtitle"></a>`iterationTitle` | [`String`](#string) | Filter by iteration title. | @@ -24241,6 +24758,7 @@ Represents an escalation rule. | <a id="negatedissuefilterinputassigneeusernames"></a>`assigneeUsernames` | [`[String!]`](#string) | Usernames of users not assigned to the issue. | | <a id="negatedissuefilterinputauthorusername"></a>`authorUsername` | [`String`](#string) | Username of a user who didn't author the issue. | | <a id="negatedissuefilterinputepicid"></a>`epicId` | [`String`](#string) | ID of an epic not associated with the issues. | +| <a id="negatedissuefilterinputhealthstatusfilter"></a>`healthStatusFilter` | [`HealthStatus`](#healthstatus) | Health status not applied to the issue. Includes issues where health status is not set. | | <a id="negatedissuefilterinputiids"></a>`iids` | [`[String!]`](#string) | List of IIDs of issues to exclude. For example, `[1, 2]`. | | <a id="negatedissuefilterinputiterationid"></a>`iterationId` | [`[ID!]`](#id) | List of iteration Global IDs not applied to the issue. | | <a id="negatedissuefilterinputiterationwildcardid"></a>`iterationWildcardId` | [`IterationWildcardId`](#iterationwildcardid) | Filter by negated iteration ID wildcard. | @@ -24297,6 +24815,18 @@ The rotation user and color palette. | <a id="oncalluserinputtypecolorweight"></a>`colorWeight` | [`DataVisualizationWeightEnum`](#datavisualizationweightenum) | Color weight to assign to for the on-call user. To view on-call schedules in GitLab, do not provide a value below 500. A value between 500 and 950 ensures sufficient contrast. | | <a id="oncalluserinputtypeusername"></a>`username` | [`String!`](#string) | Username of the user to participate in the on-call rotation. For example, `"user_one"`. | +### `PipelineScheduleVariableInput` + +Attributes for the pipeline schedule variable. + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="pipelineschedulevariableinputkey"></a>`key` | [`String!`](#string) | Name of the variable. | +| <a id="pipelineschedulevariableinputvalue"></a>`value` | [`String!`](#string) | Value of the variable. | +| <a id="pipelineschedulevariableinputvariabletype"></a>`variableType` | [`CiVariableType!`](#civariabletype) | Type of the variable. | + ### `ReleaseAssetLinkInput` Fields that are available when modifying a release asset link. @@ -24502,6 +25032,14 @@ A time-frame defined as a closed inclusive range of two dates. | ---- | ---- | ----------- | | <a id="workitemwidgetdescriptioninputdescription"></a>`description` | [`String!`](#string) | Description of the work item. | +### `WorkItemWidgetHealthStatusInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgethealthstatusinputhealthstatus"></a>`healthStatus` | [`HealthStatus`](#healthstatus) | Health status to be assigned to the work item. | + ### `WorkItemWidgetHierarchyCreateInput` #### Arguments @@ -24544,6 +25082,14 @@ A time-frame defined as a closed inclusive range of two dates. | ---- | ---- | ----------- | | <a id="workitemwidgetmilestoneinputmilestoneid"></a>`milestoneId` | [`MilestoneID`](#milestoneid) | Milestone to assign to the work item. | +### `WorkItemWidgetProgressInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgetprogressinputprogress"></a>`progress` | [`Int!`](#int) | Progress of the work item. | + ### `WorkItemWidgetStartAndDueDateUpdateInput` #### Arguments diff --git a/doc/api/graphql/users_example.md b/doc/api/graphql/users_example.md index 96bbeeb2d06..9d223f9e618 100644 --- a/doc/api/graphql/users_example.md +++ b/doc/api/graphql/users_example.md @@ -20,13 +20,13 @@ The query includes: - [`pageInfo`](#pageinfo) - [`nodes`](#nodes) -## pageInfo +## `pageInfo` This contains the data needed to implement pagination. GitLab uses cursor-based [pagination](getting_started.md#pagination). For more information, see [Pagination](https://graphql.org/learn/pagination/) in the GraphQL documentation. -## nodes +## `nodes` In a GraphQL query, `nodes` is used to represent a collection of [`nodes` on a graph](https://en.wikipedia.org/wiki/Vertex_(graph_theory)). In this case, the collection of nodes is a collection of `User` objects. For each one, diff --git a/doc/api/group_badges.md b/doc/api/group_badges.md index cc99c137a47..14146745d86 100644 --- a/doc/api/group_badges.md +++ b/doc/api/group_badges.md @@ -15,6 +15,8 @@ Badges support placeholders that are replaced in real time in both the link and <!-- vale gitlab.Spelling = NO --> - **%{project_path}**: replaced by the project path. +- **%{project_title}**: replaced by the project title. +- **%{project_name}**: replaced by the project name. - **%{project_id}**: replaced by the project ID. - **%{default_branch}**: replaced by the project default branch. - **%{commit_sha}**: replaced by the last project's commit SHA. diff --git a/doc/api/group_import_export.md b/doc/api/group_import_export.md index 1efed80699b..989b7a66285 100644 --- a/doc/api/group_import_export.md +++ b/doc/api/group_import_export.md @@ -102,7 +102,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ NOTE: The maximum import file size can be set by the Administrator, default is `0` (unlimited). -As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](settings.md#change-application-settings) or the [Admin Area](../user/admin_area/settings/account_and_limit_settings.md). Default [modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50MB to 0 in GitLab 13.8. +As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](settings.md#change-application-settings) or the [Admin Area](../user/admin_area/settings/account_and_limit_settings.md). Default [modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50 MB to 0 in GitLab 13.8. ## Important notes @@ -111,6 +111,6 @@ Note the following: - To preserve group-level relationships from imported projects, run Group Import/Export first, to allow project imports into the desired group structure. - Imported groups are given a `private` visibility level, unless imported into a parent group. -- If imported into a parent group, subgroups will inherit a similar level of visibility, unless otherwise restricted. +- If imported into a parent group, subgroups inherit a similar level of visibility, unless otherwise restricted. - To preserve the member list and their respective permissions on imported groups, review the users in these groups. Make sure these users exist before importing the desired groups. diff --git a/doc/api/group_wikis.md b/doc/api/group_wikis.md index cba64269942..d5fe7825dc6 100644 --- a/doc/api/group_wikis.md +++ b/doc/api/group_wikis.md @@ -131,13 +131,13 @@ Update an existing wiki page. At least one parameter is required to update the w PUT /groups/:id/wikis/:slug ``` -| Attribute | Type | Required | Description | -| --------------- | ------- | --------------------------------- | ------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) | -| `content` | string | yes if `title` is not provided | The content of the wiki page | -| `title` | string | yes if `content` is not provided | The title of the wiki page | -| `format` | string | no | The format of the wiki page. Available formats are: `markdown` (default), `rdoc`, `asciidoc` and `org` | -| `slug` | string | yes | URL encoded slug (a unique string) of the wiki page. Ex. dir%2Fpage_name | +| Attribute | Type | Required | Description | +|-----------|----------------|----------------------------------|-------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](index.md#namespaced-path-encoding). | +| `content` | string | yes if `title` is not provided | The content of the wiki page. | +| `title` | string | yes if `content` is not provided | The title of the wiki page. | +| `format` | string | no | The format of the wiki page. Available formats are `markdown` (default), `rdoc`, `asciidoc`, and `org`. | +| `slug` | string | yes | URL encoded slug (a unique string) of the wiki page. For example: `dir%2Fpage_name`. | ```shell curl --request PUT --data "format=rdoc&content=documentation&title=Docs" \ diff --git a/doc/api/groups.md b/doc/api/groups.md index cba54648705..d017876b9c2 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -30,7 +30,7 @@ Parameters: | `statistics` | boolean | no | Include group statistics (administrators only) | | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (administrators only) | | `owned` | boolean | no | Limit to groups explicitly owned by the current user | -| `min_access_level` | integer | no | Limit to groups where current user has at least this [access level](members.md#valid-access-levels) | +| `min_access_level` | integer | no | Limit to groups where current user has at least this [role (`access_level`)](members.md#roles) | | `top_level_only` | boolean | no | Limit to top level groups, excluding all subgroups | ```plaintext @@ -148,7 +148,7 @@ Parameters: | `statistics` | boolean | no | Include group statistics (administrators only) | | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (administrators only) | | `owned` | boolean | no | Limit to groups explicitly owned by the current user | -| `min_access_level` | integer | no | Limit to groups where current user has at least this [access level](members.md#valid-access-levels) | +| `min_access_level` | integer | no | Limit to groups where current user has at least this [role (`access_level`)](members.md#roles) | ```plaintext GET /groups/:id/subgroups @@ -206,7 +206,7 @@ Parameters: | `statistics` | boolean | no | Include group statistics (administrators only) | | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (administrators only) | | `owned` | boolean | no | Limit to groups explicitly owned by the current user | -| `min_access_level` | integer | no | Limit to groups where current user has at least this [access level](members.md#valid-access-levels) | +| `min_access_level` | integer | no | Limit to groups where current user has at least this [role (`access_level`)](members.md#roles) | ```plaintext GET /groups/:id/descendant_groups @@ -294,7 +294,7 @@ Parameters: | `with_merge_requests_enabled` | boolean | no | Limit by projects with merge requests feature enabled. Default is `false` | | `with_shared` | boolean | no | Include projects shared to this group. Default is `true` | | `include_subgroups` | boolean | no | Include projects in subgroups of this group. Default is `false` | -| `min_access_level` | integer | no | Limit to projects where current user has at least this [access level](members.md#valid-access-levels) | +| `min_access_level` | integer | no | Limit to projects where current user has at least this [role (`access_level`)](members.md#roles) | | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (administrators only) | | `with_security_reports` **(ULTIMATE)** | boolean | no | Return only projects that have security reports artifacts present in any of their builds. This means "projects with security reports enabled". Default is `false` | @@ -374,7 +374,7 @@ Parameters: | `starred` | boolean | no | Limit by projects starred by the current user | | `with_issues_enabled` | boolean | no | Limit by projects with issues feature enabled. Default is `false` | | `with_merge_requests_enabled` | boolean | no | Limit by projects with merge requests feature enabled. Default is `false` | -| `min_access_level` | integer | no | Limit to projects where current user has at least this [access level](members.md#valid-access-levels) | +| `min_access_level` | integer | no | Limit to projects where current user has at least this [role (`access_level`)](members.md#roles) | | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (administrators only) | Example response: @@ -487,7 +487,7 @@ Example response: ## Details of a group -> The `membership_lock` field was [added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82271) in GitLab 14.10. +> The `membership_lock` field was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82271) in GitLab 14.10. Get all details of a group. This endpoint can be accessed without authentication if the group is publicly accessible. In case the user that requests is an administrator @@ -1158,7 +1158,7 @@ Parameters: The response is `202 Accepted` if the user has authorization. NOTE: -A GitLab.com group can't be removed if it is linked to a subscription. To remove such a group, first [link the subscription](../subscriptions/index.md#change-the-linked-namespace) with a different group. +A GitLab.com group can't be removed if it is linked to a subscription. To remove such a group, first [link the subscription](../subscriptions/gitlab_com/index.md#change-the-linked-namespace) with a different group. ## Restore group marked for deletion **(PREMIUM)** @@ -1361,25 +1361,25 @@ PUT /groups/:id/hooks/:hook_id | Attribute | Type | Required | Description | | ---------------------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) | -| `hook_id` | integer | yes | The ID of the group hook | -| `url` | string | yes | The hook URL | -| `push_events` | boolean | no | Trigger hook on push events | +| `id` | integer or string | yes | The ID or [URL-encoded path of the group](index.md#namespaced-path-encoding). | +| `hook_id` | integer | yes | The ID of the group hook. | +| `url` | string | yes | The hook URL. | +| `push_events` | boolean | no | Trigger hook on push events. | | `push_events_branch_filter` | string | No | Trigger hook on push events for matching branches only. | -| `issues_events` | boolean | no | Trigger hook on issues events | -| `confidential_issues_events` | boolean | no | Trigger hook on confidential issues events | -| `merge_requests_events` | boolean | no | Trigger hook on merge requests events | -| `tag_push_events` | boolean | no | Trigger hook on tag push events | -| `note_events` | boolean | no | Trigger hook on note events | -| `confidential_note_events` | boolean | no | Trigger hook on confidential note events | -| `job_events` | boolean | no | Trigger hook on job events | -| `pipeline_events` | boolean | no | Trigger hook on pipeline events | -| `wiki_page_events` | boolean | no | Trigger hook on wiki page events | -| `deployment_events` | boolean | no | Trigger hook on deployment events | -| `releases_events` | boolean | no | Trigger hook on release events | -| `subgroup_events` | boolean | no | Trigger hook on subgroup events | -| `enable_ssl_verification` | boolean | no | Do SSL verification when triggering the hook | -| `token` | string | no | Secret token to validate received payloads; not returned in the response | +| `issues_events` | boolean | no | Trigger hook on issues events. | +| `confidential_issues_events` | boolean | no | Trigger hook on confidential issues events. | +| `merge_requests_events` | boolean | no | Trigger hook on merge requests events. | +| `tag_push_events` | boolean | no | Trigger hook on tag push events. | +| `note_events` | boolean | no | Trigger hook on note events. | +| `confidential_note_events` | boolean | no | Trigger hook on confidential note events. | +| `job_events` | boolean | no | Trigger hook on job events. | +| `pipeline_events` | boolean | no | Trigger hook on pipeline events. | +| `wiki_page_events` | boolean | no | Trigger hook on wiki page events. | +| `deployment_events` | boolean | no | Trigger hook on deployment events. | +| `releases_events` | boolean | no | Trigger hook on release events. | +| `subgroup_events` | boolean | no | Trigger hook on subgroup events. | +| `enable_ssl_verification` | boolean | no | Do SSL verification when triggering the hook. | +| `token` | string | no | Secret token to validate received payloads. Not returned in the response. When you change the webhook URL, the secret token is reset and not retained. | ### Delete group hook @@ -1444,7 +1444,7 @@ POST /groups/:id/ldap_group_links | `id` | integer/string | yes | The ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) | | `cn` | string | no | The CN of an LDAP group | | `filter` | string | no | The LDAP filter for the group | -| `group_access` | integer | yes | [Access level](members.md#valid-access-levels) for members of the LDAP group | +| `group_access` | integer | yes | [Role (`access_level`)](members.md#roles) for members of the LDAP group | | `provider` | string | yes | LDAP provider for the LDAP group link | NOTE: @@ -1519,7 +1519,7 @@ If successful, returns [`200`](index.md#status-codes) and the following response | Attribute | Type | Description | |:-------------------|:--------|:-----------------------------------------------------------------------------| | `[].name` | string | Name of the SAML group | -| `[].access_level` | integer | [Access level](members.md#valid-access-levels) for members of the SAML group. The attribute had a string type from GitLab 15.3.0 to GitLab 15.3.3 | +| `[].access_level` | integer | [Role (`access_level`)](members.md#roles) for members of the SAML group. The attribute had a string type from GitLab 15.3.0 to GitLab 15.3.3 | Example request: @@ -1562,7 +1562,7 @@ If successful, returns [`200`](index.md#status-codes) and the following response | Attribute | Type | Description | |:---------------|:--------|:-----------------------------------------------------------------------------| | `name` | string | Name of the SAML group | -| `access_level` | integer | [Access level](members.md#valid-access-levels) for members of the SAML group. The attribute had a string type from GitLab 15.3.0 to GitLab 15.3.3 | +| `access_level` | integer | [Role (`access_level`)](members.md#roles) for members of the SAML group. The attribute had a string type from GitLab 15.3.0 to GitLab 15.3.3 | Example request: @@ -1593,14 +1593,14 @@ Supported attributes: |:-------------------|:---------------|:---------|:-----------------------------------------------------------------------------| | `id` | integer or string | yes | ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) | | `saml_group_name` | string | yes | Name of a SAML group | -| `access_level` | integer | yes | [Access level](members.md#valid-access-levels) for members of the SAML group | +| `access_level` | integer | yes | [Role (`access_level`)](members.md#roles) for members of the SAML group | If successful, returns [`201`](index.md#status-codes) and the following response attributes: | Attribute | Type | Description | |:---------------|:--------|:-----------------------------------------------------------------------------| | `name` | string | Name of the SAML group | -| `access_level` | integer | [Access level](members.md#valid-access-levels) for members of the for members of the SAML group. The attribute had a string type from GitLab 15.3.0 to GitLab 15.3.3 | +| `access_level` | integer | [Role (`access_level`)](members.md#roles) for members of the for members of the SAML group. The attribute had a string type from GitLab 15.3.0 to GitLab 15.3.3 | Example request: @@ -1682,7 +1682,7 @@ POST /groups/:id/share | --------- | -------------- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) | | `group_id` | integer | yes | The ID of the group to share with | -| `group_access` | integer | yes | The [access level](members.md#valid-access-levels) to grant the group | +| `group_access` | integer | yes | The [role (`access_level`)](members.md#roles) to grant the group | | `expires_at` | string | no | Share expiration date in ISO 8601 format: 2016-09-26 | ### Delete link sharing group with another group diff --git a/doc/api/import.md b/doc/api/import.md index 78b9beb1815..7a1eb4fe8b3 100644 --- a/doc/api/import.md +++ b/doc/api/import.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Import repository from GitHub -Import your projects from GitHub to GitLab via the API. +Import your projects from GitHub to GitLab using the API. ```plaintext POST /import/github @@ -61,6 +61,16 @@ Example response: } ``` +### Import a public project through the API using a group access token + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362683) in GitLab 15.7, projects are not imported into a [bot user's](../user/group/settings/group_access_tokens.md#bot-users-for-groups) namespace in any circumstances. Projects imported into a bot user's namespace could not be deleted by users with valid tokens, which represented a security risk. + +When you import a project from GitHub to GitLab through the API using a group access +token: + +- The GitLab project inherits the original project's visibility settings. As a result, the project is publicly accessible if the original project is public. +- If the `path` or `target_namespace` does not exist, the project import fails. + ## Cancel GitHub project import > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/364783) in GitLab 15.5. diff --git a/doc/api/index.md b/doc/api/index.md index cc54731de81..ef054318c5c 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -26,12 +26,6 @@ Contributions are welcome. For an introduction and basic steps, see [How to make GitLab API calls](https://www.youtube.com/watch?v=0LsMC3ZiXkA). -## SCIM API **(PREMIUM SAAS)** - -GitLab provides a [SCIM API](scim.md) that both implements -[the RFC7644 protocol](https://www.rfc-editor.org/rfc/rfc7644) and provides the -`/Users` endpoint. The base URL is `/api/scim/v2/groups/:group_path/Users/`. - ## GraphQL API A GraphQL API is available in GitLab. @@ -177,7 +171,7 @@ Read more about [GitLab as an OAuth2 provider](oauth2.md). NOTE: We recommend OAuth access tokens have an expiration. You can use the `refresh_token` parameter to refresh tokens. Integrations may need to be updated to use refresh tokens prior to -expiration, which is based on the [expires_in](https://datatracker.ietf.org/doc/html/rfc6749#appendix-A.14) +expiration, which is based on the [`expires_in`](https://datatracker.ietf.org/doc/html/rfc6749#appendix-A.14) property in the token endpoint response. See [OAuth2 token](oauth2.md) documentation for examples requesting a new access token using a refresh token. diff --git a/doc/api/integrations.md b/doc/api/integrations.md index d64c67e1402..f6ad095aad6 100644 --- a/doc/api/integrations.md +++ b/doc/api/integrations.md @@ -354,7 +354,7 @@ Parameters: | --------- | ---- | -------- | ----------- | | `webhook` | string | true | The Unify Circuit webhook. For example, `https://circuit.com/rest/v2/webhooks/incoming/...`. | | `notify_only_broken_pipelines` | boolean | false | Send notifications for broken pipelines | -| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are "all", "default", "protected", and "default_and_protected". The default value is "default" | +| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are `all`, `default`, `protected`, and `default_and_protected`. The default value is "default" | | `push_events` | boolean | false | Enable notifications for push events | | `issues_events` | boolean | false | Enable notifications for issue events | | `confidential_issues_events` | boolean | false | Enable notifications for confidential issue events | @@ -444,7 +444,7 @@ Parameters: | --------- | ---- | -------- | ----------- | | `webhook` | string | true | The Webex Teams webhook. For example, `https://api.ciscospark.com/v1/webhooks/incoming/...`. | | `notify_only_broken_pipelines` | boolean | false | Send notifications for broken pipelines | -| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are "all", "default", "protected", and "default_and_protected". The default value is "default" | +| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are `all`, `default`, `protected`, and `default_and_protected`. The default value is "default" | | `push_events` | boolean | false | Enable notifications for push events | | `issues_events` | boolean | false | Enable notifications for issue events | | `confidential_issues_events` | boolean | false | Enable notifications for confidential issue events | @@ -601,7 +601,7 @@ Parameters: | `send_from_committer_email` | boolean | false | Send from committer | | `push_events` | boolean | false | Enable notifications for push events | | `tag_push_events` | boolean | false | Enable notifications for tag push events | -| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are "all", "default", "protected", and "default_and_protected". Notifications are always fired for tag pushes. The default value is "all" | +| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are `all`, `default`, `protected`, and `default_and_protected`. Notifications are always fired for tag pushes. The default value is "all" | ### Disable Emails on Push integration @@ -755,42 +755,6 @@ Get External wiki integration settings for a project. GET /projects/:id/integrations/external-wiki ``` -## Flowdock - -Flowdock is a ChatOps application for collaboration in software engineering -companies. You can send notifications from GitLab events to Flowdock flows. -For integration instructions, see the [Flowdock documentation](https://www.flowdock.com/help/gitlab). - -### Create/Edit Flowdock integration - -Set Flowdock integration for a project. - -```plaintext -PUT /projects/:id/integrations/flowdock -``` - -Parameters: - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `token` | string | true | Flowdock Git source token | - -### Disable Flowdock integration - -Disable the Flowdock integration for a project. Integration settings are preserved. - -```plaintext -DELETE /projects/:id/integrations/flowdock -``` - -### Get Flowdock integration settings - -Get Flowdock integration settings for a project. - -```plaintext -GET /projects/:id/integrations/flowdock -``` - ## GitHub **(PREMIUM)** Code collaboration software. @@ -846,7 +810,7 @@ Parameters: | `webhook` | string | true | The Hangouts Chat webhook. For example, `https://chat.googleapis.com/v1/spaces...`. | | `notify_only_broken_pipelines` | boolean | false | Send notifications for broken pipelines | | `notify_only_default_branch` | boolean | false | DEPRECATED: This parameter has been replaced with `branches_to_be_notified` | -| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are "all", "default", "protected", and "default_and_protected". The default value is "default" | +| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are `all`, `default`, `protected`, and `default_and_protected`. The default value is "default" | | `push_events` | boolean | false | Enable notifications for push events | | `issues_events` | boolean | false | Enable notifications for issue events | | `confidential_issues_events` | boolean | false | Enable notifications for confidential issue events | @@ -1106,7 +1070,7 @@ Parameters: | --------- | ---- | -------- | ----------- | | `recipients` | string | yes | Comma-separated list of recipient email addresses | | `notify_only_broken_pipelines` | boolean | no | Notify only broken pipelines | -| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are "all", "default", "protected", and "default_and_protected. The default value is "default" | +| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are `all`, `default`, `protected`, and `default_and_protected`. The default value is "default" | | `notify_only_default_branch` | boolean | no | Send notifications only for the default branch ([introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/28271)) | | `pipeline_events` | boolean | false | Enable notifications for pipeline events | @@ -1181,7 +1145,7 @@ Parameters: | --------- | ---- | -------- | ----------- | | `api_url` | string | true | Prometheus API Base URL. For example, `http://prometheus.example.com/`. | | `google_iap_audience_client_id` | string | false | Client ID of the IAP secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com) | -| `google_iap_service_account_json` | string | false | `credentials.json` file for your service account, like { "type": "service_account", "project_id": ... } | +| `google_iap_service_account_json` | string | false | `credentials.json` file for your service account, like { `"type": "service_account", "project_id": ... }` | ### Disable Prometheus integration @@ -1294,7 +1258,7 @@ Parameters: | `channel` | string | false | Default channel to use if others are not configured | | `notify_only_broken_pipelines` | boolean | false | Send notifications for broken pipelines | | `notify_only_default_branch` | boolean | false | DEPRECATED: This parameter has been replaced with `branches_to_be_notified` | -| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are "all", "default", "protected", and "default_and_protected". The default value is "default" | +| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are `all`, `default`, `protected`, and `default_and_protected`. The default value is "default" | | `commit_events` | boolean | false | Enable notifications for commit events | | `confidential_issue_channel` | string | false | The name of the channel to receive confidential issues events notifications | | `confidential_issues_events` | boolean | false | Enable notifications for confidential issue events | @@ -1353,7 +1317,7 @@ Parameters: | `webhook` | string | true | The Microsoft Teams webhook. For example, `https://outlook.office.com/webhook/...` | | `notify_only_broken_pipelines` | boolean | false | Send notifications for broken pipelines | | `notify_only_default_branch` | boolean | false | DEPRECATED: This parameter has been replaced with `branches_to_be_notified` | -| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are "all", "default", "protected", and "default_and_protected". The default value is "default" | +| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are `all`, `default`, `protected`, and `default_and_protected`. The default value is "default" | | `push_events` | boolean | false | Enable notifications for push events | | `issues_events` | boolean | false | Enable notifications for issue events | | `confidential_issues_events` | boolean | false | Enable notifications for confidential issue events | @@ -1401,7 +1365,7 @@ Parameters: | `channel` | string | false | Default channel to use if others are not configured | | `notify_only_broken_pipelines` | boolean | false | Send notifications for broken pipelines | | `notify_only_default_branch` | boolean | false | DEPRECATED: This parameter has been replaced with `branches_to_be_notified` | -| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are "all", "default", "protected", and "default_and_protected". The default value is "default" | +| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are `all`, `default`, `protected`, and `default_and_protected`. The default value is "default" | | `push_events` | boolean | false | Enable notifications for push events | | `issues_events` | boolean | false | Enable notifications for issue events | | `confidential_issues_events` | boolean | false | Enable notifications for confidential issue events | diff --git a/doc/api/invitations.md b/doc/api/invitations.md index 94362b097af..908fa0ce890 100644 --- a/doc/api/invitations.md +++ b/doc/api/invitations.md @@ -126,7 +126,7 @@ PUT /projects/:id/invitations/:email | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project or group](index.md#namespaced-path-encoding) owned by the authenticated user. | -| `email` | string | yes | The email address to which the invitation was previously sent. | +| `email` | string | yes | The email address the invitation was previously sent to. | | `access_level` | integer | no | A valid access level (defaults: `30`, the Developer role). | | `expires_at` | string | no | A date string in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`). | diff --git a/doc/api/issue_links.md b/doc/api/issue_links.md index 253be9109c7..ce3d26f1c08 100644 --- a/doc/api/issue_links.md +++ b/doc/api/issue_links.md @@ -178,7 +178,7 @@ POST /projects/:id/issues/:issue_iid/links | `issue_iid` | integer | yes | The internal ID of a project's issue | | `target_project_id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) of a target project | | `target_issue_iid` | integer/string | yes | The internal ID of a target project's issue | -| `link_type` | string | no | The type of the relation ("relates_to", "blocks", "is_blocked_by"), defaults to "relates_to"). | +| `link_type` | string | no | The type of the relation (`relates_to`, `blocks`, `is_blocked_by`), defaults to `relates_to`). | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/4/issues/1/links?target_project_id=5&target_issue_iid=1" diff --git a/doc/api/issues.md b/doc/api/issues.md index dd5a1354a3a..94547d69064 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -2010,7 +2010,7 @@ POST /projects/:id/issues/:issue_iid/time_estimate | Attribute | Type | Required | Description | |-------------|---------|----------|------------------------------------------| -| `duration` | string | yes | The duration in human format. e.g: 3h30m | +| `duration` | string | yes | The duration in human format. e.g: `3h30m` | | `id` | integer/string | yes | The global ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | | `issue_iid` | integer | yes | The internal ID of a project's issue | @@ -2067,7 +2067,7 @@ POST /projects/:id/issues/:issue_iid/add_spent_time | Attribute | Type | Required | Description | |-------------|---------|----------|------------------------------------------| -| `duration` | string | yes | The duration in human format. e.g: 3h30m | +| `duration` | string | yes | The duration in human format. e.g: `3h30m` | | `id` | integer/string | yes | The global ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | | `issue_iid` | integer | yes | The internal ID of a project's issue | | `summary` | string | no | A summary of how the time was spent | diff --git a/doc/api/jobs.md b/doc/api/jobs.md index fc2de00c3d2..992cb70c45d 100644 --- a/doc/api/jobs.md +++ b/doc/api/jobs.md @@ -520,46 +520,39 @@ Example response: ```json { - "allowed_agents": - [ - { + "allowed_agents": [ + { + "id": 1, + "config_project": { "id": 1, - "config_project": { - "id": 1, - "description": null, - "name": "project1", - "name_with_namespace": "John Doe2 / project1", - "path": "project1", - "path_with_namespace": "namespace1/project1", - "created_at": "2021-03-26T14:51:50.579Z" - } + "description": null, + "name": "project1", + "name_with_namespace": "John Doe2 / project1", + "path": "project1", + "path_with_namespace": "namespace1/project1", + "created_at": "2022-11-16T14:51:50.579Z" } - ], + } + ], "job": { - "id": 1, - "name": "test", - "stage": "test", - "project_id": 1, - "project_name": "project1" + "id": 1 }, "pipeline": { - "id": 1, - "project_id": 1, - "sha": "b83d6e391c22777fca1ed3012fce84f633d7fed0", - "ref": "main", - "status": "pending", - "created_at": "2021-03-26T14:51:51.107Z", - "updated_at": "2021-03-26T14:51:51.107Z", - "web_url": "http://localhost/namespace1/project1/-/pipelines/1" + "id": 2 }, "project": { "id": 1, - "description": null, - "name": "project1", - "name_with_namespace": "John Doe2 / project1", - "path": "project1", - "path_with_namespace": "namespace1/project1", - "created_at": "2021-03-26T14:51:50.579Z" + "groups": [ + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + } + ] }, "user": { "id": 2, diff --git a/doc/api/keys.md b/doc/api/keys.md index 74d8bc4383f..e7bdc70017c 100644 --- a/doc/api/keys.md +++ b/doc/api/keys.md @@ -32,6 +32,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt1256k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=", "created_at": "2015-09-03T07:24:44.627Z", "expires_at": "2020-05-05T00:00:00.000Z", + "usage_type": "auth", "user": { "name": "John Smith", "username": "john_smith", @@ -101,6 +102,7 @@ Example response: "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt1016k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=", "created_at": "2019-11-14T15:11:13.222Z", "expires_at": "2020-05-05T00:00:00.000Z", + "usage_type": "auth", "user": { "id": 1, "name": "Administrator", @@ -159,6 +161,7 @@ Example response: "title": "Sample key 1", "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt1016k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=", "created_at": "2019-11-14T15:11:13.222Z", + "usage_type": "auth", "user": { "id": 1, "name": "Administrator", diff --git a/doc/api/license.md b/doc/api/license.md index 72589710590..ca9d9cf386d 100644 --- a/doc/api/license.md +++ b/doc/api/license.md @@ -97,6 +97,55 @@ Returns: - `200 OK` with response containing the licenses in JSON format. This is an empty JSON array if there are no licenses. - `403 Forbidden` if the current user in not permitted to read the licenses. +## Retrieve information about a single license + +```plaintext +GET /license/:id +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +|-----------|---------|----------|---------------------------| +| `id` | integer | yes | ID of the GitLab license. | + +Returns the following status codes: + +- `200 OK`: Response contains the licenses in JSON format. +- `404 Not Found`: The requested license doesn't exist. +- `403 Forbidden`: The current user is not permitted to read the licenses. + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/license/:id" +``` + +Example response: + +```json +{ + "id": 1, + "plan": "premium", + "created_at": "2018-02-27T23:21:58.674Z", + "starts_at": "2018-01-27", + "expires_at": "2022-01-27", + "historical_max": 300, + "maximum_user_count": 300, + "expired": false, + "overage": 200, + "user_limit": 100, + "active_users": 50, + "licensee": { + "Name": "John Doe1" + }, + "add_ons": { + "GitLab_FileLocks": 1, + "GitLab_Auditor_User": 1 + } +} +``` + ## Add a new license ```plaintext diff --git a/doc/api/members.md b/doc/api/members.md index 66729abcad8..0d6fd6aabc4 100644 --- a/doc/api/members.md +++ b/doc/api/members.md @@ -8,9 +8,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w > `created_by` field [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/28789) in GitLab 14.10. -## Valid access levels +## Roles -The access levels are defined in the `Gitlab::Access` module. Currently, these levels are recognized: +The [role](../user/permissions.md) assigned to a user or group is defined +in the `Gitlab::Access` module as `access_level`. - No access (`0`) - Minimal access (`5`) ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220203) in GitLab 13.5.) @@ -28,7 +29,8 @@ In GitLab 14.8 and earlier, projects in personal namespaces have an `access_leve The `group_saml_identity` attribute is only visible to a group owner for [SSO enabled groups](../user/group/saml_sso/index.md). -The `email` attribute is only visible for users with public emails. +The `email` attribute is only visible to group owners when the user was provisioned by the group. +Users are provisioned by the group when the account was created via [SCIM](../user/group/saml_sso/scim_setup.md) or by first sign-in with [SAML SSO for GitLab.com groups](../user/group/saml_sso/index.md). ## List all members of a group or project diff --git a/doc/api/merge_request_approvals.md b/doc/api/merge_request_approvals.md index 0476035784a..d9777b87ff2 100644 --- a/doc/api/merge_request_approvals.md +++ b/doc/api/merge_request_approvals.md @@ -84,6 +84,8 @@ Supported attributes: > - Moved to GitLab Premium in 13.9. > - Pagination support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31011) in GitLab 15.3 [with a flag](../administration/feature_flags.md) named `approval_rules_pagination`. Enabled by default. > - `applies_to_all_protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. +> - Pagination support [generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/366823) in GitLab 15.7. Feature flag `approval_rules_pagination` removed. +> - `usernames` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/) in GitLab 15.8. You can request information about a project's approval rules using the following endpoint: @@ -187,6 +189,7 @@ Supported attributes: > - Introduced in GitLab 13.7. > - `applies_to_all_protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. +> - `usernames` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/) in GitLab 1x.x. You can request information about a single project approval rules using the following endpoint: @@ -288,6 +291,7 @@ Supported attributes: > - Moved to GitLab Premium in 13.9. > - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357300) the Vulnerability-Check feature in GitLab 15.0. > - `applies_to_all_protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. +> - `usernames` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/) in GitLab 1x.x. You can create project approval rules using the following endpoint: @@ -308,6 +312,7 @@ Supported attributes: | `report_type` | string | **{dotted-circle}** No | The report type required when the rule type is `report_approver`. The supported report types are `license_scanning` and `code_coverage`. | | `rule_type` | string | **{dotted-circle}** No | The type of rule. `any_approver` is a pre-configured default rule with `approvals_required` at `0`. Other rules are `regular`. | | `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | +| `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | ```json { @@ -413,6 +418,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ > - Moved to GitLab Premium in 13.9. > - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357300) the Vulnerability-Check feature in GitLab 15.0. > - `applies_to_all_protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. +> - `usernames` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/) in GitLab 1x.x. You can update project approval rules using the following endpoint: @@ -433,7 +439,9 @@ Supported attributes: | `applies_to_all_protected_branches` | boolean | **{dotted-circle}** No | Whether the rule is applied to all protected branches. If set to `true`, the value of `protected_branch_ids` is ignored. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. | | `group_ids` | Array | **{dotted-circle}** No | The IDs of groups as approvers. | | `protected_branch_ids` | Array | **{dotted-circle}** No | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | +| `remove_hidden_groups` | boolean | **{dotted-circle}** No | Whether hidden groups should be removed. | | `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | +| `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | ```json { @@ -703,6 +711,7 @@ Supported attributes: > - Moved to GitLab Premium in 13.9. > - Pagination support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31011) in GitLab 15.3 [with a flag](../administration/feature_flags.md) named `approval_rules_pagination`. Enabled by default. +> - Pagination support [generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/366823) in GitLab 15.7. Feature flag `approval_rules_pagination` removed. You can request information about a merge request's approval rules using the following endpoint: @@ -876,6 +885,7 @@ Supported attributes: | `approval_project_rule_id` | integer | **{dotted-circle}** No | The ID of a project-level approval rule. | | `group_ids` | Array | **{dotted-circle}** No | The IDs of groups as approvers. | | `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | +| `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | **Important:** When `approval_project_rule_id` is set, the `name`, `users` and `groups` of project-level rule are copied. The `approvals_required` specified @@ -964,7 +974,9 @@ Supported attributes: | `merge_request_iid` | integer | **{check-circle}** Yes | The IID of a merge request. | | `name` | string | **{check-circle}** Yes | The name of the approval rule. | | `group_ids` | Array | **{dotted-circle}** No | The IDs of groups as approvers. | +| `remove_hidden_groups` | boolean | **{dotted-circle}** No | Whether hidden groups should be removed. | | `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | +| `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | ```json { diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index 4b4d36ec23e..5843a10ca59 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -6,8 +6,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Merge requests API **(FREE)** +> - `reference` was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20354) in GitLab 12.7. > - `draft` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63473) as a replacement for `work_in_progress` in GitLab 14.0. +> - `merged_by` was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/350534) in GitLab 14.7. > - `merge_user` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/349031) as an eventual replacement for `merged_by` in GitLab 14.7. +> - `merge_status` was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/3169#note_1162532204) in favor of `detailed_merge_status` in GitLab 15.6. Every API call to merge requests must be authenticated. @@ -45,27 +48,27 @@ Supported attributes: | `assignee_id` | integer | **{dotted-circle}** No | Returns merge requests assigned to the given user `id`. `None` returns unassigned merge requests. `Any` returns merge requests with an assignee. | | `author_id` | integer | **{dotted-circle}** No | Returns merge requests created by the given user `id`. Mutually exclusive with `author_username`. Combine with `scope=all` or `scope=assigned_to_me`. | | `author_username` | string | **{dotted-circle}** No | Returns merge requests created by the given `username`. Mutually exclusive with `author_id`. | -| `created_after` | datetime | **{dotted-circle}** No | Return merge requests created on or after the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | -| `created_before` | datetime | **{dotted-circle}** No | Return merge requests created on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | -| `deployed_after` | datetime | **{dotted-circle}** No | Return merge requests deployed after the given date/time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | -| `deployed_before` | datetime | **{dotted-circle}** No | Return merge requests deployed before the given date/time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `created_after` | datetime | **{dotted-circle}** No | Returns merge requests created on or after the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `created_before` | datetime | **{dotted-circle}** No | Returns merge requests created on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `deployed_after` | datetime | **{dotted-circle}** No | Returns merge requests deployed after the given date/time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `deployed_before` | datetime | **{dotted-circle}** No | Returns merge requests deployed before the given date/time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | | `environment` | string | **{dotted-circle}** No | Returns merge requests deployed to the given environment. | | `in` | string | **{dotted-circle}** No | Modify the scope of the `search` attribute. `title`, `description`, or a string joining them with comma. Default is `title,description`. | -| `labels` | string | **{dotted-circle}** No | Return merge requests matching a comma-separated list of labels. `None` lists all merge requests with no labels. `Any` lists all merge requests with at least one label. Predefined names are case-insensitive. | -| `milestone` | string | **{dotted-circle}** No | Return merge requests for a specific milestone. `None` returns merge requests with no milestone. `Any` returns merge requests that have an assigned milestone. | -| `my_reaction_emoji` | string | **{dotted-circle}** No | Return merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. | -| `not` | Hash | **{dotted-circle}** No | Return merge requests that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `reviewer_id`, `reviewer_username`, `my_reaction_emoji`. | -| `order_by` | string | **{dotted-circle}** No | Return requests ordered by `created_at`, `title`, or `updated_at` fields. Default is `created_at`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/331625) in GitLab 14.8.| +| `labels` | string | **{dotted-circle}** No | Returns merge requests matching a comma-separated list of labels. `None` lists all merge requests with no labels. `Any` lists all merge requests with at least one label. Predefined names are case-insensitive. | +| `milestone` | string | **{dotted-circle}** No | Returns merge requests for a specific milestone. `None` returns merge requests with no milestone. `Any` returns merge requests that have an assigned milestone. | +| `my_reaction_emoji` | string | **{dotted-circle}** No | Returns merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. | +| `not` | Hash | **{dotted-circle}** No | Returns merge requests that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `reviewer_id`, `reviewer_username`, `my_reaction_emoji`. | +| `order_by` | string | **{dotted-circle}** No | Returns requests ordered by `created_at`, `title`, or `updated_at` fields. Default is `created_at`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/331625) in GitLab 14.8.| | `reviewer_id` | integer | **{dotted-circle}** No | Returns merge requests which have the user as a [reviewer](../user/project/merge_requests/getting_started.md#reviewer) with the given user `id`. `None` returns merge requests with no reviewers. `Any` returns merge requests with any reviewer. Mutually exclusive with `reviewer_username`. | | `reviewer_username` | string | **{dotted-circle}** No | Returns merge requests which have the user as a [reviewer](../user/project/merge_requests/getting_started.md#reviewer) with the given `username`. `None` returns merge requests with no reviewers. `Any` returns merge requests with any reviewer. Mutually exclusive with `reviewer_id`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49341) in GitLab 13.8. | -| `scope` | string | **{dotted-circle}** No | Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`. Defaults to `created_by_me`. | +| `scope` | string | **{dotted-circle}** No | Returns merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`. Defaults to `created_by_me`. | | `search` | string | **{dotted-circle}** No | Search merge requests against their `title` and `description`. | -| `sort` | string | **{dotted-circle}** No | Return requests sorted in `asc` or `desc` order. Default is `desc`. | -| `source_branch` | string | **{dotted-circle}** No | Return merge requests with the given source branch. | -| `state` | string | **{dotted-circle}** No | Return all merge requests or just those that are `opened`, `closed`, `locked`, or `merged`. | -| `target_branch` | string | **{dotted-circle}** No | Return merge requests with the given target branch. | -| `updated_after` | datetime | **{dotted-circle}** No | Return merge requests updated on or after the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | -| `updated_before` | datetime | **{dotted-circle}** No | Return merge requests updated on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `sort` | string | **{dotted-circle}** No | Returns requests sorted in `asc` or `desc` order. Default is `desc`. | +| `source_branch` | string | **{dotted-circle}** No | Returns merge requests with the given source branch. | +| `state` | string | **{dotted-circle}** No | Returns all merge requests or just those that are `opened`, `closed`, `locked`, or `merged`. | +| `target_branch` | string | **{dotted-circle}** No | Returns merge requests with the given target branch. | +| `updated_after` | datetime | **{dotted-circle}** No | Returns merge requests updated on or after the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `updated_before` | datetime | **{dotted-circle}** No | Returns merge requests updated on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | | `view` | string | **{dotted-circle}** No | If `simple`, returns the `iid`, URL, title, description, and basic state of merge request. | | `with_labels_details` | boolean | **{dotted-circle}** No | If `true`, response returns more details for each label in labels field: `:name`, `:color`, `:description`, `:description_html`, `:text_color`. Default is `false`. | | `with_merge_status_recheck` | boolean | **{dotted-circle}** No | If `true`, this projection requests (but does not guarantee) that the `merge_status` field be recalculated asynchronously. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890) in GitLab 13.0. | @@ -229,7 +232,7 @@ GET /projects/:id/merge_requests?labels=bug,reproduced GET /projects/:id/merge_requests?my_reaction_emoji=star ``` -`project_id` represents the ID of the project where the MR resides. +`project_id` represents the ID of the project where the merge request resides. `project_id` always equals `target_project_id`. In the case of a merge request from the same project, @@ -248,25 +251,25 @@ Supported attributes: | `assignee_id` | integer | **{dotted-circle}** No | Returns merge requests assigned to the given user `id`. `None` returns unassigned merge requests. `Any` returns merge requests with an assignee. | | `author_id` | integer | **{dotted-circle}** No | Returns merge requests created by the given user `id`. Mutually exclusive with `author_username`. | | `author_username` | string | **{dotted-circle}** No | Returns merge requests created by the given `username`. Mutually exclusive with `author_id`. | -| `created_after` | datetime | **{dotted-circle}** No | Return merge requests created on or after the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | -| `created_before` | datetime | **{dotted-circle}** No | Return merge requests created on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `created_after` | datetime | **{dotted-circle}** No | Returns merge requests created on or after the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `created_before` | datetime | **{dotted-circle}** No | Returns merge requests created on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | | `environment` | string | **{dotted-circle}** No | Returns merge requests deployed to the given environment. | -| `iids[]` | integer array | **{dotted-circle}** No | Return the request having the given `iid`. | -| `labels` | string | **{dotted-circle}** No | Return merge requests matching a comma-separated list of labels. `None` lists all merge requests with no labels. `Any` lists all merge requests with at least one label. Predefined names are case-insensitive. | -| `milestone` | string | **{dotted-circle}** No | Return merge requests for a specific milestone. `None` returns merge requests with no milestone. `Any` returns merge requests that have an assigned milestone. | -| `my_reaction_emoji` | string | **{dotted-circle}** No | Return merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. | -| `not` | Hash | **{dotted-circle}** No | Return merge requests that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `reviewer_id`, `reviewer_username`, `my_reaction_emoji`. | -| `order_by` | string | **{dotted-circle}** No | Return requests ordered by `created_at`, `title` or `updated_at` fields. Default is `created_at`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/331625) in GitLab 14.8. | +| `iids[]` | integer array | **{dotted-circle}** No | Returns the request having the given `iid`. | +| `labels` | string | **{dotted-circle}** No | Returns merge requests matching a comma-separated list of labels. `None` lists all merge requests with no labels. `Any` lists all merge requests with at least one label. Predefined names are case-insensitive. | +| `milestone` | string | **{dotted-circle}** No | Returns merge requests for a specific milestone. `None` returns merge requests with no milestone. `Any` returns merge requests that have an assigned milestone. | +| `my_reaction_emoji` | string | **{dotted-circle}** No | Returns merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. | +| `not` | Hash | **{dotted-circle}** No | Returns merge requests that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `reviewer_id`, `reviewer_username`, `my_reaction_emoji`. | +| `order_by` | string | **{dotted-circle}** No | Returns requests ordered by `created_at`, `title` or `updated_at` fields. Default is `created_at`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/331625) in GitLab 14.8. | | `reviewer_id` | integer | **{dotted-circle}** No | Returns merge requests which have the user as a [reviewer](../user/project/merge_requests/getting_started.md#reviewer) with the given user `id`. `None` returns merge requests with no reviewers. `Any` returns merge requests with any reviewer. Mutually exclusive with `reviewer_username`. | | `reviewer_username` | string | **{dotted-circle}** No | Returns merge requests which have the user as a [reviewer](../user/project/merge_requests/getting_started.md#reviewer) with the given `username`. `None` returns merge requests with no reviewers. `Any` returns merge requests with any reviewer. Mutually exclusive with `reviewer_id`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49341) in GitLab 13.8. | -| `scope` | string | **{dotted-circle}** No | Return merge requests for the given scope: `created_by_me`, `assigned_to_me`, or `all`. | +| `scope` | string | **{dotted-circle}** No | Returns merge requests for the given scope: `created_by_me`, `assigned_to_me`, or `all`. | | `search` | string | **{dotted-circle}** No | Search merge requests against their `title` and `description`. | -| `sort` | string | **{dotted-circle}** No | Return requests sorted in `asc` or `desc` order. Default is `desc`. | -| `source_branch` | string | **{dotted-circle}** No | Return merge requests with the given source branch. | -| `state` | string | **{dotted-circle}** No | Return all merge requests or just those that are `opened`, `closed`, `locked`, or `merged`. | -| `target_branch` | string | **{dotted-circle}** No | Return merge requests with the given target branch. | -| `updated_after` | datetime | **{dotted-circle}** No | Return merge requests updated on or after the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | -| `updated_before` | datetime | **{dotted-circle}** No | Return merge requests updated on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `sort` | string | **{dotted-circle}** No | Returns requests sorted in `asc` or `desc` order. Default is `desc`. | +| `source_branch` | string | **{dotted-circle}** No | Returns merge requests with the given source branch. | +| `state` | string | **{dotted-circle}** No | Returns all merge requests or just those that are `opened`, `closed`, `locked`, or `merged`. | +| `target_branch` | string | **{dotted-circle}** No | Returns merge requests with the given target branch. | +| `updated_after` | datetime | **{dotted-circle}** No | Returns merge requests updated on or after the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `updated_before` | datetime | **{dotted-circle}** No | Returns merge requests updated on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | | `view` | string | **{dotted-circle}** No | If `simple`, returns the `iid`, URL, title, description, and basic state of merge request. | | `wip` | string | **{dotted-circle}** No | Filter merge requests against their `wip` status. `yes` to return *only* draft merge requests, `no` to return *non-draft* merge requests. | | `with_labels_details` | boolean | **{dotted-circle}** No | If `true`, response returns more details for each label in labels field: `:name`, `:color`, `:description`, `:description_html`, `:text_color`. Default is `false`. | @@ -425,7 +428,7 @@ GET /groups/:id/merge_requests?labels=bug,reproduced GET /groups/:id/merge_requests?my_reaction_emoji=star ``` -`group_id` represents the ID of the group which contains the project where the MR resides. +`group_id` represents the ID of the group which contains the project where the merge request resides. Supported attributes: @@ -438,24 +441,24 @@ Supported attributes: | `assignee_id` | integer | **{dotted-circle}** No | Returns merge requests assigned to the given user `id`. `None` returns unassigned merge requests. `Any` returns merge requests with an assignee. | | `author_id` | integer | **{dotted-circle}** No | Returns merge requests created by the given user `id`. Mutually exclusive with `author_username`. | | `author_username` | string | **{dotted-circle}** No | Returns merge requests created by the given `username`. Mutually exclusive with `author_id`. | -| `created_after` | datetime | **{dotted-circle}** No | Return merge requests created on or after the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | -| `created_before` | datetime | **{dotted-circle}** No | Return merge requests created on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | -| `labels` | string | **{dotted-circle}** No | Return merge requests matching a comma-separated list of labels. `None` lists all merge requests with no labels. `Any` lists all merge requests with at least one label. Predefined names are case-insensitive. | -| `milestone` | string | **{dotted-circle}** No | Return merge requests for a specific milestone. `None` returns merge requests with no milestone. `Any` returns merge requests that have an assigned milestone. | -| `my_reaction_emoji` | string | **{dotted-circle}** No | Return merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. | -| `non_archived` | boolean | **{dotted-circle}** No | Return merge requests from non archived projects only. Default is `true`. | -| `not` | Hash | **{dotted-circle}** No | Return merge requests that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `reviewer_id`, `reviewer_username`, `my_reaction_emoji`. | -| `order_by` | string | **{dotted-circle}** No | Return merge requests ordered by `created_at`, `title` or `updated_at` fields. Default is `created_at`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/331625) in GitLab 14.8. | +| `created_after` | datetime | **{dotted-circle}** No | Returns merge requests created on or after the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `created_before` | datetime | **{dotted-circle}** No | Returns merge requests created on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `labels` | string | **{dotted-circle}** No | Returns merge requests matching a comma-separated list of labels. `None` lists all merge requests with no labels. `Any` lists all merge requests with at least one label. Predefined names are case-insensitive. | +| `milestone` | string | **{dotted-circle}** No | Returns merge requests for a specific milestone. `None` returns merge requests with no milestone. `Any` returns merge requests that have an assigned milestone. | +| `my_reaction_emoji` | string | **{dotted-circle}** No | Returns merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. | +| `non_archived` | boolean | **{dotted-circle}** No | Returns merge requests from non archived projects only. Default is `true`. | +| `not` | Hash | **{dotted-circle}** No | Returns merge requests that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `reviewer_id`, `reviewer_username`, `my_reaction_emoji`. | +| `order_by` | string | **{dotted-circle}** No | Returns merge requests ordered by `created_at`, `title` or `updated_at` fields. Default is `created_at`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/331625) in GitLab 14.8. | | `reviewer_id` | integer | **{dotted-circle}** No | Returns merge requests which have the user as a [reviewer](../user/project/merge_requests/getting_started.md#reviewer) with the given user `id`. `None` returns merge requests with no reviewers. `Any` returns merge requests with any reviewer. Mutually exclusive with `reviewer_username`. | | `reviewer_username` | string | **{dotted-circle}** No | Returns merge requests which have the user as a [reviewer](../user/project/merge_requests/getting_started.md#reviewer) with the given `username`. `None` returns merge requests with no reviewers. `Any` returns merge requests with any reviewer. Mutually exclusive with `reviewer_id`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49341) in GitLab 13.8. | -| `scope` | string | **{dotted-circle}** No | Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`. | +| `scope` | string | **{dotted-circle}** No | Returns merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`. | | `search` | string | **{dotted-circle}** No | Search merge requests against their `title` and `description`. | -| `source_branch` | string | **{dotted-circle}** No | Return merge requests with the given source branch. | -| `sort` | string | **{dotted-circle}** No | Return merge requests sorted in `asc` or `desc` order. Default is `desc`. | -| `state` | string | **{dotted-circle}** No | Return all merge requests or just those that are `opened`, `closed`, `locked`, or `merged`. | -| `target_branch` | string | **{dotted-circle}** No | Return merge requests with the given target branch. | -| `updated_after` | datetime | **{dotted-circle}** No | Return merge requests updated on or after the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | -| `updated_before` | datetime | **{dotted-circle}** No | Return merge requests updated on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `source_branch` | string | **{dotted-circle}** No | Returns merge requests with the given source branch. | +| `sort` | string | **{dotted-circle}** No | Returns merge requests sorted in `asc` or `desc` order. Default is `desc`. | +| `state` | string | **{dotted-circle}** No | Returns all merge requests or just those that are `opened`, `closed`, `locked`, or `merged`. | +| `target_branch` | string | **{dotted-circle}** No | Returns merge requests with the given target branch. | +| `updated_after` | datetime | **{dotted-circle}** No | Returns merge requests updated on or after the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `updated_before` | datetime | **{dotted-circle}** No | Returns merge requests updated on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | | `view` | string | **{dotted-circle}** No | If `simple`, returns the `iid`, URL, title, description, and basic state of merge request. | | `with_labels_details` | boolean | **{dotted-circle}** No | If `true`, response returns more details for each label in labels field: `:name`, `:color`, `:description`, `:description_html`, `:text_color`. Default is `false`. | | `with_merge_status_recheck` | boolean | **{dotted-circle}** No | If `true`, this projection requests (but does not guarantee) that the `merge_status` field be recalculated asynchronously. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890) in GitLab 13.0. | @@ -601,7 +604,7 @@ For important notes on response data, read [Merge requests list response notes]( Shows information about a single merge request. **Note**: the `changes_count` value in the response is a string, not an -integer. This is because when an MR has too many changes to display and store, +integer. This is because when an merge request has too many changes to display and store, it is capped at 1,000. In that case, the API returns the string `"1000+"` for the changes count. @@ -633,7 +636,7 @@ Supported attributes: | `closed_by` | object | User who closed this merge request. | | `created_at` | datetime | Timestamp of when the merge request was created. | | `description` | string | Description of the merge request. Contains Markdown rendered as HTML for caching. | -| `detailed_merge_status` | string | Detailed merge status of the merge request. | +| `detailed_merge_status` | string | Detailed merge status of the merge request. Read [merge status](#merge-status) for a list of potential values. | | `diff_refs` | object | References of the base SHA, the head SHA, and the start SHA for this merge request. Corresponds to the latest diff version of the merge request. | | `discussion_locked` | boolean | Indicates if comments on the merge request are locked to members only. | | `downvotes` | integer | Number of downvotes for the merge request. | @@ -651,7 +654,7 @@ Supported attributes: | `merge_commit_sha` | string | SHA of the merge request commit. Returns `null` until merged. | | `merge_error` | string | Error message due to a merge error. | | `merge_user` | object | The user who merged this merge request, the user who set it to merge when pipeline succeeds, or `null`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/349031) in GitLab 14.7. | -| `merge_status` | string | Status of the merge request. Can be `unchecked`, `checking`, `can_be_merged`, `cannot_be_merged`, or `cannot_be_merged_recheck`. Affects the `has_conflicts` property. [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/3169#note_1162532204) in GitLab 15.6. Use `detailed_merge_status` instead. | +| `merge_status` | string | Status of the merge request. Can be `unchecked`, `checking`, `can_be_merged`, `cannot_be_merged`, or `cannot_be_merged_recheck`. Affects the `has_conflicts` property. For important notes on response data, read [Single merge request response notes](#single-merge-request-response-notes). [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/3169#note_1162532204) in GitLab 15.6. Use `detailed_merge_status` instead. | | `merge_when_pipeline_succeeds` | boolean | Indicates if the merge has been set to be merged when its pipeline succeeds. | | `merged_at` | datetime | Timestamp of when the merge request was merged. | | `merged_by` | object | User who merged this merge request or set it to merge when pipeline succeeds. [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/350534) in GitLab 14.7, and scheduled for removal in [API version 5](https://gitlab.com/groups/gitlab-org/-/epics/8115). Use `merge_user` instead. | @@ -831,27 +834,27 @@ Supported attributes: ### Merge status -> - The `detailed_merge_status` field was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101724) in GitLab 15.6. > - The `merge_status` field was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/3169#note_1162532204) in GitLab 15.6. +> - The `detailed_merge_status` field was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101724) in GitLab 15.6. Use `detailed_merge_status` instead of `merge_status` to account for all potential statuses. -- The `detailed_merge_status` field may hold one of the following values: - - `blocked_status`: Merge request is blocked by another merge request. - - `broken_status`: Can not merge the source into the target branch, potential conflict. - - `checking`: currently checking for mergeability. - - `ci_must_pass`: Pipeline must succeed before merging. - - `ci_still_running`: Pipeline is still running. - - `discussions_not_resolved`: Discussions must be resolved before merging. - - `draft_status`: Merge request must not be draft before merging. - - `external_status_checks`: Status checks must pass. - - `mergeable`: branch can be merged. - - `not_approved`: Merge request must be approved before merging. - - `not_open`: merge request must be open before merging. - - `policies_denied`: There are denied policies for the merge request. - - `unchecked`: merge status has not been checked. - -## Get single MR participants +- The `detailed_merge_status` field can contain one of the following values related to the merge request: + - `blocked_status`: Blocked by another merge request. + - `broken_status`: Can't merge into the target branch due to a potential conflict. + - `checking`: Mergeability checks are still in progress. + - `ci_must_pass`: A CI/CD pipeline must succeed before merge. + - `ci_still_running`: A CI/CD pipeline is still running. + - `discussions_not_resolved`: All discussions must be resolved before merge. + - `draft_status`: Can't merge because the merge request is a draft. + - `external_status_checks`: All status checks must pass before merge. + - `mergeable`: The branch can merge cleanly into the target branch. + - `not_approved`: Approval is required before merge. + - `not_open`: The merge request must be open before merge. + - `policies_denied`: The merge request contains denied policies. + - `unchecked`: The merge status has not been checked. + +## Get single merge request participants Get a list of merge request participants. @@ -887,7 +890,7 @@ Supported attributes: ] ``` -## Get single MR reviewers +## Get single merge request reviewers Get a list of merge request reviewers. @@ -931,7 +934,7 @@ Supported attributes: ] ``` -## Get single MR commits +## Get single merge request commits Get a list of merge request commits. @@ -969,7 +972,11 @@ Supported attributes: ] ``` -## Get single MR changes +## Get single merge request changes + +WARNING: +This endpoint was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/322117) in GitLab 15.7 +and will be removed in API v5. Use the [List merge request diffs](#list-merge-request-diffs) endpoint instead. Shows information about the merge request including its files and changes. @@ -1101,7 +1108,71 @@ Supported attributes: } ``` -## List MR pipelines +## List merge request diffs + +List diffs of the files changed in a merge request. + +```plaintext +GET /projects/:id/merge_requests/:merge_request_iid/diffs +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +|-----------|------|----------|-------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | +| `merge_request_iid` | integer | **{check-circle}** Yes | The internal ID of the merge request. | +| `page` | integer | **{dotted-circle}** no | The page of results to return. Defaults to 1. | +| `per_page` | integer | **{dotted-circle}** no | The number of results per page. Defaults to 20. | + +If successful, returns [`200 OK`](index.md#status-codes) and the +following response attributes: + +| Attribute | Type | Description | +|:----------|:-----|:------------| +| `old_path` | string | Old path of the file. | +| `new_path` | string | New path of the file. | +| `a_mode` | string | Old file mode of the file. | +| `b_mode` | string | New file mode of the file. | +| `diff` | string | Diff representation of the changes made on the file. | +| `new_file` | boolean | Indicates if the file has just been added. | +| `renamed_file` | boolean | Indicates if the file has been renamed. | +| `deleted_file` | boolean | Indicates if the file has been removed. | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/merge_requests/1/diffs?page=1&per_page=2" +``` + +Example response: + +```json +[ + { + "old_path": "README", + "new_path": "README", + "a_mode": "100644", + "b_mode": "100644", + "diff": "--- a/README\ +++ b/README\ @@ -1 +1 @@\ -Title\ +README", + "new_file": false, + "renamed_file": false, + "deleted_file": false + }, + { + "old_path": "VERSION", + "new_path": "VERSION", + "a_mode": "100644", + "b_mode": "100644", + "diff": "--- a/VERSION\ +++ b/VERSION\ @@ -1 +1 @@\ -1.9.7\ +1.9.8", + "new_file": false, + "renamed_file": false, + "deleted_file": false + } +] +``` + +## List merge request pipelines Get a list of merge request pipelines. @@ -1127,7 +1198,7 @@ Supported attributes: ] ``` -## Create MR Pipeline +## Create merge request pipeline Create a new [pipeline for a merge request](../ci/pipelines/merge_request_pipelines.md). A pipeline created via this endpoint doesn't run a regular branch/tag pipeline. @@ -1207,7 +1278,7 @@ POST /projects/:id/merge_requests | `allow_maintainer_to_push` | boolean | **{dotted-circle}** No | Alias of `allow_collaboration`. | | `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | Number of approvals required before this can be merged (see below). | | `assignee_id` | integer | **{dotted-circle}** No | Assignee user ID. | -| `assignee_ids` | integer array | **{dotted-circle}** No | The ID of the users to assign the MR to. Set to `0` or provide an empty value to unassign all assignees. | +| `assignee_ids` | integer array | **{dotted-circle}** No | The ID of the users to assign the merge request to. Set to `0` or provide an empty value to unassign all assignees. | | `description` | string | **{dotted-circle}** No | Description of the merge request. Limited to 1,048,576 characters. | | `labels` | string | **{dotted-circle}** No | Labels for the merge request, as a comma-separated list. | | `milestone_id` | integer | **{dotted-circle}** No | The global ID of a milestone. | @@ -1559,7 +1630,7 @@ curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://git ## Merge a merge request -Accept and merge changes submitted with MR using this API. +Accept and merge changes submitted with merge request using this API. ```plaintext PUT /projects/:id/merge_requests/:merge_request_iid/merge @@ -2574,7 +2645,7 @@ Example response: } ``` -## Get MR diff versions +## Get merge request diff versions Get a list of merge request diff versions. For an explanation of the SHAs in the response, read [SHAs in the API response](#shas-in-the-api-response). @@ -2624,7 +2695,7 @@ Example response: | `head_commit_sha` | The HEAD commit of the source branch. | | `start_commit_sha` | The HEAD commit SHA of the target branch when this version of the diff was created. | -## Get a single MR diff version +## Get a single merge request diff version Get a single merge request diff version. For an explanation of the SHAs in the response, read [SHAs in the API response](#shas-in-the-api-response). diff --git a/doc/api/merge_trains.md b/doc/api/merge_trains.md index 111cf5255d6..e8912aac759 100644 --- a/doc/api/merge_trains.md +++ b/doc/api/merge_trains.md @@ -154,3 +154,69 @@ Example response: } ] ``` + +## Get the status of a merge request on a merge train + +Get merge train information for the requested merge request. + +```plaintext +GET /projects/:id/merge_trains/merge_requests/:merge_request_iid +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +| ------------------- | -------------- | -------- | ------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding). | +| `merge_request_iid` | integer | yes | The internal ID of the merge request. | + +Example request: + +```shell +curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/597/merge_trains/merge_requests/1" +``` + +Example response: + +```json +{ + "id": 267, + "merge_request": { + "id": 273, + "iid": 1, + "project_id": 597, + "title": "My title 9", + "description": null, + "state": "opened", + "created_at": "2022-10-31T19:06:05.725Z", + "updated_at": "2022-10-31T19:06:05.725Z", + "web_url": "http://localhost/namespace18/project21/-/merge_requests/1" + }, + "user": { + "id": 933, + "username": "user12", + "name": "Sidney Jones31", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/6c8365de387cb3db10ecc7b1880203c4?s=80\u0026d=identicon", + "web_url": "http://localhost/user12" + }, + "pipeline": { + "id": 273, + "iid": 1, + "project_id": 598, + "sha": "b83d6e391c22777fca1ed3012fce84f633d7fed0", + "ref": "main", + "status": "pending", + "source": "push", + "created_at": "2022-10-31T19:06:06.231Z", + "updated_at": "2022-10-31T19:06:06.231Z", + "web_url": "http://localhost/namespace19/project22/-/pipelines/273" + }, + "created_at": "2022-10-31T19:06:06.237Z", + "updated_at":"2022-10-31T19:06:06.237Z", + "target_branch":"main", + "status":"idle", + "merged_at":null, + "duration":null +} +``` diff --git a/doc/api/packages.md b/doc/api/packages.md index ffdd9fe7d11..6c5cf6bee1e 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -180,7 +180,7 @@ can result in malformed data or broken packages. ## Get a project package -Get a single project package. +Get a single project package. Only packages with status `default` are returned. ```plaintext GET /projects/:id/packages/:package_id @@ -256,7 +256,7 @@ Example response: The `_links` object contains the following properties: -- `web_path`: The path which you can visit in GitLab and see the details of the package. +- `web_path`: The path which you can visit in GitLab and see the details of the package. Only available if the package has status `default`. - `delete_api_path`: The API path to delete the package. Only available if the request user has permission to do so. ## List package files diff --git a/doc/api/packages/pypi.md b/doc/api/packages/pypi.md index 4e4c060d99b..e9546f50e36 100644 --- a/doc/api/packages/pypi.md +++ b/doc/api/packages/pypi.md @@ -32,7 +32,7 @@ Download a PyPI package file. The [simple API](#group-level-simple-api-entry-poi normally supplies this URL. ```plaintext -GET groups/:id/packages/pypi/files/:sha256/:file_identifier +GET groups/:id/-/packages/pypi/files/:sha256/:file_identifier ``` | Attribute | Type | Required | Description | @@ -42,13 +42,13 @@ GET groups/:id/packages/pypi/files/:sha256/:file_identifier | `file_identifier` | string | yes | The PyPI package file's name. | ```shell -curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1.tar.gz" +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1.tar.gz" ``` To write the output to a file: ```shell -curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1.tar.gz" >> my.pypi.package-0.0.1.tar.gz +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1.tar.gz" >> my.pypi.package-0.0.1.tar.gz ``` This writes the downloaded file to `my.pypi.package-0.0.1.tar.gz` in the current diff --git a/doc/api/packages/terraform-modules.md b/doc/api/packages/terraform-modules.md index bb5b39b5161..6b3d6b477b2 100644 --- a/doc/api/packages/terraform-modules.md +++ b/doc/api/packages/terraform-modules.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w This is the API documentation for [Terraform Modules](../../user/packages/terraform_module_registry/index.md). WARNING: -This API is used by the [terraform cli](https://www.terraform.io/) +This API is used by the [Terraform CLI](https://www.terraform.io/) and is generally not meant for manual consumption. For instructions on how to upload and install Terraform modules from the GitLab diff --git a/doc/api/pipelines.md b/doc/api/pipelines.md index 8242f8cff00..669161049d5 100644 --- a/doc/api/pipelines.md +++ b/doc/api/pipelines.md @@ -274,7 +274,7 @@ Sample response: Get the latest pipeline for a specific ref in a project. ```plaintext -POST /projects/:id/pipeline/latest +GET /projects/:id/pipelines/latest ``` | Attribute | Type | Required | Description | diff --git a/doc/api/product_analytics.md b/doc/api/product_analytics.md index e10327bc59b..90df1090f62 100644 --- a/doc/api/product_analytics.md +++ b/doc/api/product_analytics.md @@ -16,12 +16,13 @@ This feature is not ready for production use. NOTE: Make sure to define the `cube_api_base_url` and `cube_api_key` application settings first using [the API](settings.md). -## Send request to Cube +## Send query request to Cube Generate an access token that can be used to query the Cube API. For example: ```plaintext POST /projects/:id/product_analytics/request/load +POST /projects/:id/product_analytics/request/dry-run ``` | Attribute | Type | Required | Description | @@ -30,7 +31,7 @@ POST /projects/:id/product_analytics/request/load ### Request body -The body of the request should be a valid Cube query. +The body of the load request must be a valid Cube query. ```json { @@ -66,3 +67,15 @@ The body of the request should be a valid Cube query. "queryType": "multi" } ``` + +## Send metadata request to Cube + +Return Cube Metadata for the Analytics data. For example: + +```plaintext +GET /projects/:id/product_analytics/request/meta +``` + +| Attribute | Type | Required | Description | +| --------- |------------------| -------- |---------------------------------------------------------------| +| `id` | integer | yes | The ID of a project that the current user has read access to. | diff --git a/doc/api/project_badges.md b/doc/api/project_badges.md index d83aa370808..52d32ab17b9 100644 --- a/doc/api/project_badges.md +++ b/doc/api/project_badges.md @@ -13,6 +13,8 @@ Badges support placeholders that are replaced in real-time in both the link and <!-- vale gitlab.Spelling = NO --> - **%{project_path}**: Replaced by the project path. +- **%{project_title}**: Replaced by the project title. +- **%{project_name}**: Replaced by the project name. - **%{project_id}**: Replaced by the project ID. - **%{default_branch}**: Replaced by the project default branch. - **%{commit_sha}**: Replaced by the last project's commit SHA. diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index 83d8746e1d0..a14f385cdb5 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -30,6 +30,9 @@ project to a web server or to any S3-compatible platform. For exports, GitLab: The `upload[url]` parameter is required if the `upload` parameter is present. +For uploads to Amazon S3, refer to [Generating a pre-signed URL for uploading objects](https://docs.aws.amazon.com/AmazonS3/latest/userguide/PresignedUrlUploadObject.html) +documentation scripts to generate the `upload[url]`. + ```plaintext POST /projects/:id/export ``` @@ -187,7 +190,7 @@ requests.post(url, headers=headers, data=data, files=files) NOTE: The maximum import file size can be set by the Administrator, default is `0` (unlimited).. -As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](settings.md#change-application-settings) or the [Admin Area](../user/admin_area/settings/account_and_limit_settings.md). Default [modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50MB to 0 in GitLab 13.8. +As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](settings.md#change-application-settings) or the [Admin Area](../user/admin_area/settings/account_and_limit_settings.md). Default [modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50 MB to 0 in GitLab 13.8. ## Import a file from a remote object storage diff --git a/doc/api/project_snippets.md b/doc/api/project_snippets.md index 29d3b38f977..afb7519d5f3 100644 --- a/doc/api/project_snippets.md +++ b/doc/api/project_snippets.md @@ -16,7 +16,7 @@ Constants for snippet visibility levels are: | visibility | Description | | ---------- | ----------- | | `private` | The snippet is visible only to project members | -| `internal` | The snippet is visible for any logged in user except [external users](../user/permissions.md#external-users) | +| `internal` | The snippet is visible for any logged in user except [external users](../user/admin_area/external_users.md) | | `public` | The snippet can be accessed without any authentication | NOTE: diff --git a/doc/api/projects.md b/doc/api/projects.md index 638af168f22..9ddb58b1436 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -16,7 +16,7 @@ The visibility level is determined by the `visibility` field in the project. Values for the project visibility level are: - `private`: project access must be granted explicitly to each user. -- `internal`: the project can be cloned by any signed-in user except [external users](../user/permissions.md#external-users). +- `internal`: the project can be cloned by any signed-in user except [external users](../user/admin_area/external_users.md). - `public`: the project can be accessed without any authentication. For more, read [Project visibility](../user/public_access.md). @@ -51,10 +51,10 @@ GET /projects | `id_after` | integer | **{dotted-circle}** No | Limit results to projects with IDs greater than the specified ID. | | `id_before` | integer | **{dotted-circle}** No | Limit results to projects with IDs less than the specified ID. | | `imported` | boolean | **{dotted-circle}** No | Limit results to projects which were imported from external systems by current user. | -| `last_activity_after` | datetime | **{dotted-circle}** No | Limit results to projects with last_activity after specified time. Format: ISO 8601 (`YYYY-MM-DDTHH:MM:SSZ`) | -| `last_activity_before` | datetime | **{dotted-circle}** No | Limit results to projects with last_activity before specified time. Format: ISO 8601 (`YYYY-MM-DDTHH:MM:SSZ`) | +| `last_activity_after` | datetime | **{dotted-circle}** No | Limit results to projects with last activity after specified time. Format: ISO 8601 (`YYYY-MM-DDTHH:MM:SSZ`) | +| `last_activity_before` | datetime | **{dotted-circle}** No | Limit results to projects with last activity before specified time. Format: ISO 8601 (`YYYY-MM-DDTHH:MM:SSZ`) | | `membership` | boolean | **{dotted-circle}** No | Limit by projects that the current user is a member of. | -| `min_access_level` | integer | **{dotted-circle}** No | Limit by current user minimal [access level](members.md#valid-access-levels). | +| `min_access_level` | integer | **{dotted-circle}** No | Limit by current user minimal [role (`access_level`)](members.md#roles). | | `order_by` | string | **{dotted-circle}** No | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, `last_activity_at`, or `similarity` fields. `repository_size`, `storage_size`, `packages_size` or `wiki_size` fields are only allowed for administrators. `similarity` ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332890) in GitLab 14.1) is only available when searching and is limited to projects that the current user is a member of. Default is `created_at`. | | `owned` | boolean | **{dotted-circle}** No | Limit by projects explicitly owned by the current user. | | `repository_checksum_failed` **(PREMIUM)** | boolean | **{dotted-circle}** No | Limit projects where the repository checksum calculation has failed. | @@ -333,7 +333,7 @@ GET /users/:user_id/projects | `id_after` | integer | **{dotted-circle}** No | Limit results to projects with IDs greater than the specified ID. | | `id_before` | integer | **{dotted-circle}** No | Limit results to projects with IDs less than the specified ID. | | `membership` | boolean | **{dotted-circle}** No | Limit by projects that the current user is a member of. | -| `min_access_level` | integer | **{dotted-circle}** No | Limit by current user minimal [access level](members.md#valid-access-levels). | +| `min_access_level` | integer | **{dotted-circle}** No | Limit by current user minimal [role (`access_level`)](members.md#roles). | | `order_by` | string | **{dotted-circle}** No | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at`. | | `owned` | boolean | **{dotted-circle}** No | Limit by projects explicitly owned by the current user. | | `search` | string | **{dotted-circle}** No | Return list of projects matching the search criteria. | @@ -590,7 +590,7 @@ GET /users/:user_id/starred_projects | `user_id` | string | **{check-circle}** Yes | The ID or username of the user. | | `archived` | boolean | **{dotted-circle}** No | Limit by archived status. | | `membership` | boolean | **{dotted-circle}** No | Limit by projects that the current user is a member of. | -| `min_access_level` | integer | **{dotted-circle}** No | Limit by current user minimal [access level](members.md#valid-access-levels). | +| `min_access_level` | integer | **{dotted-circle}** No | Limit by current user minimal [role (`access_level`)](members.md#roles). | | `order_by` | string | **{dotted-circle}** No | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at`. | | `owned` | boolean | **{dotted-circle}** No | Limit by projects explicitly owned by the current user. | | `search` | string | **{dotted-circle}** No | Return list of projects matching the search criteria. | @@ -1150,7 +1150,7 @@ GET /projects/:id/groups |-----------------------------|-------------------|------------------------|-------------| | `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding). | | `search` | string | **{dotted-circle}** No | Search for specific groups. | -| `shared_min_access_level` | integer | **{dotted-circle}** No | Limit to shared groups with at least this [access level](members.md#valid-access-levels). | +| `shared_min_access_level` | integer | **{dotted-circle}** No | Limit to shared groups with at least this [role (`access_level`)](members.md#roles). | | `shared_visible_only` | boolean | **{dotted-circle}** No | Limit to shared groups user has access to. | | `skip_groups` | array of integers | **{dotted-circle}** No | Skip the group IDs passed. | | `with_shared` | boolean | **{dotted-circle}** No | Include projects shared with this group. Default is `false`. | @@ -1250,6 +1250,10 @@ curl --request POST --header "PRIVATE-TOKEN: <your-token>" \ | `printing_merge_request_link_enabled` | boolean | **{dotted-circle}** No | Show link to create/view merge request when pushing from the command line. | | `public_builds` | boolean | **{dotted-circle}** No | If `true`, jobs can be viewed by non-project members. | | `releases_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `environments_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `feature_flags_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `infrastructure_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `monitor_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `remove_source_branch_after_merge` | boolean | **{dotted-circle}** No | Enable `Delete source branch` option by default for all new merge requests. | | `repository_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `repository_storage` | string | **{dotted-circle}** No | Which storage shard the repository is on. _(administrator only)_ | @@ -1330,6 +1334,10 @@ POST /projects/user/:user_id | `printing_merge_request_link_enabled` | boolean | **{dotted-circle}** No | Show link to create/view merge request when pushing from the command line. | | `public_builds` | boolean | **{dotted-circle}** No | If `true`, jobs can be viewed by non-project-members. | | `releases_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `environments_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `feature_flags_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `infrastructure_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `monitor_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `remove_source_branch_after_merge` | boolean | **{dotted-circle}** No | Enable `Delete source branch` option by default for all new merge requests. | | `repository_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `repository_storage` | string | **{dotted-circle}** No | Which storage shard the repository is on. _(administrators only)_ | @@ -1434,6 +1442,10 @@ Supported attributes: | `printing_merge_request_link_enabled` | boolean | **{dotted-circle}** No | Show link to create/view merge request when pushing from the command line. | | `public_builds` | boolean | **{dotted-circle}** No | If `true`, jobs can be viewed by non-project members. | | `releases_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `environments_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `feature_flags_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `infrastructure_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `monitor_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `remove_source_branch_after_merge` | boolean | **{dotted-circle}** No | Enable `Delete source branch` option by default for all new merge requests. | | `repository_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `repository_storage` | string | **{dotted-circle}** No | Which storage shard the repository is on. _(administrators only)_ | @@ -1496,7 +1508,7 @@ GET /projects/:id/forks | `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding). | | `archived` | boolean | **{dotted-circle}** No | Limit by archived status. | | `membership` | boolean | **{dotted-circle}** No | Limit by projects that the current user is a member of. | -| `min_access_level` | integer | **{dotted-circle}** No | Limit by current user minimal [access level](members.md#valid-access-levels). | +| `min_access_level` | integer | **{dotted-circle}** No | Limit by current user minimal [role (`access_level`)](members.md#roles). | | `order_by` | string | **{dotted-circle}** No | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at`. | | `owned` | boolean | **{dotted-circle}** No | Limit by projects explicitly owned by the current user. | | `search` | string | **{dotted-circle}** No | Return list of projects matching the search criteria. | @@ -2310,7 +2322,7 @@ POST /projects/:id/share | Attribute | Type | Required | Description | |----------------|----------------|------------------------|-------------| -| `group_access` | integer | **{check-circle}** Yes | The [access level](members.md#valid-access-levels) to grant the group. | +| `group_access` | integer | **{check-circle}** Yes | The [role (`access_level`)](members.md#roles) to grant the group. | | `group_id` | integer | **{check-circle}** Yes | The ID of the group to share with. | | `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding). | | `expires_at` | string | **{dotted-circle}** No | Share expiration date in ISO 8601 format: 2016-09-26 | @@ -2462,7 +2474,7 @@ PUT /projects/:id/hooks/:hook_id | `push_events` | boolean | **{dotted-circle}** No | Trigger hook on push events. | | `releases_events` | boolean | **{dotted-circle}** No | Trigger hook on release events. | | `tag_push_events` | boolean | **{dotted-circle}** No | Trigger hook on tag push events. | -| `token` | string | **{dotted-circle}** No | Secret token to validate received payloads; this isn't returned in the response. | +| `token` | string | **{dotted-circle}** No | Secret token to validate received payloads. Not returned in the response. When you change the webhook URL, the secret token is reset and not retained. | | `wiki_page_events` | boolean | **{dotted-circle}** No | Trigger hook on wiki page events. | ### Delete project hook diff --git a/doc/api/protected_branches.md b/doc/api/protected_branches.md index 96bd5c15f13..96d4240b3ef 100644 --- a/doc/api/protected_branches.md +++ b/doc/api/protected_branches.md @@ -464,7 +464,7 @@ To delete: ```shell curl --header 'Content-Type: application/json' --request PATCH \ - --data '{"push_access_levels": [{"group_id": 9899829, access_level: 40}]}' \ + --data '{"allowed_to_push": [{"access_level": 40}]}' \ --header "PRIVATE-TOKEN: <your_access_token>" \ "https://gitlab.example.com/api/v4/projects/22034114/protected_branches/master" ``` @@ -474,13 +474,13 @@ Example response: ```json { "name": "master", - "allowed_to_push": [ + "push_access_levels": [ { "id": 12, "access_level": 40, - "access_level_description": "Administrator", + "access_level_description": "Maintainers", "user_id": null, - "group_id": 9899829 + "group_id": null } ] } @@ -489,21 +489,23 @@ Example response: ### Example: update a `push_access_level` record ```shell -curl --header 'Content-Type: application/json' --request PUT \ - --data '{"push_access_levels": [{"id": 12, "group_id": 22034120}]' \ +curl --header 'Content-Type: application/json' --request PATCH \ + --data '{"allowed_to_push": [{"id": 12, "access_level": 0}]' \ --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/22034114/protected_branches/master" ``` +Example response: + ```json { "name": "master", - "deploy_access_levels": [ + "push_access_levels": [ { "id": 12, - "access_level": 40, - "access_level_description": "Administrator", + "access_level": 0, + "access_level_description": "No One", "user_id": null, - "group_id": 22034120 + "group_id": null } ] } @@ -512,8 +514,8 @@ curl --header 'Content-Type: application/json' --request PUT \ ### Example: delete a `push_access_level` record ```shell -curl --header 'Content-Type: application/json' --request PUT \ - --data '{"push_access_levels": [{"id": 12, "_destroy": true}]' \ +curl --header 'Content-Type: application/json' --request PATCH \ + --data '{"allowed_to_push": [{"id": 12, "_destroy": true}]}' \ --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/22034114/protected_branches/master" ``` diff --git a/doc/api/repository_files.md b/doc/api/repository_files.md index 1db614fce36..e105f5ee5e3 100644 --- a/doc/api/repository_files.md +++ b/doc/api/repository_files.md @@ -214,6 +214,7 @@ GET /projects/:id/repository/files/:file_path/raw | `id` | integer or string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | | `file_path` | string | yes | URL-encoded full path to new file, such as `lib%2Fclass%2Erb`. | | `ref` | string | yes | The name of branch, tag or commit. Default is the `HEAD` of the project. | +| `lfs` | boolean | no | Determines if the response should be Git LFS file contents, rather than the pointer. If the file is not tracked by Git LFS, ignored. Defaults to `false`. | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/13083/repository/files/app%2Fmodels%2Fkey%2Erb/raw?ref=master" diff --git a/doc/api/runners.md b/doc/api/runners.md index f690e0cb9c1..62d5e41c877 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -14,8 +14,8 @@ There are two tokens to take into account when connecting a runner with GitLab. | Token | Description | | ----- | ----------- | -| Registration token | Token used to [register the runner](https://docs.gitlab.com/runner/register/). It can be [obtained through GitLab](../ci/runners/index.md). | -| Authentication token | Token used to authenticate the runner with the GitLab instance. It is obtained automatically when you [register a runner](https://docs.gitlab.com/runner/register/) or by the Runner API when you manually [register a runner](#register-a-new-runner-deprecated) or [reset the authentication token](#reset-runners-authentication-token-by-using-the-runner-id). | +| Registration token | Token used to [register the runner](https://docs.gitlab.com/runner/register/). It can be [obtained through GitLab](../ci/runners/index.md). | +| Authentication token | Token used to authenticate the runner with the GitLab instance. It is obtained automatically when you [register a runner](https://docs.gitlab.com/runner/register/) or by the Runner API when you manually [register a runner](#register-a-new-runner) or [reset the authentication token](#reset-runners-authentication-token-by-using-the-runner-id). | Here's an example of how the two tokens are used in runner registration: @@ -640,12 +640,7 @@ Example response: ] ``` -## Register a new runner (deprecated) - -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/102579) in GitLab 15.6. - -WARNING: -This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/102579) in GitLab 15.6 and is planned for removal in 16.0. This change is a breaking change. +## Register a new runner Register a new runner for the instance. @@ -655,18 +650,18 @@ POST /runners | Attribute | Type | Required | Description | |--------------------|--------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `token` | string | yes | [Registration token](#registration-and-authentication-tokens) | -| `description` | string | no | Runner's description | +| `token` | string | yes | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): [Registration token](#registration-and-authentication-tokens). The registration token will be replaced by an authentication token in GitLab 16.0 | +| `description` | string | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): Runner's description | | `info` | hash | no | Runner's metadata. You can include `name`, `version`, `revision`, `platform`, and `architecture`, but only `version`, `platform`, and `architecture` are displayed in the Admin Area of the UI | -| `active` | boolean | no | Deprecated: Use `paused` instead. Specifies whether the runner is allowed to receive new jobs | -| `paused` | boolean | no | Specifies whether the runner should ignore new jobs | -| `locked` | boolean | no | Specifies whether the runner should be locked for the current project | -| `run_untagged` | boolean | no | Specifies whether the runner should handle untagged jobs | -| `tag_list` | string array | no | A list of runner tags | -| `access_level` | string | no | The access level of the runner; `not_protected` or `ref_protected` | -| `maximum_timeout` | integer | no | Maximum timeout that limits the amount of time (in seconds) that runners can run jobs | -| `maintainer_note` | string | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/350730), see `maintenance_note` | -| `maintenance_note` | string | no | Free-form maintenance notes for the runner (1024 characters) | +| `active` | boolean | no | Deprecated: Use `paused` instead. Specifies whether the runner is allowed to receive new jobs | +| `paused` | boolean | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): Specifies whether the runner should ignore new jobs | +| `locked` | boolean | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): Specifies whether the runner should be locked for the current project | +| `run_untagged` | boolean | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): Specifies whether the runner should handle untagged jobs | +| `tag_list` | string array | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): A list of runner tags | +| `access_level` | string | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): The access level of the runner; `not_protected` or `ref_protected` | +| `maximum_timeout` | integer | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): Maximum timeout that limits the amount of time (in seconds) that runners can run jobs | +| `maintainer_note` | string | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/350730), see `maintenance_note` | +| `maintenance_note` | string | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): Free-form maintenance notes for the runner (1024 characters) | ```shell curl --request POST "https://gitlab.example.com/api/v4/runners" \ @@ -762,7 +757,11 @@ Response: ## Reset instance's runner registration token -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30942) in GitLab 14.3. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30942) in GitLab 14.3. +> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104691) in GitLab 15.7. + +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383341) in GitLab 15.7 and is planned for removal in 16.0. This change is a breaking change. Reset the runner registration token for the GitLab instance. @@ -777,7 +776,11 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ ## Reset project's runner registration token -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30942) in GitLab 14.3. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30942) in GitLab 14.3. +> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104691) in GitLab 15.7. + +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383341) in GitLab 15.7 and is planned for removal in 16.0. This change is a breaking change. Reset the runner registration token for a project. @@ -792,7 +795,11 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ ## Reset group's runner registration token -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30942) in GitLab 14.3. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30942) in GitLab 14.3. +> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104691) in GitLab 15.7. + +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383341) in GitLab 15.7 and is planned for removal in 16.0. This change is a breaking change. Reset the runner registration token for a group. diff --git a/doc/api/saml.md b/doc/api/saml.md index 810ed382d49..4b0e57111cc 100644 --- a/doc/api/saml.md +++ b/doc/api/saml.md @@ -35,9 +35,7 @@ response attributes: Example request: ```shell -curl --location --request GET "https://gdk.test:3443/api/v4/groups/33/saml/identities" \ ---header "<PRIVATE-TOKEN>" \ ---form "extern_uid=<ID_TO_BE_UPDATED>" \ +curl --location --request GET "https://gitlab.example.com/api/v4/groups/33/saml/identities" --header "<PRIVATE-TOKEN>" ``` Example response: @@ -53,7 +51,7 @@ Example response: ## Update `extern_uid` field for a SAML identity -Update `extern_uid` field for a SAML identity. Field that can be updated are: +Update `extern_uid` field for a SAML identity: | SAML IdP attribute | GitLab field | | ------------------ | ------------ | @@ -72,7 +70,7 @@ Parameters: Example request: ```shell -curl --location --request PATCH "https://gdk.test:3443/api/v4/groups/33/saml/sydney_jones" \ +curl --location --request PATCH "https://gitlab.example.com/api/v4/groups/33/saml/sydney_jones" \ --header "<PRIVATE TOKEN>" \ --form "extern_uid=sydney_jones_new" \ ``` diff --git a/doc/api/scim.md b/doc/api/scim.md index 53897cadf90..0ee9779ccbd 100644 --- a/doc/api/scim.md +++ b/doc/api/scim.md @@ -8,6 +8,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/98354) in GitLab 15.5. +GitLab provides an SCIM API that both implements [the RFC7644 protocol](https://tools.ietf.org/html/rfc7644) +and provides the `/Users` endpoint. The base URL is `/api/scim/v2/groups/:group_path/Users/`. + To use this API, [Group SSO](../user/group/saml_sso/index.md) must be enabled for the group. This API is only in use where [SCIM for Group SSO](../user/group/saml_sso/scim_setup.md) is enabled. It's a prerequisite to the creation of SCIM identities. @@ -50,11 +53,10 @@ Example request: ```shell curl --location --request GET "https://gitlab.example.com/api/v4/groups/33/scim/identities" \ ---header "PRIVATE-TOKEN: <PRIVATE-TOKEN>" \ ---form "extern_uid=<ID_TO_BE_UPDATED>" \ +--header "PRIVATE-TOKEN: <PRIVATE-TOKEN>" ``` -## Update extern_uid field for a SCIM identity +## Update `extern_uid` field for a SCIM identity > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227841) in GitLab 15.5. @@ -79,5 +81,5 @@ Example request: ```shell curl --location --request PATCH "https://gitlab.example.com/api/v4/groups/33/scim/sydney_jones" \ --header "PRIVATE-TOKEN: <PRIVATE TOKEN>" \ ---form "extern_uid=sydney_jones_new" \ +--form "extern_uid=sydney_jones_new" ``` diff --git a/doc/api/settings.md b/doc/api/settings.md index 78dc81c4f84..7d55180db94 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -36,6 +36,7 @@ Example response: "signup_enabled" : true, "id" : 1, "default_branch_protection" : 2, + "default_preferred_language" : "en", "restricted_visibility_levels" : [], "password_authentication_enabled_for_web" : true, "after_sign_out_path" : null, @@ -104,12 +105,15 @@ Example response: "floc_enabled": false, "external_pipeline_validation_service_timeout": null, "external_pipeline_validation_service_token": null, - "external_pipeline_validation_service_url": null + "external_pipeline_validation_service_url": null, + "jira_connect_application_key": null, + "jira_connect_proxy_url": null } ``` Users on [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) may also see -the `group_owners_can_manage_default_branch_protection`, `file_template_project_id`, `delayed_project_deletion`, `delayed_group_deletion`, `deletion_adjourned_period`, or the `geo_node_allowed_ips` parameters: +the `group_owners_can_manage_default_branch_protection`, `file_template_project_id`, `delayed_project_deletion`, +`delayed_group_deletion`, `deletion_adjourned_period`, `disable_personal_access_tokens`, or the `geo_node_allowed_ips` parameters: ```json { @@ -121,6 +125,7 @@ the `group_owners_can_manage_default_branch_protection`, `file_template_project_ "delayed_project_deletion": false, "delayed_group_deletion": false, "deletion_adjourned_period": 7, + "disable_personal_access_tokens": false, ... } ``` @@ -144,6 +149,7 @@ Example response: { "id": 1, "default_projects_limit": 100000, + "default_preferred_language": "en", "signup_enabled": false, "password_authentication_enabled_for_web": true, "gravatar_enabled": true, @@ -218,7 +224,9 @@ Example response: "external_pipeline_validation_service_timeout": null, "external_pipeline_validation_service_token": null, "external_pipeline_validation_service_url": null, - "can_create_group": false + "can_create_group": false, + "jira_connect_application_key": "123", + "jira_connect_proxy_url": "http://gitlab.example.com" } ``` @@ -232,6 +240,7 @@ these parameters: - `delayed_project_deletion` - `delayed_group_deletion` - `deletion_adjourned_period` +- `disable_personal_access_tokens` Example responses: **(PREMIUM SELF)** @@ -286,6 +295,7 @@ listed in the descriptions of the relevant settings. | `default_branch_protection` | integer | no | Determine if developers can push to the default branch. Can take: `0` _(not protected, both users with the Developer role or Maintainer role can push new commits and force push)_, `1` _(partially protected, users with the Developer role or Maintainer role can push new commits, but cannot force push)_ or `2` _(fully protected, users with the Developer or Maintainer role cannot push new commits, but users with the Developer or Maintainer role can; no one can force push)_ as a parameter. Default is `2`. | | `default_ci_config_path` | string | no | Default CI/CD configuration file and path for new projects (`.gitlab-ci.yml` if not set). | | `default_group_visibility` | string | no | What visibility level new groups receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | +| `default_preferred_language` | string | no | Default preferred language for users who are not logged in. | | `default_project_creation` | integer | no | Default project creation protection. Can take: `0` _(No one)_, `1` _(Maintainers)_ or `2` _(Developers + Maintainers)_| | `default_project_visibility` | string | no | What visibility level new projects receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | | `default_projects_limit` | integer | no | Project limit per user. Default is `100000`. | @@ -299,10 +309,11 @@ listed in the descriptions of the relevant settings. | `diff_max_lines` | integer | no | Maximum [lines in a diff](../user/admin_area/diff_limits.md). | | `disable_admin_oauth_scopes` | boolean | no | Stops administrators from connecting their GitLab accounts to non-trusted OAuth 2.0 applications that have the `api`, `read_api`, `read_repository`, `write_repository`, `read_registry`, `write_registry`, or `sudo` scopes. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/375043) in GitLab 15.6. | | `disable_feed_token` | boolean | no | Disable display of RSS/Atom and calendar feed tokens. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/231493) in GitLab 13.7. | +| `disable_personal_access_token` **(PREMIUM SELF)** | boolean | no | Disable personal access tokens. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/384201) in GitLab 15.7. | | `disabled_oauth_sign_in_sources` | array of strings | no | Disabled OAuth sign-in sources. | | `dns_rebinding_protection_enabled` | boolean | no | Enforce DNS rebinding attack protection. | | `domain_denylist_enabled` | boolean | no | (**If enabled, requires:** `domain_denylist`) Allows blocking sign-ups from emails from specific domains. | -| `domain_denylist` | array of strings | no | Users with email addresses that match these domains **cannot** sign up. Wildcards allowed. Use separate lines for multiple entries. Ex: `domain.com`, `*.domain.com`. | +| `domain_denylist` | array of strings | no | Users with email addresses that match these domains **cannot** sign up. Wildcards allowed. Use separate lines for multiple entries. For example: `domain.com`, `*.domain.com`. | | `domain_allowlist` | array of strings | no | Force people to use only corporate emails for sign-up. Default is `null`, meaning there is no restriction. | | `dsa_key_restriction` | integer | no | The minimum allowed bit length of an uploaded DSA key. Default is `0` (no restriction). `-1` disables DSA keys. | | `ecdsa_key_restriction` | integer | no | The minimum allowed curve size (in bits) of an uploaded ECDSA key. Default is `0` (no restriction). `-1` disables ECDSA keys. | @@ -331,6 +342,7 @@ listed in the descriptions of the relevant settings. | `elasticsearch_password` **(PREMIUM)** | string | no | The password of your Elasticsearch instance. | | `email_additional_text` **(PREMIUM)** | string | no | Additional text added to the bottom of every email for legal/auditing/compliance reasons. | | `email_author_in_body` | boolean | no | Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead. | +| `email_confirmation_setting` | string | no | Specifies whether users must confirm their email before sign in. Possible values are `off`, `soft`, and `hard`. | | `enabled_git_access_protocol` | string | no | Enabled protocols for Git access. Allowed values are: `ssh`, `http`, and `nil` to allow both protocols. | | `enforce_namespace_storage_limit` | boolean | no | Enabling this permits enforcement of namespace storage limits. | | `enforce_terms` | boolean | no | (**If enabled, requires:** `terms`) Enforce application ToS to all users. | @@ -389,6 +401,7 @@ listed in the descriptions of the relevant settings. | `max_pages_size` | integer | no | Maximum size of pages repositories in MB. | | `max_personal_access_token_lifetime` **(ULTIMATE SELF)** | integer | no | Maximum allowable lifetime for access tokens in days. | | `max_ssh_key_lifetime` **(ULTIMATE SELF)** | integer | no | Maximum allowable lifetime for SSH keys in days. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1007) in GitLab 14.6. | +| `max_terraform_state_size_bytes` | integer | no | Maximum size in bytes of the [Terraform state](../administration/terraform_state.md) files. Set this to 0 for unlimited file size. | | `metrics_method_call_threshold` | integer | no | A method call is only tracked when it takes longer than the given amount of milliseconds. | | `max_number_of_repository_downloads` **(ULTIMATE SELF)** | integer | no | Maximum number of unique repositories a user can download in the specified time period before they are banned. Default: 0, Maximum: 10,000 repositories. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87980) in GitLab 15.1. | | `max_number_of_repository_downloads_within_time_period` **(ULTIMATE SELF)** | integer | no | Reporting time period (in seconds). Default: 0, Maximum: 864000 seconds (10 days). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87980) in GitLab 15.1. | @@ -439,13 +452,12 @@ listed in the descriptions of the relevant settings. | `require_two_factor_authentication` | boolean | no | (**If enabled, requires:** `two_factor_grace_period`) Require all users to set up Two-factor authentication. | | `restricted_visibility_levels` | array of strings | no | Selected levels cannot be used by non-Administrator users for groups, projects or snippets. Can take `private`, `internal` and `public` as a parameter. Default is `null` which means there is no restriction. | | `rsa_key_restriction` | integer | no | The minimum allowed bit length of an uploaded RSA key. Default is `0` (no restriction). `-1` disables RSA keys. | -| `send_user_confirmation_email` | boolean | no | Send confirmation email on sign-up. | | `session_expire_delay` | integer | no | Session duration in minutes. GitLab restart is required to apply changes. | | `shared_runners_enabled` | boolean | no | (**If enabled, requires:** `shared_runners_text` and `shared_runners_minutes`) Enable shared runners for new projects. | | `shared_runners_minutes` **(PREMIUM)** | integer | required by: `shared_runners_enabled` | Set the maximum number of CI/CD minutes that a group can use on shared runners per month. | | `shared_runners_text` | string | required by: `shared_runners_enabled` | Shared runners text. | | `sidekiq_job_limiter_mode` | string | no | `track` or `compress`. Sets the behavior for [Sidekiq job size limits](../user/admin_area/settings/sidekiq_job_limits.md). Default: 'compress'. | -| `sidekiq_job_limiter_compression_threshold_bytes` | integer | no | The threshold in bytes at which Sidekiq jobs are compressed before being stored in Redis. Default: 100 000 bytes (100KB). | +| `sidekiq_job_limiter_compression_threshold_bytes` | integer | no | The threshold in bytes at which Sidekiq jobs are compressed before being stored in Redis. Default: 100,000 bytes (100 KB). | | `sidekiq_job_limiter_limit_bytes` | integer | no | The threshold in bytes at which Sidekiq jobs are rejected. Default: 0 bytes (doesn't reject any job). | | `sign_in_text` | string | no | Text on the login page. | | `signin_enabled` | string | no | (Deprecated: Use `password_authentication_enabled_for_web` instead) Flag indicating if password authentication is enabled for the web interface. | @@ -455,7 +467,7 @@ listed in the descriptions of the relevant settings. | `slack_app_secret` **(PREMIUM)** | string | required by: `slack_app_enabled` | The app secret of the Slack-app. | | `slack_app_signing_secret` **(PREMIUM)** | string | no | The signing secret of the Slack-app. | | `slack_app_verification_token` **(PREMIUM)** | string | required by: `slack_app_enabled` | The verification token of the Slack-app. | -| `snippet_size_limit` | integer | no | Max snippet content size in **bytes**. Default: 52428800 Bytes (50MB).| +| `snippet_size_limit` | integer | no | Max snippet content size in **bytes**. Default: 52428800 Bytes (50 MB).| | `snowplow_app_id` | string | no | The Snowplow site name / application ID. (for example, `gitlab`) | | `snowplow_collector_hostname` | string | required by: `snowplow_enabled` | The Snowplow collector hostname. (for example, `snowplow.trx.gitlab.net`) | | `snowplow_cookie_domain` | string | no | The Snowplow cookie domain. (for example, `.gitlab.com`) | @@ -505,6 +517,8 @@ listed in the descriptions of the relevant settings. | `whats_new_variant` | string | no | What's new variant, possible values: `all_tiers`, `current_tier`, and `disabled`. | | `web_ide_clientside_preview_enabled` | boolean | no | Live Preview (allow live previews of JavaScript projects in the Web IDE using CodeSandbox Live Preview). | | `wiki_page_max_content_bytes` | integer | no | Maximum wiki page content size in **bytes**. Default: 52428800 Bytes (50 MB). The minimum value is 1024 bytes. | +| `jira_connect_application_key` | String | no | Application ID of the OAuth application that should be used to authenticate with the GitLab.com for Jira Cloud app | +| `jira_connect_proxy_url` | String | no | URL of the GitLab instance that should be used as a proxy for the GitLab.com for Jira Cloud app | ### Package Registry: Package file size limits diff --git a/doc/api/snippets.md b/doc/api/snippets.md index 593985b5d5f..c312642a450 100644 --- a/doc/api/snippets.md +++ b/doc/api/snippets.md @@ -20,7 +20,7 @@ Valid values for snippet visibility levels are: | Visibility | Description | |:-----------|:----------------------------------------------------| | `private` | Snippet is visible only to the snippet creator. | -| `internal` | Snippet is visible for any logged in user except [external users](../user/permissions.md#external-users). | +| `internal` | Snippet is visible for any logged in user except [external users](../user/admin_area/external_users.md). | | `public` | Snippet can be accessed without any authentication. | ## List all snippets for a user diff --git a/doc/api/status_checks.md b/doc/api/status_checks.md index e6a9c633418..7299e529bda 100644 --- a/doc/api/status_checks.md +++ b/doc/api/status_checks.md @@ -71,6 +71,186 @@ POST /projects/:id/merge_requests/:merge_request_iid/status_check_responses NOTE: `sha` must be the SHA at the `HEAD` of the merge request's source branch. +## Retry failed status check for a merge request + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/383200) in GitLab 15.7. + +For a single merge request, retry the specified failed external status check. Even +though the merge request hasn't changed, this endpoint resends the current state of +merge request to the defined external service. + +```plaintext +POST /projects/:id/merge_requests/:merge_request_iid/status_checks/:external_status_check_id/retry +``` + +**Parameters:** + +| Attribute | Type | Required | Description | +| -------------------------- | ------- | -------- | ------------------------------------- | +| `id` | integer | yes | ID of a project | +| `merge_request_iid` | integer | yes | IID of a merge request | +| `external_status_check_id` | integer | yes | ID of a failed external status check | + +## Response + +In case of success status code is 202. + +```json +{ + "message": "202 Accepted" +} +``` + +In case status check is already passed status code is 422 + +```json +{ + "message": "External status check must be failed" +} +``` + +## Example payload sent to external service + +```json +{ + "object_kind": "merge_request", + "event_type": "merge_request", + "user": { + "id": 1, + "name": "Administrator", + "username": "root", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "email": "[REDACTED]" + }, + "project": { + "id": 6, + "name": "Flight", + "description": "Ipsa minima est consequuntur quisquam.", + "web_url": "http://example.com/flightjs/Flight", + "avatar_url": null, + "git_ssh_url": "ssh://example.com/flightjs/Flight.git", + "git_http_url": "http://example.com/flightjs/Flight.git", + "namespace": "Flightjs", + "visibility_level": 20, + "path_with_namespace": "flightjs/Flight", + "default_branch": "master", + "ci_config_path": null, + "homepage": "http://example.com/flightjs/Flight", + "url": "ssh://example.com/flightjs/Flight.git", + "ssh_url": "ssh://example.com/flightjs/Flight.git", + "http_url": "http://example.com/flightjs/Flight.git" + }, + "object_attributes": { + "assignee_id": null, + "author_id": 1, + "created_at": "2022-12-07 07:53:43 UTC", + "description": "", + "head_pipeline_id": 558, + "id": 144, + "iid": 4, + "last_edited_at": null, + "last_edited_by_id": null, + "merge_commit_sha": null, + "merge_error": null, + "merge_params": { + "force_remove_source_branch": "1" + }, + "merge_status": "can_be_merged", + "merge_user_id": null, + "merge_when_pipeline_succeeds": false, + "milestone_id": null, + "source_branch": "root-master-patch-30152", + "source_project_id": 6, + "state_id": 1, + "target_branch": "master", + "target_project_id": 6, + "time_estimate": 0, + "title": "Update README.md", + "updated_at": "2022-12-07 07:53:43 UTC", + "updated_by_id": null, + "url": "http://example.com/flightjs/Flight/-/merge_requests/4", + "source": { + "id": 6, + "name": "Flight", + "description": "Ipsa minima est consequuntur quisquam.", + "web_url": "http://example.com/flightjs/Flight", + "avatar_url": null, + "git_ssh_url": "ssh://example.com/flightjs/Flight.git", + "git_http_url": "http://example.com/flightjs/Flight.git", + "namespace": "Flightjs", + "visibility_level": 20, + "path_with_namespace": "flightjs/Flight", + "default_branch": "master", + "ci_config_path": null, + "homepage": "http://example.com/flightjs/Flight", + "url": "ssh://example.com/flightjs/Flight.git", + "ssh_url": "ssh://example.com/flightjs/Flight.git", + "http_url": "http://example.com/flightjs/Flight.git" + }, + "target": { + "id": 6, + "name": "Flight", + "description": "Ipsa minima est consequuntur quisquam.", + "web_url": "http://example.com/flightjs/Flight", + "avatar_url": null, + "git_ssh_url": "ssh://example.com/flightjs/Flight.git", + "git_http_url": "http://example.com/flightjs/Flight.git", + "namespace": "Flightjs", + "visibility_level": 20, + "path_with_namespace": "flightjs/Flight", + "default_branch": "master", + "ci_config_path": null, + "homepage": "http://example.com/flightjs/Flight", + "url": "ssh://example.com/flightjs/Flight.git", + "ssh_url": "ssh://example.com/flightjs/Flight.git", + "http_url": "http://example.com/flightjs/Flight.git" + }, + "last_commit": { + "id": "141be9714669a4c1ccaa013c6a7f3e462ff2a40f", + "message": "Update README.md", + "title": "Update README.md", + "timestamp": "2022-12-07T07:52:11+00:00", + "url": "http://example.com/flightjs/Flight/-/commit/141be9714669a4c1ccaa013c6a7f3e462ff2a40f", + "author": { + "name": "Administrator", + "email": "admin@example.com" + } + }, + "work_in_progress": false, + "total_time_spent": 0, + "time_change": 0, + "human_total_time_spent": null, + "human_time_change": null, + "human_time_estimate": null, + "assignee_ids": [ + ], + "reviewer_ids": [ + ], + "labels": [ + ], + "state": "opened", + "blocking_discussions_resolved": true, + "first_contribution": false, + "detailed_merge_status": "mergeable" + }, + "labels": [ + ], + "changes": { + }, + "repository": { + "name": "Flight", + "url": "ssh://example.com/flightjs/Flight.git", + "description": "Ipsa minima est consequuntur quisquam.", + "homepage": "http://example.com/flightjs/Flight" + }, + "external_approval_rule": { + "id": 1, + "name": "QA", + "external_url": "https://example.com/" + } +} +``` + ## Get project external status checks You can request information about a project's external status checks using the following endpoint: diff --git a/doc/api/tags.md b/doc/api/tags.md index 35085baf93f..099448d5609 100644 --- a/doc/api/tags.md +++ b/doc/api/tags.md @@ -176,3 +176,56 @@ Parameters: | ---------- | -------------- | -------- | --------------------------------------------------------------------------------------------------------------- | | `id` | integer or string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | | `tag_name` | string | yes | The name of a tag | + +## Get X.509 signature of a tag + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106578) in GitLab 15.7. + +Get the [X.509 signature from a tag](../user/project/repository/x509_signed_commits/index.md#sign-commits-and-tags-with-x509-certificates), +if it is signed. Unsigned tags return a `404 Not Found` response. + +```plaintext +GET /projects/:id/repository/tags/:tag_name/signature +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | +| `tag_name` | string | yes | The name of a tag. | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/repository/tags/v1.1.1/signature" +``` + +Example response if tag is X.509 signed: + +```json +{ + "signature_type": "X509", + "verification_status": "unverified", + "x509_certificate": { + "id": 1, + "subject": "CN=gitlab@example.org,OU=Example,O=World", + "subject_key_identifier": "BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC:BC", + "email": "gitlab@example.org", + "serial_number": 278969561018901340486471282831158785578, + "certificate_status": "good", + "x509_issuer": { + "id": 1, + "subject": "CN=PKI,OU=Example,O=World", + "subject_key_identifier": "AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB:AB", + "crl_url": "http://example.com/pki.crl" + } + } +} +``` + +Example response if tag is unsigned: + +```json +{ + "message": "404 GPG Signature Not Found" +} +``` diff --git a/doc/api/templates/gitlab_ci_ymls.md b/doc/api/templates/gitlab_ci_ymls.md index b7048795313..29ee93c4e45 100644 --- a/doc/api/templates/gitlab_ci_ymls.md +++ b/doc/api/templates/gitlab_ci_ymls.md @@ -5,9 +5,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# GitLab CI YAMLs API **(FREE)** +# GitLab CI YAML API **(FREE)** -In GitLab, there is an API endpoint available to work with GitLab CI/CD YAMLs. For more +In GitLab, there is an API endpoint available to work with GitLab CI/CD YAML. For more information on CI/CD pipeline configuration in GitLab, see the [configuration reference documentation](../../ci/yaml/index.md). diff --git a/doc/api/todos.md b/doc/api/todos.md index 8fd76864f1c..2bfae1972b7 100644 --- a/doc/api/todos.md +++ b/doc/api/todos.md @@ -20,14 +20,14 @@ GET /todos Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `action` | string | no | The action to be filtered. Can be `assigned`, `mentioned`, `build_failed`, `marked`, `approval_required`, `unmergeable`, `directly_addressed` or `merge_train_removed`. | -| `author_id` | integer | no | The ID of an author | -| `project_id` | integer | no | The ID of a project | -| `group_id` | integer | no | The ID of a group | -| `state` | string | no | The state of the to-do item. Can be either `pending` or `done` | -| `type` | string | no | The type of to-do item. Can be either `Issue`, `MergeRequest`, `Commit`, `Epic`, `DesignManagement::Design` or `AlertManagement::Alert` | +| Attribute | Type | Required | Description | +| --------- | ---- | -------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `action` | string | no | The action to be filtered. Can be `assigned`, `mentioned`, `build_failed`, `marked`, `approval_required`, `unmergeable`, `directly_addressed`, `merge_train_removed` or `member_access_requested`. | +| `author_id` | integer | no | The ID of an author | +| `project_id` | integer | no | The ID of a project | +| `group_id` | integer | no | The ID of a group | +| `state` | string | no | The state of the to-do item. Can be either `pending` or `done` | +| `type` | string | no | The type of to-do item. Can be either `Issue`, `MergeRequest`, `Commit`, `Epic`, `DesignManagement::Design`, `AlertManagement::Alert` or `Namespace` | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/todos" diff --git a/doc/api/users.md b/doc/api/users.md index 19c25236f55..382d5fe03c1 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -39,7 +39,10 @@ GET /users ] ``` -You can also search for users by name, username, or public email by using `?search=`. For example. `/users?search=John`. +You can also use `?search=` to search for users by name, username, or public email. For example, `/users?search=John`. When you search for a: + +- Public email, you must use the full email address to get an exact match. +- Name or username, you do not have to get an exact match because this is a fuzzy search. In addition, you can lookup users by username: @@ -240,6 +243,11 @@ the `group_saml` provider option and `provisioned_by_group_id` parameter: ] ``` +You can also use `?search=` to search for users by name, username, or email. For example, `/users?search=John`. When you search for a: + +- Email, you must use the full email address to get an exact match. As an administrator, you can search for both public and private email addresses. +- Name or username, you do not have to get an exact match because this is a fuzzy search. + You can lookup users by external UID and provider: ```plaintext @@ -1087,6 +1095,8 @@ Parameters: ## Add SSH key +> The `usage_type` parameter was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105551) in GitLab 15.7. + Creates a new key owned by the currently authenticated user. ```plaintext @@ -1100,12 +1110,14 @@ Parameters: | `title` | string | yes | New SSH key's title | | `key` | string | yes | New SSH key | | `expires_at` | string | no | Expiration date of the SSH key in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`) | +| `usage_type` | string | no | Scope of usage for the SSH key: `auth`, `signing` or `auth_and_signing`. Default: `auth_and_signing` | ```json { "title": "ABC", "key": "ssh-dss AAAAB3NzaC1kc3MAAACBAMLrhYgI3atfrSD6KDas1b/3n6R/HP+bLaHHX6oh+L1vg31mdUqK0Ac/NjZoQunavoyzqdPYhFz9zzOezCrZKjuJDS3NRK9rspvjgM0xYR4d47oNZbdZbwkI4cTv/gcMlquRy0OvpfIvJtjtaJWMwTLtM5VhRusRuUlpH99UUVeXAAAAFQCVyX+92hBEjInEKL0v13c/egDCTQAAAIEAvFdWGq0ccOPbw4f/F8LpZqvWDydAcpXHV3thwb7WkFfppvm4SZte0zds1FJ+Hr8Xzzc5zMHe6J4Nlay/rP4ewmIW7iFKNBEYb/yWa+ceLrs+TfR672TaAgO6o7iSRofEq5YLdwgrwkMmIawa21FrZ2D9SPao/IwvENzk/xcHu7YAAACAQFXQH6HQnxOrw4dqf0NqeKy1tfIPxYYUZhPJfo9O0AmBW2S36pD2l14kS89fvz6Y1g8gN/FwFnRncMzlLY/hX70FSc/3hKBSbH6C6j8hwlgFKfizav21eS358JJz93leOakJZnGb8XlWvz1UJbwCsnR2VEY8Dz90uIk1l/UqHkA= loic@call", - "expires_at": "2016-01-21T00:00:00.000Z" + "expires_at": "2016-01-21T00:00:00.000Z", + "usage_type": "auth" } ``` @@ -1127,6 +1139,8 @@ error occurs a `400 Bad Request` is returned with a message explaining the error ## Add SSH key for user **(FREE SELF)** +> The `usage_type` parameter was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105551) in GitLab 15.7. + Create new key owned by specified user. Available only for administrator. ```plaintext @@ -1141,6 +1155,32 @@ Parameters: | `title` | string | yes | New SSH key's title | | `key` | string | yes | New SSH key | | `expires_at` | string | no | Expiration date of the SSH key in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`) | +| `usage_type` | string | no | Scope of usage for the SSH key: `auth`, `signing` or `auth_and_signing`. Default: `auth_and_signing` | + +```json +{ + "title": "ABC", + "key": "ssh-dss AAAAB3NzaC1kc3MAAACBAMLrhYgI3atfrSD6KDas1b/3n6R/HP+bLaHHX6oh+L1vg31mdUqK0Ac/NjZoQunavoyzqdPYhFz9zzOezCrZKjuJDS3NRK9rspvjgM0xYR4d47oNZbdZbwkI4cTv/gcMlquRy0OvpfIvJtjtaJWMwTLtM5VhRusRuUlpH99UUVeXAAAAFQCVyX+92hBEjInEKL0v13c/egDCTQAAAIEAvFdWGq0ccOPbw4f/F8LpZqvWDydAcpXHV3thwb7WkFfppvm4SZte0zds1FJ+Hr8Xzzc5zMHe6J4Nlay/rP4ewmIW7iFKNBEYb/yWa+ceLrs+TfR672TaAgO6o7iSRofEq5YLdwgrwkMmIawa21FrZ2D9SPao/IwvENzk/xcHu7YAAACAQFXQH6HQnxOrw4dqf0NqeKy1tfIPxYYUZhPJfo9O0AmBW2S36pD2l14kS89fvz6Y1g8gN/FwFnRncMzlLY/hX70FSc/3hKBSbH6C6j8hwlgFKfizav21eS358JJz93leOakJZnGb8XlWvz1UJbwCsnR2VEY8Dz90uIk1l/UqHkA= loic@call", + "expires_at": "2016-01-21T00:00:00.000Z", + "usage_type": "auth" +} +``` + +Returns a created key with status `201 Created` on success. If an +error occurs a `400 Bad Request` is returned with a message explaining the error: + +```json +{ + "message": { + "fingerprint": [ + "has already been taken" + ], + "key": [ + "has already been taken" + ] + } +} +``` NOTE: This also adds an audit event, as described in [audit instance events](../administration/audit_events.md#instance-events). **(PREMIUM)** diff --git a/doc/api/vulnerabilities.md b/doc/api/vulnerabilities.md index b82e2b6cbdd..6ee2bbf9811 100644 --- a/doc/api/vulnerabilities.md +++ b/doc/api/vulnerabilities.md @@ -23,9 +23,9 @@ instead. See the [GraphQL examples](#replace-vulnerability-rest-api-with-graphql Every API call to vulnerabilities must be [authenticated](index.md#authentication). -Vulnerability permissions inherit permissions from their project. If a project is -private, and a user isn't a member of the project to which the vulnerability -belongs, requests to that project returns a `404 Not Found` status code. +If an authenticated user does not have permission to +[view vulnerabilities](../user/permissions.md#project-members-permissions), +this request returns a `403 Forbidden` status code. ## Single vulnerability diff --git a/doc/api/vulnerability_exports.md b/doc/api/vulnerability_exports.md index 8c166c2ba2a..94f373c1c0a 100644 --- a/doc/api/vulnerability_exports.md +++ b/doc/api/vulnerability_exports.md @@ -19,14 +19,11 @@ Every API call to vulnerability exports must be [authenticated](index.md#authent Creates a new vulnerability export for a project. -Vulnerability export permissions inherit permissions from their project. If a project is -private and a user isn't a member of the project to which the vulnerability -belongs, requests to that project return a `404 Not Found` status code. -Vulnerability exports can be only accessed by the export's author. - If an authenticated user doesn't have permission to [create a new vulnerability](../user/permissions.md#project-members-permissions), -this request results in a `403` status code. +this request returns a `403 Forbidden` status code. + +Vulnerability exports can be only accessed by the export's author. ```plaintext POST /security/projects/:id/vulnerability_exports @@ -65,14 +62,11 @@ Example response: Creates a new vulnerability export for a group. -Vulnerability export permissions inherit permissions from their group. If a group is -private and a user isn't a member of the group to which the vulnerability -belongs, requests to that group return a `404 Not Found` status code. -Vulnerability exports can be only accessed by the export's author. - If an authenticated user doesn't have permission to [create a new vulnerability](../user/permissions.md#group-members-permissions), -this request results in a `403` status code. +this request returns a `403 Forbidden` status code. + +Vulnerability exports can be only accessed by the export's author. ```plaintext POST /security/groups/:id/vulnerability_exports diff --git a/doc/api/vulnerability_findings.md b/doc/api/vulnerability_findings.md index a236960bc68..89acbdee98a 100644 --- a/doc/api/vulnerability_findings.md +++ b/doc/api/vulnerability_findings.md @@ -16,13 +16,9 @@ To fix any broken integrations with the former Vulnerabilities API, change the ` Every API call to vulnerability findings must be [authenticated](index.md#authentication). -Vulnerability findings are project-bound entities. If a user is not -a member of a project and the project is private, a request on -that project results in a `404` status code. - -If a user is able to access the project but does not have permission to +If a user does not have permission to [use the Project Security Dashboard](../user/permissions.md#project-members-permissions), -any request for vulnerability findings of this project results in a `403` status code. +any request for vulnerability findings of this project returns a `403 Forbidden` status code. WARNING: This API is in the process of being deprecated and considered unstable. diff --git a/doc/architecture/blueprints/_template.md b/doc/architecture/blueprints/_template.md index 798d51a97ad..e39c2b51a5b 100644 --- a/doc/architecture/blueprints/_template.md +++ b/doc/architecture/blueprints/_template.md @@ -1,6 +1,6 @@ --- status: proposed -creation-date: yyyy-mm-dd +creation-date: "yyyy-mm-dd" authors: [ "@username" ] coach: "@username" approvers: [ "@product-manager", "@engineering-manager" ] @@ -10,7 +10,8 @@ participating-stages: [] <!-- **Note:** Please remove comment blocks for sections you've filled in. -When your blueprint is complete, all of these comment blocks should be removed. +When your blueprint ready for review, all of these comment blocks should be +removed. To get started with a blueprint you can use this template to inform you about what you may want to document in it at the beginning. This content will change diff --git a/doc/architecture/blueprints/ci_data_decay/index.md b/doc/architecture/blueprints/ci_data_decay/index.md index b7c3bdde2f8..6df37e28992 100644 --- a/doc/architecture/blueprints/ci_data_decay/index.md +++ b/doc/architecture/blueprints/ci_data_decay/index.md @@ -23,7 +23,7 @@ builds [continues to grow exponentially](../ci_scale/index.md). GitLab CI/CD has come a long way since the initial release, but the design of the data storage for pipeline builds remains almost the same since 2012. In 2021 we started working on database decomposition and extracting CI/CD data to -ia separate database. Now we want to improve the architecture of GitLab CI/CD +a separate database. Now we want to improve the architecture of GitLab CI/CD product to enable further scaling. ## Goals @@ -78,7 +78,7 @@ pipeline processing in such pipeline. It means that all the metadata, we store in PostgreSQL, that is needed to efficiently and reliably process builds can be safely moved to a different data store. -Currently, storing pipeline processing data is expensive as this kind of CI/CD +Storing pipeline processing data is expensive as this kind of CI/CD data represents a significant portion of data stored in CI/CD tables. Once we restrict access to processing archived pipelines, we can move this metadata to a different place - preferably object storage - and make it accessible on diff --git a/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md b/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md index d61412ae1ed..261390d1d14 100644 --- a/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md +++ b/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md @@ -75,7 +75,7 @@ incidents, over the last couple of months, for example: - S2: 2022-04-12 [Transactions detected that have been running for more than 10m](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6821) - S2: 2022-04-06 [Database contention plausibly caused by excessive `ci_builds` reads](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6773) - S2: 2022-03-18 [Unable to remove a foreign key on `ci_builds`](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6642) -- S2: 2022-10-10 [The queuing_queries_duration SLI apdex violating SLO](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/7852#note_1130123525) +- S2: 2022-10-10 [The `queuing_queries_duration` SLI apdex violating SLO](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/7852#note_1130123525) We have approximately 50 `ci_*` prefixed database tables, and some of them would benefit from partitioning. @@ -180,6 +180,49 @@ respective database tables. Using `RANGE` partitioning works similarly to using of `partition_id` values, using `RANGE` partitioning might be a better strategy. +### Multi-project pipelines + +Parent-child pipeline will always be part of the same partition because child +pipelines are considered a resource of the parent pipeline. They can't be +viewed individually in the project pipeline list page. + +On the other hand, multi-project pipelines can be viewed in the pipeline list page. +They can also be accessed from the pipeline graph as downstream/upstream links +when created via the `trigger` token or the API using a job token. +They can also be created from other pipelines by using trigger tokens, but in this +case we don't store the source pipeline. + +While partitioning `ci_builds` we need to update the foreign keys to the +`ci_sources_pipelines` table: + +```plain +Foreign-key constraints: + "fk_be5624bf37" FOREIGN KEY (source_job_id) REFERENCES ci_builds(id) ON DELETE CASCADE + "fk_d4e29af7d7" FOREIGN KEY (source_pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE + "fk_e1bad85861" FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE +``` + +A `ci_sources_pipelines` record references two `ci_pipelines` rows (parent and +the child). Our usual strategy has been to add a `partition_id` to the +table, but if we do it here we will force all multi-project +pipelines to be part of the same partition. + +We should add two `partition_id` columns for this table, a +`partition_id` and a `source_partition_id`: + +```plain +Foreign-key constraints: + "fk_be5624bf37" FOREIGN KEY (source_job_id, source_partition_id) REFERENCES ci_builds(id, source_partition_id) ON DELETE CASCADE + "fk_d4e29af7d7" FOREIGN KEY (source_pipeline_id, source_partition_id) REFERENCES ci_pipelines(id, source_partition_id) ON DELETE CASCADE + "fk_e1bad85861" FOREIGN KEY (pipeline_id, partition_id) REFERENCES ci_pipelines(id, partition_id) ON DELETE CASCADE +``` + +This solution is the closest to a two way door decision because: + +- We retain the ability to reference pipelines in different partitions. +- If we later decide that we want to force multi-project pipelines in the same partition + we could add a constraint to validate that both columns have the same value. + ## Why do we want to use explicit logical partition ids? Partitioning CI/CD data using a logical `partition_id` has several benefits. We @@ -248,10 +291,9 @@ smart enough to move rows between partitions on its own. A partitioned table is called a **routing** table and it will use the `p_` prefix which should help us with building automated tooling for query analysis. -A table partition will be called **partition** and it can use the a -physical partition ID as suffix, leaded by a `p` letter, for example -`ci_builds_p101`. Existing CI tables will become **zero partitions** of the -new routing tables. Depending on the chosen +A table partition will be called **partition** and it can use the a physical +partition ID as suffix, for example `ci_builds_101`. Existing CI tables will +become **zero partitions** of the new routing tables. Depending on the chosen [partitioning strategy](#how-do-we-want-to-partition-cicd-data) for a given table, it is possible to have many logical partitions per one physical partition. @@ -273,6 +315,37 @@ during a low traffic period([after `00:00 UTC`](https://dashboards.gitlab.net/d/ See an example of this strategy in our [partition tooling](../../../development/database/table_partitioning.md#step-6---create-parent-table-and-attach-existing-table-as-the-initial-partition)). +### Partitioning steps + +The database [partition tooling](../../../development/database/table_partitioning.md#partitioning-a-table-list) +docs contain a list of steps to partition a table, but the steps are not enough +for our iterative strategy. As our dataset continues to grow we want to take +advantage of partitioning performance right away and not wait until all tables +are partitioned. For example, after partitioning the `ci_builds_metadata` table +we want to start writing and reading data to/from a new partition. This means +that we will increase the `partition_id` value from `100`, the default value, +to `101`. Now all of the new resources for the pipeline hierarchy will be +persisted with `partition_id = 101`. We can continue following the database +tooling instructions for the next table that will be partitioned, but we require +a few extra steps: + +- add `partition_id` column for the FK references with default value of `100` + since the majority of records should have that value. +- change application logic to cascade the `partition_id` value +- correct `partition_id` values for recent records with a post deploy/background + migration, similar to this: + + ```sql + UPDATE ci_pipeline_metadata + SET partition_id = ci_pipelines.partition_id + FROM ci_pipelines + WHERE ci_pipelines.id = ci_pipeline_metadata.pipeline_id + AND ci_pipelines.partition_id in (101, 102); + ``` + +- change the foreign key definitions +- ... + ## Storing partitions metadata in the database To build an efficient mechanism that will be responsible for creating @@ -297,7 +370,7 @@ system - any letter from `g` to `z` in Latin alphabet, for example `x`. In that case an example of an URI would look like `1e240x5ba0`. If we decide to update the primary identifier of a partitioned resource (today it is just a big integer) it is important to design a system that is resilient to migrating data -between partitions, to avoid changing idenfiers when rebalancing happens. +between partitions, to avoid changing identifiers when rebalancing happens. `ci_partitions` table will store information about a partition identifier, pipeline ids range it is valid for and whether the partitions have been diff --git a/doc/architecture/blueprints/ci_pipeline_components/index.md b/doc/architecture/blueprints/ci_pipeline_components/index.md index a3c72227f3e..100c9e67fda 100644 --- a/doc/architecture/blueprints/ci_pipeline_components/index.md +++ b/doc/architecture/blueprints/ci_pipeline_components/index.md @@ -3,7 +3,7 @@ status: proposed creation-date: "2022-09-14" authors: [ "@fabio", "@grzesiek" ] coach: "@kamil" -approvers: [ "@dov" ] +approvers: [ "@dhershkovitch", "@marknuzzo" ] owning-stage: "~devops::verify" participating-stages: [] --- @@ -110,7 +110,8 @@ identifying abstract concepts and are subject to changes as we refine the design ## Definition of pipeline component -A pipeline component is a reusable single-purpose building block that abstracts away a single pipeline configuration unit. Components are used to compose a part or entire pipeline configuration. +A pipeline component is a reusable single-purpose building block that abstracts away a single pipeline configuration unit. +Components are used to compose a part or entire pipeline configuration. It can optionally take input parameters and set output data to be adaptable and reusable in different pipeline contexts, while encapsulating and isolating implementation details. @@ -133,27 +134,319 @@ For best experience with any systems made of components it's fundamental that co The version identifies the exact interface and behavior of the component. - **Resolvable**: when a component depends on another component, this dependency must be explicit and trackable. -## Proposal +## Structure of a component -Prerequisites to create a component: +A pipeline component is identified by the path to a repository or directory that defines it +and a specific version: `<component-path>@<version>`. -- Create a project. Description and avatar are highly recommended to improve discoverability. -- Add a `README.md` in the top level directory that documents the component. - What it does, how to use it, how to contribute, etc. - This file is mandatory. -- Add a `.gitlab-ci.yml` in the top level directory to test that the components works as expected. - This file is highly recommended. +For example: `gitlab-org/dast@1.0`. -Characteristics of a component: +### The component path -- It must have a **name** to be referenced to and **description** for extra details. -- It must specify its **type** which defines how it can be used (raw configuration to be `include`d, child pipeline workflow, job step). -- It must define its **content** based on the type. -- It must specify **input parameters** that it accepts. Components should depend on input parameters for dynamic values and not environment variables. -- It can optionally define **output data** that it returns. -- Its YAML specification should be **validated statically** (for example: using JSON schema validators). -- It should be possible to use specific **versions** of a component by referencing official releases and SHA. -- It should be possible to use components defined locally in the same repository. +A component path must contain at least the metadata YAML and optionally a related `README.md` documentation file. + +The component path can be: + +- A path to a project: `gitlab-org/dast`. In this case the 2 files are defined in the root directory of the repository. +- A path to a project subdirectory: `gitlab-org/dast/api-scan`. In this case the 2 files are defined in the `api-scan` directory. +- A path to a local directory: `/path/to/component`. This path must contain the metadata YAML that defines the component. + The path must start with `/` to indicate a full path in the repository. + +The metadata YAML file follows the filename convention `gitlab-<component-type>.yml` where component type is one of: + +| Component type | Context | +| -------------- | ------- | +| `template` | For components used under `include:` keyword | +| `step` | For components used under `steps:` keyword | +| `workflow` | For components used under `trigger:` keyword | + +Based on the context where the component is used we fetch the correct YAML file. +For example, if we are including a component `gitlab-org/dast@1.0` we expect a YAML file named `gitlab-template.yml` in the +top level directory of `gitlab-org/dast` repository. + +A `gitlab-<component-type>.yml` file: + +- Must have a **name** to be referenced to and **description** for extra details. +- Must specify its **type** in the filename, which defines how it can be used (raw configuration to be `include`d, child pipeline workflow, job step). +- Must define its **content** based on the type. +- Must specify **input parameters** that it accepts. Components should depend on input parameters for dynamic values and not environment variables. +- Can optionally define **output data** that it returns. +- Should be **validated statically** (for example: using JSON schema validators). + +```yaml +spec: + inputs: + website: + environment: + default: test + test_run: + options: + - unit + - integration + - system +content: { ... } +``` + +Components that are released in the catalog must have a `README.md` file in the same directory as the +metadata YAML file. The `README.md` represents the documentation for the specific component, hence it's recommended +even when not releasing versions in the catalog. + +### The component version + +The version of the component can be (in order of highest priority first): + +1. A commit SHA - For example: `gitlab-org/dast@e3262fdd0914fa823210cdb79a8c421e2cef79d8` +1. A released tag - For example: `gitlab-org/dast@1.0` +1. A special moving target version that points to the most recent released tag - For example: `gitlab-org/dast@~latest` +1. An unreleased tag - For example: `gitlab-org/dast@rc-1.0` +1. A branch name - For example: `gitlab-org/dast@master` + +If a tag and branch exist with the same name, the tag takes precedence over the branch. +Similarly, if a tag is named `e3262fdd0914fa823210cdb79a8c421e2cef79d8`, a commit SHA (if exists) +takes precedence over the tag. + +As we want to be able to reference any revisions (even those not released), a component must be defined in a Git repository. + +NOTE: +When referencing a component by local path (for example `./path/to/component`), its version is implicit and matches +the commit SHA of the current pipeline context. + +## Components project + +A components project is a GitLab project/repository that exclusively hosts one or more pipeline components. + +For components projects it's highly recommended to set an appropriate avatar and project description +to improve discoverability in the catalog. + +### Structure of a components project + +A project can host one or more components depending on whether the author wants to define a single component +per project or include multiple cohesive components under the same project. + +Let's imagine we are developing a component that runs RSpec tests for a Rails app. We create a component project +called `myorg/rails-rspec`. + +The following directory structure would support 1 component per project: + +```plaintext +. +├── gitlab-<type>.yml +├── README.md +└── .gitlab-ci.yml +``` + +The `.gitlab-ci.yml` is recommended for the project to ensure changes are verified accordingly. + +The component is now identified by the path `myorg/rails-rspec`. In other words, this means that +the `gitlab-<type>.yml` and `README.md` are located in the root directory of the repository. + +The following directory structure would support multiple components per project: + +```plaintext +. +├── .gitlab-ci.yml +├── unit/ +│ ├── gitlab-workflow.yml +│ └── README.md +├── integration/ +│ ├── gitlab-workflow.yml +│ └── README.md +└── feature/ + ├── gitlab-workflow.yml + └── README.md +``` + +In this example we are defining multiple test profiles that are executed with RSpec. +The user could choose to use one or more of these. + +Each of these components are identified by their path `myorg/rails-rspec/unit`, `myorg/rails-rspec/integration` +and `myorg/rails-rspec/feature`. + +This directory structure could also support both strategies: + +```plaintext +. +├── gitlab-template.yml # myorg/rails-rspec +├── README.md +├── .gitlab-ci.yml +├── unit/ +│ ├── gitlab-workflow.yml # myorg/rails-rspec/unit +│ └── README.md +├── integration/ +│ ├── gitlab-workflow.yml # myorg/rails-rspec/integration +│ └── README.md +└── feature/ + ├── gitlab-workflow.yml # myorg/rails-rspec/feature + └── README.md +``` + +With the above structure we could have a top-level component that can be used as the +default component. For example, `myorg/rails-rspec` could run all the test profiles together. +However, more specific test profiles could be used separately (for example `myorg/rails-rspec/integration`). + +NOTE: +Any nesting more than 1 level is initially not permitted. +This limitation encourages cohesion at project level and keeps complexity low. + +## Input parameters `spec:inputs:` parameters + +If the component takes any input parameters they must be specified according to the following schema: + +```yaml +spec: + inputs: + website: # by default all declared inputs are mandatory. + environment: + default: test # apply default if not provided. This makes the input optional. + test_run: + options: # a choice must be made from the list since there is no default value. + - unit + - integration + - system +``` + +When using the component we pass the input parameters as follows: + +```yaml +include: + - component: org/my-component@1.0 + with: + website: ${MY_WEBSITE} # variables expansion + test_run: system + environment: $[[ inputs.environment ]] # interpolation of upstream inputs +``` + +Variables expansion must be supported for `with:` syntax as well as interpolation of +possible [inputs provided upstream](#input-parameters-for-pipelines). + +Input parameters are validated as soon as possible: + +1. Read the file `gitlab-template.yml` inside `org/my-component`. +1. Parse `spec:inputs` and validate the parameters against this schema. +1. If successfully validated, proceed with parsing `content:`. Return an error otherwise. +1. Interpolate input parameters inside the component's `content:`. + +```yaml +spec: + inputs: + environment: + options: [test, staging, production] +content: + "run-tests-$[[ inputs.environment ]]": + script: ./run-test + + scan-website: + script: ./scan-website $[[ inputs.environment ]] + rules: + - if: $[[ inputs.environment ]] == 'staging' + - if: $[[ inputs.environment ]] == 'production' +``` + +With `$[[ inputs.XXX ]]` inputs are interpolated immediately after parsing the `content:`. + +### Why input parameters and not environment variables? + +Until today we have been leveraging environment variables to pass information around. +For example, we use environment variables to pass information from an upstream pipeline to a +downstream pipeline. + +Using environment variables for passing information to a component is like declaring global +variables in programming languages. The more variables we declare the more we risk variable +conflicts and increase variables scope. + +Input parameters are like variables passed to the component which exist inside a specific +scope and they don't leak to the outside. +Inputs are not inherited from upstream `include`s. They must be passed explicitly. + +This paradigm allows to build more robust and isolated components as well as declare and +enforce contracts. + +### Input parameters for existing `include:` syntax + +Because we are adding input parameters to components used via `include:component` we have an opportunity to +extend it to other `include:` types support inputs via `with:` syntax: + +```yaml +include: + - component: org/my-component@1.0 + with: + foo: bar + - local: path/to/file.yml + with: + foo: bar + - project: org/another + file: .gitlab-ci.yml + with: + foo: bar + - remote: http://example.com/ci/config + with: + foo: bar + - template: Auto-DevOps.gitlab-ci.yml + with: + foo: bar +``` + +Then the configuration being included must specify the inputs: + +```yaml +spec: + inputs: + foo: + +# rest of the configuration +``` + +If a YAML includes content using `with:` but the including YAML doesn't specify `inputs:`, an error should be raised. + +|`with:`| `inputs:` | result | +| --- | --- | --- | +| specified | | raise error | +| specified | specified | validate inputs | +| | specified | use defaults | +| | | legacy `include:` without input passing | + +### Input parameters for pipelines + +Inputs can also be used to pass parameters to a pipeline when triggered and benefit from immediate validation. + +Today we have different use cases where using explicit input parameters would be beneficial: + +1. `Run Pipeline` UI form. + - **Problem today**: We are using top-level variables with `variables:*:description` to surface environment variables to the UI. + The problem with this is the mix of responsibilities as well as the jump in [precedence](../../../ci/variables/index.md#cicd-variable-precedence) + that a variable gets (from a YAML variable to a pipeline variable). + Building validation and features on top of this solution is challenging and complex. +1. Trigger a pipeline via API. For example `POST /projects/:id/pipelines/trigger` with `{ inputs: { provider: 'aws' } }` +1. Trigger a pipeline via `trigger:` syntax. + +```yaml +deploy-app: + trigger: + project: org/deployer + with: + provider: aws + deploy_environment: staging +``` + +To solve the problem of `Run Pipeline` UI form we could fully leverage the `spec:inputs` schema: + +```yaml +spec: + inputs: + concurrency: + default: 10 # displayed as default value in the input box + provider: # can enforce `required` in the form validation + description: Deployment provider # optional: render as input label. + deploy_environment: + options: # render a selectbox with options in order of how they are defined below + - staging # 1st option + - canary # 2nd option + - production # 3rd option + default: staging # selected by default in the UI. + # if `default:` is not specified, the user must explicitly select + # an option. + description: Deployment environment # optional: render as input label. +``` ## Limits @@ -188,3 +481,34 @@ Some limits we could consider adding: - Allow self-managed administrators to populate their self-managed catalog by importing/updating components from GitLab.com or from repository exports. - Iterate on feedback. + +## Who + +Proposal: + +<!-- vale gitlab.Spelling = NO --> + +| Role | Who +|--------------------------------|-------------------------| +| Author | Fabio Pitino | +| Engineering Leaders | Cheryl Li, Mark Nuzzo | +| Product Manager | Dov Hershkovitch | +| Architecture Evolution Coaches | Kamil Trzciński, Grzegorz Bizon | + +DRIs: + +| Role | Who +|------------------------------|------------------------| +| Leadership | Mark Nuzzo | +| Product | Dov Hershkovitch | +| Engineering | Fabio Pitino | +| UX | Kevin Comoli (interim), Sunjung Park | + +Domain experts: + +| Area | Who +|------------------------------|------------------------| +| Verify / Pipeline authoring | Avielle Wolfe | +| Verify / Pipeline authoring | Laura Montemayor-Rodriguez | + +<!-- vale gitlab.Spelling = YES --> diff --git a/doc/architecture/blueprints/ci_scale/index.md b/doc/architecture/blueprints/ci_scale/index.md index c02fb35974b..574a79c86a5 100644 --- a/doc/architecture/blueprints/ci_scale/index.md +++ b/doc/architecture/blueprints/ci_scale/index.md @@ -1,8 +1,10 @@ --- -stage: none -group: unassigned -comments: false -description: 'Improve scalability of GitLab CI/CD' +status: in progress +creation-date: "2021-01-21" +authors: [ "@grzesiek" ] +coach: "@grzesiek" +approvers: [ "@cheryl.li", "@jreporter" ] +owning-stage: "~devops::verify" --- # CI/CD Scaling diff --git a/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md b/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md index 53f38fa85fd..3ef98c33035 100644 --- a/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md +++ b/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md @@ -74,7 +74,7 @@ This blueprint explicitly talks about **horizontal** split and **Application Lay The Bounded Contexts is a topic that was discussed extensively number of times for a couple of years. Reflected in number of issues: -- [Create new models / classes within a module / namespace](https://gitlab.com/gitlab-org/gitlab/-/issues/212156) +- [Create new models / classes in a module / namespace](https://gitlab.com/gitlab-org/gitlab/-/issues/212156) - [Make teams to be maintainers of their code](https://gitlab.com/gitlab-org/gitlab/-/issues/25872) - [Use nested structure to organize CI classes](https://gitlab.com/gitlab-org/gitlab/-/issues/209745) - [WIP: Make it simple to build and use "Decoupled Services"](https://gitlab.com/gitlab-org/gitlab/-/issues/31121) @@ -86,7 +86,7 @@ We are partially executing a **Bounded Contexts** idea: - Since we use namespaces, individual contributor or reviewer can know who to reach from domain experts about help with the given context -The module namespaces are actively being used today to model codebase around team boundaries. Currently, the most +The module namespaces are actively being used today to model codebase around team boundaries. The most prominent namespaces being used today are `Ci::` and `Packages::`. They provide a good way to contain the code owned by a group in a well-defined structure. @@ -125,7 +125,7 @@ application layers. This list is not exhaustive, but shows a general list of the - Web Packages API: provide a REST API compatible with the packaging tools: Debian, Maven, Container Registry Proxy, etc. - Git nodes: all code required to authorize `git pull/push` over `SSH` or `HTTPS` - Sidekiq: run background jobs -- Services/Models/DB: all code required to maintain our database structure, data validation, business logic and policies models that needs to be shared with other components +- Services/Models/DB: all code required to maintain our database structure, data validation, business logic, and policies models that needs to be shared with other components The best way to likely describe how the actual GitLab Rails split would look like. It is a satellite model. Where we have a single core, that is shared across all satellite components. The design of that implies @@ -301,7 +301,7 @@ All work can be found in these merge requests: - [Draft: PoC - Move GraphQL to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50180) - [Draft: PoC - Move Controllers and Grape API:API to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53720) - [Draft: PoC - Move only Grape API:API to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53982) -- [Measure performance impact for proposed web_engine](https://gitlab.com/gitlab-org/gitlab/-/issues/300548) +- [Measure performance impact for proposed `web_engine`](https://gitlab.com/gitlab-org/gitlab/-/issues/300548) What was done? @@ -328,8 +328,8 @@ What was done? 1. Move gems to the `engines/web_engine/` - - We moved all GraphQL gems to the actual web_engine Gemfile - - We moved Grape API gem to the actual web_engine Gemfile + - We moved all GraphQL gems to the actual `web_engine` Gemfile + - We moved Grape API gem to the actual `web_engine` Gemfile ```ruby Gem::Specification.new do |spec| @@ -344,9 +344,9 @@ What was done? 1. Move routes to the `engines/web_engine/config/routes.rb` file - - We moved GraphQL routes to the web_engine routes. - - We moved API routes to the web_engine routes. - - We moved most of the controller routes to the web_engine routes. + - We moved GraphQL routes to the `web_engine` routes. + - We moved API routes to the `web_engine` routes. + - We moved most of the controller routes to the `web_engine` routes. ```ruby Rails.application.routes.draw do @@ -367,7 +367,7 @@ What was done? 1. Connect GitLab application with the WebEngine - In GitLab Gemfile.rb, add web_engine to the engines group + In GitLab Gemfile.rb, add `web_engine` to the engines group ```ruby # Gemfile @@ -376,7 +376,7 @@ What was done? end ``` - Since the gem is inside :engines group, it will not be automatically required by default. + Since the gem is inside :engines group, it is not automatically required by default. 1. Configure GitLab when to load the engine. @@ -432,7 +432,7 @@ What was done? - We control specs from main application using environment variable `TEST_WEB_ENGINE` - We added new CI job that will run `engines/web_engine/spec` tests separately using `TEST_WEB_ENGINE` environment variable. - We added new CI job that will run `engines/web_engine/ee/spec` tests separately using `TEST_WEB_ENGINE` environment variable. - - We are running all whitebox frontend tests with `TEST_WEB_ENGINE=true` + - We are running all white box frontend tests with `TEST_WEB_ENGINE=true` #### Results @@ -451,7 +451,7 @@ Savings on Sidekiq `start-up` event, for a single Sidekiq cluster without GraphQ - Boot-up time was reduced from 45.31 to 21.80 seconds. It was 23.51 seconds faster (51.89%) - We have 805,772 less live objects, 4,587,535 less allocated objects, 2,866 less allocated pages and 3.65 MB less allocated space for objects outside of the heap - We loaded 2,326 less code files (15.64%) -- We reduced the duration of a single full GC cycle from 0.80s to 0.70 (12.64%) +- We reduced the duration of a single full GC cycle from 0.80 seconds to 0.70 seconds (12.64%) Puma single, showed very little difference as expected. @@ -461,20 +461,20 @@ More details can be found in the [issue](https://gitlab.com/gitlab-org/gitlab/-/ Estimating the results for the scale of running GitLab.com, today we use: -- Currently individual GC cycle takes around [130ms for Web](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=avg(rate(ruby_gc_duration_seconds_sum%7Bstage%3D%22main%22%2Ctype%3D%22web%22%7D%5B5m%5D)%2Frate(ruby_gc_duration_seconds_count%5B5m%5D))&g0.tab=0) - and [200ms for Sidekiq](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=avg(rate(ruby_gc_duration_seconds_sum%7Bstage%3D%22main%22%2Ctype%3D%22sidekiq%22%7D%5B5m%5D)%2Frate(ruby_gc_duration_seconds_count%5B5m%5D))&g0.tab=0) on GitLab.com +- Individual GC cycle takes around [130 ms for Web](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=avg(rate(ruby_gc_duration_seconds_sum%7Bstage%3D%22main%22%2Ctype%3D%22web%22%7D%5B5m%5D)%2Frate(ruby_gc_duration_seconds_count%5B5m%5D))&g0.tab=0) + and [200 ms for Sidekiq](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=avg(rate(ruby_gc_duration_seconds_sum%7Bstage%3D%22main%22%2Ctype%3D%22sidekiq%22%7D%5B5m%5D)%2Frate(ruby_gc_duration_seconds_count%5B5m%5D))&g0.tab=0) on GitLab.com - On average we do around [2 GC cycles per-second](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.end_input=2021-02-17%2017%3A56&g0.max_source_resolution=0s&g0.expr=avg(rate(ruby_gc_duration_seconds_count%7Bstage%3D%22main%22%2Ctype%3D%22web%22%7D%5B5m%5D))&g0.tab=0) or [0.12 cycles per second for Sidekiq](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.end_input=2021-02-17%2017%3A56&g0.max_source_resolution=0s&g0.expr=avg(rate(ruby_gc_duration_seconds_count%7Bstage%3D%22main%22%2Ctype%3D%22sidekiq%22%7D%5B5m%5D))&g0.tab=0) - This translates to using [around 9.5 vCPUs per-second for Web](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=sum(rate(ruby_gc_duration_seconds_sum%7Bstage%3D%22main%22%2Ctype%3D%22web%22%7D%5B5m%5D))&g0.tab=0) and [around 8 vCPUs per-second for Sidekiq](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=sum(rate(ruby_gc_duration_seconds_sum%7Bstage%3D%22main%22%2Ctype%3D%22sidekiq%22%7D%5B5m%5D))&g0.tab=0) of spend on GC alone -- Sidekiq [uses 2.1GB on average](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=max(ruby_process_unique_memory_bytes%7Btype%3D%22sidekiq%22%7D)%2F1024%2F1024%2F1024&g0.tab=1) - or [550GB in total](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=sum(ruby_process_unique_memory_bytes%7Btype%3D%22sidekiq%22%7D)%2F1024%2F1024%2F1024&g0.tab=0) of memory on GitLab.com +- Sidekiq [uses 2.1 GB on average](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=max(ruby_process_unique_memory_bytes%7Btype%3D%22sidekiq%22%7D)%2F1024%2F1024%2F1024&g0.tab=1) + or [550 GB in total](https://thanos-query.ops.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=sum(ruby_process_unique_memory_bytes%7Btype%3D%22sidekiq%22%7D)%2F1024%2F1024%2F1024&g0.tab=0) of memory on GitLab.com We estimate the possible maximum savings for introducing `web_engine`: -- Reduce a GC cycle time by 20%, from to 200ms to 160ms +- Reduce a GC cycle time by 20%, from to 200 ms to 160 ms - The amount of GC cycles per-second would stay the same, but due to GC cycle time reduction we would use around 6 vCPUs instead of 8 vCPUs -- In the best case we would be looking at Sidekiq alone we would be estimating to save up-to 137GB of memory on GitLab.com +- In the best case we would be looking at Sidekiq alone we would be estimating to save up-to 137 GB of memory on GitLab.com This model could be extended to introduce `sidekiq_engine` giving a similar benefits (even more important due to visible impact on users) for Web nodes. @@ -498,14 +498,14 @@ Cons: - It is harder to implement GraphQL subscriptions as in case of Sidekiq as we need another way to pass subscriptions - `api_v4` paths can be used in some services that are used by Sidekiq (for example `api_v4_projects_path`) -- url_helpers paths are used in models and services, that could be used by Sidekiq (for example `Gitlab::Routing.url_helpers.project_pipelines_path` is used by [ExpirePipelineCacheService](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/ci/expire_pipeline_cache_service.rb#L20) in [ExpirePipelineCacheWorker](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/expire_pipeline_cache_worker.rb#L18)) +- `url_helpers` paths are used in models and services, that could be used by Sidekiq (for example `Gitlab::Routing.url_helpers.project_pipelines_path` is used by [ExpirePipelineCacheService](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/ci/expire_pipeline_cache_service.rb#L20) in [ExpirePipelineCacheWorker](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/expire_pipeline_cache_worker.rb#L18)) #### Example: GraphQL [Draft: PoC - Move GraphQL to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50180) - The [99% of changes](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50180/diffs?commit_id=49c9881c6696eb620dccac71532a3173f5702ea8) as visible in the above MRs is moving files as-is. -- The [actual work](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50180/diffs?commit_id=1d9a9edfa29ea6638e7d8a6712ddf09f5be77a44) on fixing cross-dependencies, specs, and configuring web_engine +- The [actual work](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50180/diffs?commit_id=1d9a9edfa29ea6638e7d8a6712ddf09f5be77a44) on fixing cross-dependencies, specs, and configuring `web_engine` - We [adapted](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50180/diffs?commit_id=d7f862cc209ce242000b2aec88ff7f4485acdd92) CI to test `engines/web_engine/` as a self-sufficient component of stack Today, loading GraphQL requires a bunch of [dependencies](https://gitlab.com/gitlab-org/gitlab/-/issues/288044): @@ -581,7 +581,7 @@ to be created to ensure that we do not have explosion of engines. - [Draft: PoC - Move GraphQL to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50180) - [Draft: PoC - Move Controllers and Grape API:API to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53720) - [Draft: PoC - Move only Grape API:API to the WebEngine](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53982) -- [Measure performance impact for proposed web_engine](https://gitlab.com/gitlab-org/gitlab/-/issues/300548) +- [Measure performance impact for proposed `web_engine`](https://gitlab.com/gitlab-org/gitlab/-/issues/300548) - [Create new models / classes within a module / namespace](https://gitlab.com/gitlab-org/gitlab/-/issues/212156) - [Make teams to be maintainers of their code](https://gitlab.com/gitlab-org/gitlab/-/issues/25872) - [Use nested structure to organize CI classes](https://gitlab.com/gitlab-org/gitlab/-/issues/209745) diff --git a/doc/architecture/blueprints/container_registry_metadata_database/index.md b/doc/architecture/blueprints/container_registry_metadata_database/index.md index 63e27286756..f3bcf1e4e59 100644 --- a/doc/architecture/blueprints/container_registry_metadata_database/index.md +++ b/doc/architecture/blueprints/container_registry_metadata_database/index.md @@ -258,7 +258,7 @@ The expected registry behavior will be covered with integration tests, using a p If unable to pull a connection from the pool to serve a given request, the registry will timeout and return an HTTP `500 Internal Server Error` error to the client and report the error to Sentry. These issues should trigger a development escalation to investigate why the pool is being exhausted. There might be too much load for the preconfigured pool size, or there might be transactions holding on to connections for too long. -Prometheus metrics should be used to create alerts to act on a possible saturation before the application starts erroring. Special attention will be paid to these scenarios during the gradual migration of the GitLab.com registry, where we will have limited, gradual, and controlled exposure on the new registry nodes. During that process, we can identify usage patterns, observe metrics, and fine tune both infrastructure and application settings accordingly as the load increases. If needed, a rate limiting algorithm may be applied to limit impact. Decisions will be based on real data to avoid overly restrictive measures and premature optimizations. +Prometheus metrics should be used to create alerts to act on a possible saturation before the application starts returning errors. Special attention will be paid to these scenarios during the gradual migration of the GitLab.com registry, where we will have limited, gradual, and controlled exposure on the new registry nodes. During that process, we can identify usage patterns, observe metrics, and fine tune both infrastructure and application settings accordingly as the load increases. If needed, a rate limiting algorithm may be applied to limit impact. Decisions will be based on real data to avoid overly restrictive measures and premature optimizations. The expected registry behavior will be covered with integration tests by manipulating the pool size and spawning multiple concurrent requests against the API, putting pressure on the pool and eventually exhausting its capacity. @@ -294,7 +294,7 @@ Together, these resources should provide an adequate level of insight into the r GitLab ships with the GitLab Container Registry by default, but it's also compatible with third-party registries, as long as they comply with the [Docker Distribution V2 Specification](https://docs.docker.com/registry/spec/api/), now superseded by the [Open Container Initiative (OCI) Image Specification](https://github.com/opencontainers/image-spec/blob/master/spec.md). -So far, we strived to maintain full compatibility with third-party registries when adding new features. For example, in 12.8, we introduced a new [tag delete feature](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23325) to delete a single tag without deleting the underlying manifest. Because this feature is not part of the Docker or OCI specifications, we have kept the previous behavior as a fallback option to maintain compatibility with third-party registries. +So far, we have tried to maintain full compatibility with third-party registries when adding new features. For example, in 12.8, we introduced a new [tag delete feature](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23325) to delete a single tag without deleting the underlying manifest. Because this feature is not part of the Docker or OCI specifications, we have kept the previous behavior as a fallback option to maintain compatibility with third-party registries. However, this will likely change in the future. Apart from online garbage collection, and as described in [challenges](#challenges), the metadata database will unblock the implementation of many requested features for the GitLab Container Registry in the mid/long term. Most of these features will only be available for instances using the GitLab Container Registry. They are not part of the Docker Distribution or OCI specifications, neither we will be able to provide a compatible fallback option. @@ -316,7 +316,7 @@ All GitLab Rails features dependent on a specific version of the registry should This is already done to determine whether a tag should be deleted using the new tag delete feature (only available in the GitLab Container Registry v2.8.1+) or the old method. In this case, GitLab Rails sends an `OPTIONS` request to the registry tag route to determine whether the `DELETE` method is supported or not. -Alternatively, and as the universal long-term solution, we need to determine the registry vendor, version, and supported features (the last two are only applicable if the vendor is GitLab) and persist it in the GitLab Rails database. This information can then be used in realtime to toggle features or fallback to alternative methods, if possible. The initial implementation of this approach was introduced as part of [#204839](https://gitlab.com/gitlab-org/gitlab/-/issues/204839). Currently, it's only used for metrics purposes. Further improvements are required to guarantee that the version information is kept up to date in self-managed instances, where the registry may be hot swapped. +Alternatively, and as the universal long-term solution, we need to determine the registry vendor, version, and supported features (the last two are only applicable if the vendor is GitLab) and persist it in the GitLab Rails database. This information can then be used in real time to toggle features or fallback to alternative methods, if possible. The initial implementation of this approach was introduced as part of [#204839](https://gitlab.com/gitlab-org/gitlab/-/issues/204839). Currently, it's only used for metrics purposes. Further improvements are required to guarantee that the version information is kept up to date in self-managed instances, where the registry may be hot swapped. ##### Release and Deployment diff --git a/doc/architecture/blueprints/database/scalability/patterns/time_decay.md b/doc/architecture/blueprints/database/scalability/patterns/time_decay.md index 93f5dffd3f5..2b36a43a6db 100644 --- a/doc/architecture/blueprints/database/scalability/patterns/time_decay.md +++ b/doc/architecture/blueprints/database/scalability/patterns/time_decay.md @@ -154,7 +154,7 @@ factors: The perfect partitioning scheme keeps **all queries over a dataset almost always over a single partition**, with some cases going over two partitions and seldom over multiple partitions being an acceptable balance. We should also target for **partitions that are as small as possible**, below -5-10M records and/or 10GB each maximum. +5-10M records and/or 10 GB each maximum. Partitioning can be combined with other strategies to either prune (drop) old partitions, move them to cheaper storage inside the database or move them outside of the database (archive or use of other @@ -241,7 +241,7 @@ Related epic: [Partitioning: `web_hook_logs` table](https://gitlab.com/groups/gi The important characteristics of `web_hook_logs` are the following: 1. Size of the dataset: it is a really large table. At the moment we decided to - partition it (`2021-03-01`), it had roughly 527M records and a total size of roughly 1TB + partition it (`2021-03-01`), it had roughly 527M records and a total size of roughly 1 TB - Table: `web_hook_logs` - Rows: approximately 527M @@ -261,7 +261,7 @@ As a result, on March 2021 there were still not deleted records since July 2020 increasing in size by more than 2 million records per day instead of staying at a more or less stable size. -Finally, the rate of inserts has grown to more than 170GB of data per month by March 2021 and keeps +Finally, the rate of inserts has grown to more than 170 GB of data per month by March 2021 and keeps on growing, so the only viable solution to pruning old data was through partitioning. Our approach was to partition the table per month as it aligned with the 90 days retention policy. diff --git a/doc/architecture/blueprints/gitlab_observability_backend/metrics/index.md b/doc/architecture/blueprints/gitlab_observability_backend/metrics/index.md new file mode 100644 index 00000000000..a6efe68310e --- /dev/null +++ b/doc/architecture/blueprints/gitlab_observability_backend/metrics/index.md @@ -0,0 +1,688 @@ +--- +status: proposed +creation-date: "2022-11-09" +authors: [ "@ankitbhatnagar" ] +coach: "@mappelman" +approvers: [ "@sebastienpahl", "@nicholasklick" ] +owning-stage: "~monitor::observability" +participating-stages: [] +--- + +# GitLab Observability Backend - Metrics + +## Summary + +Developing a multi-user system to store & query observability data typically formatted in widely accepted, industry-standard formats using Clickhouse as underlying storage, with support for long-term data retention and aggregation. + +## Motivation + +From the six pillars of Observability, commonly abbreviated as `TEMPLE` - Traces, Events, Metrics, Profiles, Logs & Errors, Metrics constitute one of the most important pillars of observability data for modern day systems, helping their users gather insights about their operational posture. + +Metrics which are commonly structured as timeseries data have the following characteristics: + +- indexed by their corresponding timestamps; +- continuously expanding in size; +- usually aggregated, down-sampled, and queried in ranges; and +- have very write-intensive requirements. + +Within GitLab Observability Backend, we aim to add the support for our customers to ingest and query observability data around their systems & applications, helping them improve the operational health of their systems. + +### Goals + +With the development of the proposed system, we have the following goals: + +- Scalable, low latency & cost-effective monitoring system backed by Clickhouse whose performance has been proven via repeatable benchmarks. + +- Support for long-term storage for Prometheus/OpenTelemetry formatted metrics, ingested via Prometheus remote_write API and queried via Prometheus remote_read API, PromQL or SQL with support for metadata and exemplars. + +The aformentioned goals can further be broken down into the following four sub-goals: + +#### Ingesting data + +- For the system to be capable of ingesting large volumes of writes and reads, we aim to ensure that it must be horizontally scalable & provide durability guarantees to ensure no writes are dropped once ingested. + +#### Persisting data + +- We aim to support ingesting telemetry/data sent using Prometheus `remote_write` protocol. Any persistence we design for our dataset must be multi-tenant by default, ensuring we can store observability data for multiple tenants/groups/projects within the same storage backend. + +- We aim to develop a test suite for data correctness, seeking inspiration from how Prometheus compliance test suite checks the correctness of a given Metrics implementation and running it as a part of our CI setup. + +NOTE: +Although remote_write_sender does not test the correctness of a remote write receiver itself as is our case, it does bring some inspiration to implement/develop one within the scope of this project. + +- We aim to also ensure compatibility for special Prometheus data types, e.g. Prometheus histogram(s), summary(s). + +#### Reading data + +- We aim to support querying data using PromQL which means translating PromQL queries into Clickhouse SQL. To do this, [PromQL](https://github.com/prometheus/prometheus/tree/main/promql/parser) or [MetricsQL](https://github.com/VictoriaMetrics/metricsql) parsers are good alternatives. + +- We aim to provide additional value by exposing all ingested data via the native Clickhouse SQL interface subject to the following reliability characteristics: + - query validation, sanitation + - rate limiting + - resource limiting - memory, cpu, network bandwidth + +- We aim to pass Prometheus test suits for correctness via the [Prometheus Compliance test suite](https://github.com/prometheus/compliance/tree/main/promql) with a target goal of 100% success rate. + +#### Deleting data + +- We aim to support being able to delete any ingested data should such a need arise. This is also in addition to us naturally deleting data when a configured TTL expires and/or respective retention policies are enforced. We must, within our schemas, build a way to delete data by labels OR their content, also add to our offering the necessary tooling to do so. + +### Non-Goals + +With the goals established above, we also want to establish what specific things are non-goals with the current proposal. They are: + +- We do not aim to support ingestion using OpenTelemetry/OpenMetrics formats with our first iteration, though our users can still use the Opentelemetry exporters(s) internally consuming the standard Prometheus `remote_write` protocol. More information [here](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/prometheusremotewriteexporter). + +- We do not aim to support ingesting Prometheus exemplars in our first iteration, though we do aim to account for them in our design from the beginning. + +NOTE: +Worth noting that we intend to model exemplars the same way we’re modeling metric-labels, so building on top of the same data structure should help implementt support for metadata/exemplars rather easily. + +## Proposal + +We intend to use GitLab Observability Backend as a framework for the Metrics implementation so that its lifecycle is also managed via already existing Kubernetes controllers e.g. scheduler, tenant-operator. + + + +From a development perspective, what’s been marked as our “Application Server” above needs to be developed as a part of this proposal while the remaining peripheral components either already exist or can be provisioned via existing code in `scheduler`/`tenant-operator`. + +**On the write path**, we expect to receive incoming data via `HTTP`/`gRPC` `Ingress` similar to what we do for our existing services, e.g. errortracking, tracing. + +NOTE: +Additionally, since we intend to ingest data via Prometheus `remote_write` API, the received data will be Protobuf-encoded, Snappy-compressed. All received data therefore needs to be decompressed & decoded to turn it into a set of `prompb.TimeSeries` objects, which the rest of our components interact with. + +We also need to make sure to avoid writing a lot of small writes into Clickhouse, therefore it’d be prudent to batch data before writing it into Clickhouse. + +We must also make sure ingestion remains decoupled with `Storage` so as to reduce undue dependence on a given storage implementation. While we do intend to use Clickhouse as our backing storage for any foreseeable future, this ensures we do not tie ourselves in into Clickhouse too much should future business requirements warrant the usage of a different backend/technology. A good way to implement this in Golang would be our implementations adhering to a standard interface, the following for example: + +```golang +type Storage interface { + Read( + ctx context.Context, + request *prompb.ReadRequest + ) (*prompb.ReadResponse, error) + Write( + ctx context.Context, + request *prompb.WriteRequest + ) error +} +``` + +NOTE: +We understand this couples the implementation with Prometheus data format/request types, but adding methods to the interface to support more data formats should be trivial looking forward with minimal changes to code. + +**On the read path**, we aim to allow our users to use the Prometheus `remote_read` API and be able to query ingested data via PromQL & SQL. Support for `remote_read` API should be trivial to implement, while supporting PromQL would need translating it into SQL. We can however employ the usage of already existing [PromQL](https://github.com/prometheus/prometheus/tree/main/promql/parser) parsing libraries. + +We aim to focus on implementing query validation & sanitation, rate-limiting and regulating resource-consumption to ensure underlying systems, esp. storage, remain in good operational health at all times. + +### Supported deployments + +In this first iteration of the metrics backend, we intend to support a generic deployment model that makes sure we can capture as much usage as possible and begin dogfooding the product as soon as possible. This is well illustrated in the [aforementioned architecture diagram](#proposal). + +In its most vanilla form, metrics support in GitLab Observability Backend can be used via the Prometheus remote read & write APIs. If a user already uses Prometheus as their monitoring abstraction, it can be configured to use this backend directly. + +- remote_write: [configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write) +- remote_read: [configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_read) + +For users of the system that do not use a Prometheus instance for scraping their telemetry data, they can export their metrics via a multitude of collectors/agents such as the OpenTelemetry collector or the Prometheus Agent for example, all of which can be configured to use our remote_write endpoint. For reads however, we intend to run a Prometheus within GOB (alongside the application server) itself, then hook it up automatically with the GitLab Observability UI (GOUI) preconfigured to consume our remote_read endpoint. + +Notably, the ability to use a GOB-run Prometheus instance is applicable while we can only support remote_read API for running queries. Looking forward towards our next iteration, we should be able to get rid of this additional component altogether when we have full support for executing PromQL and/or SQL queries directly from GOUI. + +**Per-group deployments**: From a scalability perspective, we deploy an instance of Ingress, a Prometheus instance & the application server per group to make sure we can scale them subject to traffic volumes of the respective tenant. It also helps isolate resource consumption across tenants in an otherwise multi-tenant system. + +### Metric collection and storage + +It is important to separate metric collection on the client side with the storage we provision at our end. + +### State of the art for storage + +Existing long-term Prometheus compatible metrics vendors provide APIs that are compatible with Prometheus remote_write. + +### State of the art for Prometheus clients + +Metric collection clients such as Prometheus itself, Grafana Cloud Agent, Datadog Agent, etc. will scrape metrics endpoints typically from within a firewalled environment, store locally scraped metrics in a [Write Ahead Log (WAL)](https://en.wikipedia.org/wiki/Write-ahead_logging) and then batch send them to an external environment (i.e. the vendor or an internally managed system like Thanos) via the Prometheus `remote_write` protocol. + +- A client-side collector is an important part of the overall architecture, though it's owned by the customer/user since it needs to run in their environment. This gives the end user full control over their data because they control how it is collected and to where it is delivered. + +- It's **not** feasible to provide an external vendor with credentials to access and scrape endpoints within a user's firewalled environment. + +- It's also critically important that our `remote_write` APIs respond correctly with the appropriate rate-limiting status codes so that Prometheus Clients can respect them. + +[Here](https://grafana.com/blog/2021/05/26/the-future-of-prometheus-remote-write/) is a good background/history on Prometheus `remote_write` and its importance in Prometheus based observability. + +## Design and implementation details + +Following are details of how we aim to design & implement the proposed solution. To that end, a reference implementation was also developed to understand the scope of the problem and provide early data to ensure our proposal was drafted around informed decisions and/or results of our experimentation. + +## Reference implementation(s) + +- [Application server](https://gitlab.com/gitlab-org/opstrace/opstrace/-/merge_requests/1823) +- [Metrics generator](https://gitlab.com/ankitbhatnagar/metrics-gen/-/blob/main/main.go) + +## Target environments + +Keeping inline with our current operational structure, we intend to deploy the metrics offering as a part of GitLab Observability Backend, deployed on the following two target environments: + +- kind cluster (for local development) +- GKE cluster (for staging/production environments) + +## Schema Design + +### **Proposed solution**: Fully normalized tables for decreased redundancy & increased read performance + +### primary, denormalized data table + +```sql +CREATE TABLE IF NOT EXISTS samples ON CLUSTER '{cluster}' ( + series_id UUID, + timestamp DateTime64(3, ‘UTC’) CODEC(Delta(4), ZSTD), + value Float64 CODEC(Gorilla, ZSTD) +) ENGINE = ReplicatedMergeTree() +PARTITION BY toYYYYMMDD(timestamp) +ORDER BY (series_id, timestamp) +``` + +### metadata table to support timeseries metadata/exemplars + +```sql +CREATE TABLE IF NOT EXISTS samples_metadata ON CLUSTER '{cluster}' ( + series_id UUID, + timestamp DateTime64(3, ‘UTC’) CODEC(Delta(4), ZSTD), + metadata Map(String, String) CODEC(ZSTD), +) ENGINE = ReplicatedMergeTree() +PARTITION BY toYYYYMMDD(timestamp) +ORDER BY (series_id, timestamp) +``` + +### lookup table(s) + +```sql +CREATE TABLE IF NOT EXISTS labels_to_series ON CLUSTER '{cluster}' ( + labels Map(String, String) CODEC(ZSTD) + series_id UUID +) engine=ReplicatedMergeTree +PRIMARY KEY (labels, series_id) +``` + +```sql +CREATE TABLE IF NOT EXISTS group_to_series ON CLUSTER ‘{cluster}’ ( + group_id Uint64, + series_id UUID, +) ORDER BY (group_id, series_id) +``` + +### Refinements + +- sharding considerations for a given tenant when ingesting/persisting data if we intend to co-locate data specific to multiple tenants within the same database tables. To simplify things, segregating tenant-specific data to their own dedicated set of tables would make a lot of sense. + +- structural considerations for “timestamps” when ingesting data across tenants. + +- creation_time vs ingestion_time + +- No support for transactions in the native client yet, to be able to effectively manage writes across multiple tables. + +NOTE: +Slightly non-trivial but we can potentially investigate the possibility of using ClickHouse/ch-go directly, it supposedly promises a better performance profile too. + +### Pros - multiple tables + +- Normalised data structuring allows for efficient storage of data, removing any redundancy across multiple samples for a given timeseries. Evidently, for the “samples” schema, we expect to store 32 bytes of data per metric point. + +- Better search complexity when filtering timeseries by labels/metadata, via the use of better indexed columns. + +- All data is identifiable via a unique identifier, which can be used to maintain data consistency across tables. + +### Cons - multiple tables + +- Writes are trivially expensive considering writes across multiple tables. + +- Writes across tables also need to be implemented as a transaction to guarantee consistency when ingesting data. + +### Operational characteristics - multiple tables + +### Storage - multiple tables + +A major portion of our writes are made into the `samples` schema which contains a tuple containing three data points per metric point written: +|column|data type|byte size| +|---|---|---| +|series_id|UUID|16 bytes| +|timestamp|DateTime64|8 bytes| +|value|Float64|8 bytes| + +Therefore, we estimate to use 32 bytes per sample ingested. + +### Compression - multiple tables + +Inspecting the amount of compression we’re able to get with the given design on our major schemas, we see it as a good starting point. Following measurements for both primary tables: + +**Schema**: `labels_to_series` containing close to 12k unique `series_id`, each mapping to a set of 10-12 label string pairs + +```sql +SELECT + table, + column, + formatReadableSize(sum(data_compressed_bytes) AS x) AS compressedsize, + formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed +FROM system.parts_columns +WHERE table LIKE 'labels_to_series_1' +GROUP BY + database, + table, + column +ORDER BY x ASC + +Query id: 723b4145-14f7-4e74-9ada-01c17c2f1fd5 + +┌─table──────────────┬─column────┬─compressedsize─┬─uncompressed─┐ +│ labels_to_series_1 │ labels │ 586.66 KiB │ 2.42 MiB │ +│ labels_to_series_1 │ series_id │ 586.66 KiB │ 2.42 MiB │ +└────────────────────┴───────────┴────────────────┴──────────────┘ +``` + +**Schema**: `samples` containing about 20k metric samples each containing a tuple comprising `series_id` (16 bytes), `timestamp` (8 bytes) and `value` (8 bytes). + +```sql +SELECT + table, + column, + formatReadableSize(sum(data_compressed_bytes) AS x) AS compressedsize, + formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed +FROM system.parts_columns +WHERE table LIKE 'samples_1' +GROUP BY + database, + table, + column +ORDER BY x ASC + +Query id: 04219cea-06ea-4c5f-9287-23cb23c023d2 + +┌─table─────┬─column────┬─compressedsize─┬─uncompressed─┐ +│ samples_1 │ value │ 373.21 KiB │ 709.78 KiB │ +│ samples_1 │ timestamp │ 373.21 KiB │ 709.78 KiB │ +│ samples_1 │ series_id │ 373.21 KiB │ 709.78 KiB │ +└───────────┴───────────┴────────────────┴──────────────┘ +``` + +### Performance - multiple tables + +From profiling our reference implementation, it can also be noted that most of our time right now is spent in the application writing data to Clickhouse and/or its related operations. A “top” pprof profile sampled from the implementation looked like: + +```shell +(pprof) top +Showing nodes accounting for 42253.20kB, 100% of 42253.20kB total +Showing top 10 nodes out of 58 + flat flat% sum% cum cum% +13630.30kB 32.26% 32.26% 13630.30kB 32.26% github.com/ClickHouse/clickhouse-go/v2/lib/compress.NewWriter (inline) +11880.92kB 28.12% 60.38% 11880.92kB 28.12% github.com/ClickHouse/clickhouse-go/v2/lib/compress.NewReader (inline) + 5921.37kB 14.01% 74.39% 5921.37kB 14.01% bufio.NewReaderSize (inline) + 5921.37kB 14.01% 88.41% 5921.37kB 14.01% bufio.NewWriterSize (inline) + 1537.69kB 3.64% 92.04% 1537.69kB 3.64% runtime.allocm + 1040.73kB 2.46% 94.51% 1040.73kB 2.46% github.com/aws/aws-sdk-go/aws/endpoints.init + 1024.41kB 2.42% 96.93% 1024.41kB 2.42% runtime.malg + 768.26kB 1.82% 98.75% 768.26kB 1.82% go.uber.org/zap/zapcore.newCounters + 528.17kB 1.25% 100% 528.17kB 1.25% regexp.(*bitState).reset + 0 0% 100% 5927.73kB 14.03% github.com/ClickHouse/clickhouse-go/v2.(*clickhouse).Ping +``` + +As is evident above from our preliminary analysis, writing data into Clickhouse can be a potential bottleneck. Therefore, on the write path, it'd be prudent to batch our writes into Clickhouse so as to reduce the amount of work the application server ends up doing making the ingestion path more efficient. + +On the read path, it’s also possible to parallelize reads for the samples table either by series_id(s) OR by blocks of time between the queried start and end timestamps. + +### Caveats + +- When dropping labels from already existing metrics, we treat their new counterparts as completely new series and hence attribute them to a new series_id. This avoids having to merge series data and/or values. The old series, if not actively written into, should eventually fall off their retention and get deleted. + +- We have not yet accounted for any data aggregation. Our assumption is that the backing store (in Clickhouse) should allow us to keep a “sufficient” amount of data in its raw form and that we should be able to query against it within our query latency SLOs. + +### **Rejected alternative**: Single, centralized table + +### single, centralized data table + +```sql +CREATE TABLE IF NOT EXISTS metrics ON CLUSTER ‘{cluster}’ ( + group_id UInt64, + name LowCardinality(String) CODEC(ZSTD), + labels Map(String, String) CODEC(ZSTD), + metadata Map(String, String) CODEC(ZSTD), + value Float64 CODEC (Gorilla, ZSTD), + timestamp DateTime64(3, ‘UTC’) CODEC(Delta(4),ZSTD) +) ENGINE = ReplicatedMergeTree() +PARTITION BY toYYYYMMDD(timestamp) +ORDER BY (group_id, name, timestamp); +``` + +### Pros - single table + +- Single source of truth, so all metrics data lives in one big table. + +- Querying data is easier to express in terms of writing SQL queries without having to query data across multiple tables. + +### Cons - single table + +- Huge redundancy built into the data structure since attributes such as name, labels, metadata are stored repeatedly for each sample collected. + +- Non-trivial complexity to search timeseries with values for labels/metadata given how they’re stored when backed by Maps/Arrays. + +- High query latencies by virtue of having to scan large amounts of data per query made. + +### Operational Characteristics - single table + +### Storage - single table + +|column|data type|byte size| +|---|---|---| +|group_id|UUID|16 bytes| +|name|String|-| +|labels|Map(String, String)|-| +|metadata|Map(String, String)|-| +|value|Float64|8 bytes| +|timestamp|DateTime64|8 bytes| + +NOTE: +Strings are of an arbitrary length, the length is not limited. Their value can contain an arbitrary set of bytes, including null bytes. We will need to regulate what we write into these columns application side. + +### Compression - single table + +**Schema**: `metrics` containing about 20k metric samples each consisting of a `group_id`, `metric name`, `labels`, `metadata`, `timestamp` & corresponding `value`. + +```sql +SELECT count(*) +FROM metrics_1 + +Query id: e580f20b-b422-4d93-bb1f-eb1435761604 + +┌─count()─┐ +│ 12144 │ + + +SELECT + table, + column, + formatReadableSize(sum(data_compressed_bytes) AS x) AS compressedsize, + formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed +FROM system.parts_columns +WHERE table LIKE 'metrics_1' +GROUP BY + database, + table, + column +ORDER BY x ASC + +Query id: b2677493-3fbc-46c1-a9a7-4524a7a86cb4 + +┌─table─────┬─column────┬─compressedsize─┬─uncompressed─┐ +│ metrics_1 │ labels │ 283.02 MiB │ 1.66 GiB │ +│ metrics_1 │ metadata │ 283.02 MiB │ 1.66 GiB │ +│ metrics_1 │ group_id │ 283.02 MiB │ 1.66 GiB │ +│ metrics_1 │ value │ 283.02 MiB │ 1.66 GiB │ +│ metrics_1 │ name │ 283.02 MiB │ 1.66 GiB │ +│ metrics_1 │ timestamp │ 283.02 MiB │ 1.66 GiB │ +└───────────┴───────────┴────────────────┴──────────────┘ +``` + +Though we see a good compression factor for the aforementioned schema, the amount of storage needed to store the corresponding dataset is approximately 300MiB. We also expect to see this footprint increase linearly given the redundancy baked into the schema design itself, also one of the reasons we intend **not** to proceed with this design further. + +### Performance - single table + +```shell +(pprof) top +Showing nodes accounting for 12844.95kB, 100% of 12844.95kB total +Showing top 10 nodes out of 40 + flat flat% sum% cum cum% + 2562.81kB 19.95% 19.95% 2562.81kB 19.95% runtime.allocm + 2561.90kB 19.94% 39.90% 2561.90kB 19.94% github.com/aws/aws-sdk-go/aws/endpoints.init + 2374.91kB 18.49% 58.39% 2374.91kB 18.49% github.com/ClickHouse/clickhouse-go/v2/lib/compress.NewReader (inline) + 1696.32kB 13.21% 71.59% 1696.32kB 13.21% bufio.NewWriterSize (inline) + 1184.27kB 9.22% 80.81% 1184.27kB 9.22% bufio.NewReaderSize (inline) + 1184.27kB 9.22% 90.03% 1184.27kB 9.22% github.com/ClickHouse/clickhouse-go/v2/lib/compress.NewWriter (inline) + 768.26kB 5.98% 96.01% 768.26kB 5.98% go.uber.org/zap/zapcore.newCounters + 512.20kB 3.99% 100% 512.20kB 3.99% runtime.malg + 0 0% 100% 6439.78kB 50.13% github.com/ClickHouse/clickhouse-go/v2.(*clickhouse).Ping + 0 0% 100% 6439.78kB 50.13% github.com/ClickHouse/clickhouse-go/v2.(*clickhouse).acquire +``` + +Writes against this schema perform much better in terms of compute, given it's concentrated on one table and does not need looking up `series_id` from a side table. + +### General storage considerations - Clickhouse + +The following sections intend to deep-dive into specific characteristics of our schema design and/or their interaction with Clickhouse - the database system. + +- table engines + + - [MergeTree](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/) + - [S3 Table Engine](https://clickhouse.com/docs/en/engines/table-engines/integrations/s3/) + +- efficient partitioning and/or sharding + + - Configuring our schemas with the right partitioning keys so as to have the least amount of blocks scanned when reading back the data. + - Sharding here would refer to how we design our data placement strategy to make sure the cluster remains optimally balanced at all times. + +- data compression + +As is visible from the aforementioned preliminary results, we see good compression results with dictionary and delta encoding for strings and floats respectively. When storing labels with a `Map` of `LowCardinality(String)`s, we were able to pack data efficiently. + +- materialized views + +Can be updated dynamically as the need be, help make read paths performant + +- async inserts + +- batch inserts + +- retention/TTLs + +We should only store data for a predetermined period of time, post which we either delete data, aggregate it or ship it to an archival store to reduce operational costs of having to store data for longer periods of time. + +- data aggregation/rollups + +- index granularity + +- skip indexes + +- `max_server_memory_usage_to_ram_ratio` + +### Data access via SQL + +While our corpus of data is PromQL-queryable, it would be prudent to make sure we make the SQL interface “generally available” as well. This capability opens up multiple possibilities to query resident data and allows our users to slice and dice their datasets whichever way they prefer to and/or need to. + +#### Challenges + +- Resource/cost profiling. +- Query validation and sanitation. + +### Illustrative example(s) of data access + +### Writes + +On the write path, we first ensure registering a given set labels to a unique `series_id` and/or re-using one should we have seen the timeseries already in the past. For example: + +```plaintext +redis{region="us-east-1",'os':'Ubuntu15.10',...} <TIMESTAMP> <VALUE> +``` + +**Schema**: labels_to_series + +```sql +SELECT * +FROM labels_to_series_1 +WHERE series_id = '6d926ae8-c3c3-420e-a9e2-d91aff3ac125' +FORMAT Vertical + +Query id: dcbc4bd8-0bdb-4c35-823a-3874096aab6e + +Row 1: +────── +labels: {'arch':'x64','service':'1','__name__':'redis','region':'us-east-1','os':'Ubuntu15.10','team':'LON','service_environment':'production','rack':'36','service_version':'0','measurement':'pubsub_patterns','hostname':'host_32','datacenter':'us-east-1a'} +series_id: 6d926ae8-c3c3-420e-a9e2-d91aff3ac125 + +1 row in set. Elapsed: 0.612 sec. +``` + +Post which, we register each metric point in the `samples` table attributing it to the corresponding `series_id`. + +**Schema**: samples + +```sql +SELECT * +FROM samples_1 +WHERE series_id = '6d926ae8-c3c3-420e-a9e2-d91aff3ac125' +LIMIT 1 +FORMAT Vertical + +Query id: f3b410af-d831-4859-8828-31c89c0385b5 + +Row 1: +────── +series_id: 6d926ae8-c3c3-420e-a9e2-d91aff3ac125 +timestamp: 2022-11-10 12:59:14.939 +value: 0 +``` + +### Reads + +On the read path, we first query all timeseries identifiers by searching for the labels under consideration. Once we have all the `series_id`(s), we then look up all corresponding samples between the query start timestamp and end timestamp. + +For e.g. + +```plaintext +kernel{service_environment=~"prod.*", measurement="boot_time"} +``` + +which gets translated into first looking for all related timeseries: + +```sql +SELECT * +FROM labels_to_series +WHERE +((labels['__name__']) = 'kernel') AND +match(labels['service_environment'], 'prod.*') AND +((labels['measurement']) = 'boot_time'); +``` + +yielding a bunch of `series_id`(s) corresponding to the labels just looked up. + +**Sidenote**, this mostly-static dataset can also be cached and built up in-memory gradually to reduce paying the latency cost the second time, which should reduce the number of lookups considerably. + +To account for newer writes when maintaining this cache: + +- Have an out-of-band process/goroutine maintain this cache, so even if a few queries miss the most recent data, subsequent ones eventually catch up. + +- Have TTLs on the keys, jittered per key so as to rebuild them frequently enough to account for new writes. + +Once we know which timeseries we’re querying for, from there, we can easily look up all samples via the following query: + +```sql +SELECT * +FROM samples +WHERE series_id IN ( + 'a12544be-0a3a-4693-86b0-c61a4553aea3', + 'abd42fc4-74c7-4d80-9b6c-12f673db375d', + … +) +AND timestamp >= '1667546789' +AND timestamp <= '1667633189' +ORDER BY timestamp; +``` + +yielding all timeseries samples we were interested in. + +We then render these into an array of `prometheus.QueryResult` object(s) and return back to the caller as a `prometheus.ReadResponse` object. + +NOTE: +The queries have been broken down into multiple queries only during our early experimentation/iteration, it’d be prudent to use subqueries within the same roundtrip to the database going forward into production/benchmarking. + +## Production Readiness + +### Batching + +Considering we’ll need to batch data before ingesting large volumes of small writes into Clickhouse, the design must account for app-local persistence to allow it to locally batch incoming data before landing it into Clickhouse in batches of a predetermined size in order to increase performance and allow the table engine to continue to persist data successfully. + +We have considered the following alternatives to implement app-local batching: + +- In-memory - non durable +- BadgerDB - durable, embedded, performant +- Redis - trivial, external dependency +- Kafka - non-trivial, external dependency but it can augment multiple other use-cases and help other problem domains at GitLab. + +**Note**: Similar challenges have also surfaced with the CH interactions `errortracking` - the subsystem has in its current implementation. There have been multiple attempts to solve this problem domain in the past - [this MR](https://gitlab.com/gitlab-org/opstrace/opstrace/-/merge_requests/1660) implemented an in-memory alternative while [this one](https://gitlab.com/gitlab-org/opstrace/opstrace/-/merge_requests/1767) attempted an on-disk alternative. + +Any work done in this area of concern would also benefit other subsystems such as errortracking, logging, etc. + +### Scalability + +We intend to start testing the proposed implementation with 10K metric-points per second to test/establish our initial hypothesis, though ideally, we must design the underlying backend for 1M points ingested per second. + +### Benchmarking + +We propose the following three dimensions be tested while benchmarking the proposed implementation: + +- Data ingest performance +- On-disk storage requirements (accounting for replication if applicable) +- Mean query response times + +For understanding performance, we’ll need to first compile a list of such queries given the data we ingest for our tests. Clickhouse query logging is super helpful while doing this. + +NOTE: +Ideally, we aim to benchmark the system to be able to ingest >1M metric points/sec while consistently serving most queries under <1 sec. + +### Past work & references + +- [Benchmark ClickHouse for metrics](https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/1666) +- [Incubation:APM ClickHouse evaluation](https://gitlab.com/gitlab-org/incubation-engineering/apm/apm/-/issues/4) +- [Incubation:APM ClickHouse metrics schema](https://gitlab.com/gitlab-org/incubation-engineering/apm/apm/-/issues/10) +- [Our research around TimescaleDB](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/14137) +- [Current Workload on our Thanos-based setup](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/15420#current-workload) +- [Scaling-200m-series](https://opstrace.com/blog/scaling-200m-series) + +### Cost-estimation + +- We aim to make sure the system's not too expensive, especially given our biggest footprint is on Clickhouse and the underlying storage. + +- We must consider the usage of multiple storage medium(s), especially: + - Tiered storage + - Object storage + +### Tooling + +- We aim to building visibility into high cardinality metrics to be able to assist with keeping our databases healthy by pruning/dropping unused metrics. + +- Similarly, we aim to develop the ability to see unused metrics for the end-user, which can be easily & dynamically built into the system by parsing all read requests and building usage statistics. + +- We aim to add monitoring for per-metric scrape frequencies to make sure the end-user is not ingesting data at a volume they do not need and/or find useful. + +## Looking forward + +### Linkage across telemetry pillars, exemplars + +We must build the metrics system in a way to be able cross-reference ingested data with other telemetry pillars, such as traces, logs and errors, so as to provide a more holistic view of all instrumentation a system sends our way. + +### User-defined SQL queries to aggregate data and/or generate materialized views + +We should allow users of the system to be able to run user-defined, ad-hoc queries similar to how Prometheus recording rules help generate custom metrics from existing ones. + +### Write Ahead Logs (WALs) + +We believe that should we feel the need to start buffering data local to the ingestion application and/or move away from Clickhouse for persisting data, on-disk WALs would be a good direction to proceed into given their prevelant usage among other monitoring system. + +### Custom DSLs or query builders + +Using PromQL directly could be a steep learning curve for users. It would be really nice to have a query builder (as is common in Grafana) to allow building of the typical queries you'd expect to run and to allow exploration of the available metrics. It also serves as a way to learn the DSL, so more complex queries can be created later. + +## Roadmap & Next Steps + +The following section enlists how we intend to implement the aforementioned proposal around building Metrics support into GitLab Observability Service. Each corresponding document and/or issue contains further details of how each next step is planned to be executed. + +- **[DONE]** [Research & draft design proposal and/or requirements](https://docs.google.com/document/d/1kHyIoWEcs14sh3CGfKGiI8QbCsdfIHeYkzVstenpsdE/edit?usp=sharing) +- **[IN-PROGRESS]** [Submit system/schema designs (proposal) & gather feedback](https://docs.google.com/document/d/1kHyIoWEcs14sh3CGfKGiI8QbCsdfIHeYkzVstenpsdE/edit?usp=sharing) +- **[IN-PROGRESS]** [Develop table definitions and/or storage interfaces](https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/1666) +- **[IN-PROGRESS]** [Prototype reference implementation, instrument key metrics](https://gitlab.com/gitlab-org/opstrace/opstrace/-/merge_requests/1823) +- [Benchmark Clickhouse and/or proposed schemas, gather expert advice from Clickhouse Inc.](https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/1666) +- Develop write path(s) - `remote_write` API +- Develop read path(s) - `remote_read` API, `PromQL`-based querier. +- Setup testbed(s) for repeatable benchmarking/testing +- Schema design and/or application server improvements if needed +- Production Readiness v1.0-alpha/beta +- Implement vanguarded/staged rollouts +- Run extended alpha/beta testing +- Release v1.0 diff --git a/doc/architecture/blueprints/gitlab_observability_backend/metrics/supported-deployments.png b/doc/architecture/blueprints/gitlab_observability_backend/metrics/supported-deployments.png Binary files differnew file mode 100644 index 00000000000..54c4ed3b48f --- /dev/null +++ b/doc/architecture/blueprints/gitlab_observability_backend/metrics/supported-deployments.png diff --git a/doc/architecture/blueprints/image_resizing/index.md b/doc/architecture/blueprints/image_resizing/index.md index 948378d8834..99a7c4ae144 100644 --- a/doc/architecture/blueprints/image_resizing/index.md +++ b/doc/architecture/blueprints/image_resizing/index.md @@ -14,7 +14,7 @@ Currently, we are showing all uploaded images 1:1, which is of course not ideal. ## MVC Avatar Resizing -When implementing a dynamic image resizing solution, images should be resized and optimized on the fly so that if we define new targeted sizes later we can add them dynamically. This would mean a huge improvement in performance as some of the measurements suggest that we can save up to 95% of our current load size. Our initial investigations indicate that we have uploaded approximately 1.65 million avatars totaling approximately 80GB in size and averaging approximately 48kb each. Early measurements indicate we can reduce the most common avatar dimensions to between 1-3kb in size, netting us a greater than 90% size reduction. For the MVC we don't consider application level caching and rely purely on HTTP based caches as implemented in CDNs and browsers, but might revisit this decision later on. To easily mitigate performance issues with avatar resizing, especially in the case of self managed, an operations feature flag is implemented to disable dynamic image resizing. +When implementing a dynamic image resizing solution, images should be resized and optimized on the fly so that if we define new targeted sizes later we can add them dynamically. This would mean a huge improvement in performance as some of the measurements suggest that we can save up to 95% of our current load size. Our initial investigations indicate that we have uploaded approximately 1.65 million avatars totaling approximately 80 GB in size and averaging approximately 48 KB each. Early measurements indicate we can reduce the most common avatar dimensions to between 1-3 KB in size, netting us a greater than 90% size reduction. For the MVC we don't consider application level caching and rely purely on HTTP based caches as implemented in CDNs and browsers, but might revisit this decision later on. To mitigate performance issues with avatar resizing, especially in the case of self managed, an operations feature flag is implemented to disable dynamic image resizing. ```mermaid sequenceDiagram @@ -38,10 +38,10 @@ sequenceDiagram Content image resizing is a more complex problem to tackle. There are no set size restrictions and there are additional features or requirements to consider. - Dynamic WebP support - the WebP format typically achieves an average of 30% more compression than JPEG without the loss of image quality. More details are in [this Google Comparative Study](https://developers.google.com/speed/webp/docs/c_study) -- Extract first image of GIF's so we can prevent from loading 10MB pixels +- Extract first GIF image so we can prevent from loading 10 MB pixels - Check Device Pixel Ratio to deliver nice images on High DPI screens - Progressive image loading, similar to what is described in [this article about how to build a progressive image loader](https://www.sitepoint.com/how-to-build-your-own-progressive-image-loader/) -- Resizing recommendations (size, clarity, and so on) +- Resizing recommendations (for example, size and clarity) - Storage The MVC Avatar resizing implementation is integrated into Workhorse. With the extra requirements for content image resizing, this may require further use of GraphicsMagik (GM) or a similar library and breaking it out of Workhorse. diff --git a/doc/architecture/blueprints/object_storage/index.md b/doc/architecture/blueprints/object_storage/index.md index 61dc37d7706..950a5f13c38 100644 --- a/doc/architecture/blueprints/object_storage/index.md +++ b/doc/architecture/blueprints/object_storage/index.md @@ -62,7 +62,7 @@ This has led to increased complexity across the board, from development that would normally be "free". - In many cases, we copy around object storage files needlessly (for example, [issue #285597](https://gitlab.com/gitlab-org/gitlab/-/issues/285597)). - Large files (LFS, packages, and so on) are slow to finalize or don't work + Large files (for example, LFS and packages) are slow to finalize or don't work at all as a result. ## Improvements over the current situation @@ -113,7 +113,7 @@ Because every group of features requires its own bucket, we don't have direct upload enabled everywhere. Contributing a new upload requires coding it in both Ruby on Rails and Go. -Implementing a new feature that does not yet have a dedicated bucket +Implementing a new feature that does not have a dedicated bucket requires the developer to also create a merge request in Omnibus and CNG, as well as coordinate with SREs to configure the new bucket for our own environments. @@ -138,7 +138,7 @@ access to new features without infrastructure chores. Our implementation is built on top of a 3rd-party framework where every object storage client is a 3rd-party library. Unfortunately some of them are unmaintained. -[We have customers who cannot push 5GB Git LFS objects](https://gitlab.com/gitlab-org/gitlab/-/issues/216442), +[We have customers who cannot push 5 GB Git LFS objects](https://gitlab.com/gitlab-org/gitlab/-/issues/216442), but with such a vital feature implemented in 3rd-party libraries we are slowed down in fixing it, and we also rely on external maintainers to merge and release fixes. diff --git a/doc/architecture/blueprints/pods/images/pods-and-fulfillment.png b/doc/architecture/blueprints/pods/images/pods-and-fulfillment.png Binary files differindex aab8556a5d3..fea32d1800e 100644 --- a/doc/architecture/blueprints/pods/images/pods-and-fulfillment.png +++ b/doc/architecture/blueprints/pods/images/pods-and-fulfillment.png diff --git a/doc/architecture/blueprints/pods/index.md b/doc/architecture/blueprints/pods/index.md index 3ba319d169b..0a36de5790f 100644 --- a/doc/architecture/blueprints/pods/index.md +++ b/doc/architecture/blueprints/pods/index.md @@ -324,6 +324,18 @@ This is the list of known affected features with the proposed solutions. - [Pods: GraphQL](pods-feature-graphql.md) - [Pods: Organizations](pods-feature-organizations.md) - [Pods: Router Endpoints Classification](pods-feature-router-endpoints-classification.md) +- [Pods: Schema changes (Postgres and Elasticsearch migrations)](pods-feature-schema-changes.md) +- [Pods: Global Search](pods-feature-global-search.md) +- [Pods: CI Runners](pods-feature-ci-runners.md) +- [Pods: Admin Area](pods-feature-admin-area.md) +- [Pods: Container Registry](pods-feature-container-registry.md) +- [Pods: Contributions: Forks](pods-feature-contributions-forks.md) +- [Pods: Personal Namespaces](pods-feature-personal-namespaces.md) +- [Pods: Dashboard: Projects, Todos, Issues, Merge Requests, Activity, ...](pods-feature-dashboard.md) +- [Pods: Snippets](pods-feature-snippets.md) +- [Pods: Uploads](pods-feature-uploads.md) +- [Pods: GitLab Pages](pods-feature-gitlab-pages.md) +- [Pods: Agent for Kubernetes](pods-feature-agent-for-kubernetes.md) ## Links diff --git a/doc/architecture/blueprints/pods/pods-feature-admin-area.md b/doc/architecture/blueprints/pods/pods-feature-admin-area.md new file mode 100644 index 00000000000..7efaa383510 --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-admin-area.md @@ -0,0 +1,58 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: Admin Area' +--- + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: Admin Area + +In our Pods architecture proposal we plan to share all admin related tables in +GitLab. This allows simpler management of all Pods in one interface and reduces +the risk of settings diverging in different Pods. This introduces challenges +with admin pages that allow you to manage data that will be spread across all +Pods. + +## 1. Definition + +There are consequences for admin pages that contain data that spans "the whole +instance" as the Admin pages may be served by any Pod or possibly just 1 pod. +There are already many parts of the Admin interface that will have data that +spans many pods. For example lists of all Groups, Projects, Topics, Jobs, +Analytics, Applications and more. There are also administrative monitoring +capabilities in the Admin page that will span many pods such as the "Background +Jobs" and "Background Migrations" pages. + +## 2. Data flow + +## 3. Proposal + +We will need to decide how to handle these exceptions with a few possible +options: + +1. Move all these pages out into a dedicated per-pod Admin section. Probably + the URL will need to be routable to a single Pod like `/pods/<pod_id>/admin`, + then we can display this data per Pod. These pages will be distinct from + other Admin pages which control settings that are shared across all Pods. We + will also need to consider how this impacts self-managed customers and + whether, or not, this should be visible for single-pod instances of GitLab. +1. Build some aggregation interfaces for this data so that it can be fetched + from all Pods and presented in a single UI. This may be beneficial to an + administrator that needs to see and filter all data at a glance, especially + when they don't know which Pod the data is on. The downside, however, is + that building this kind of aggregation is very tricky when all the Pods are + designed to be totally independent, and it does also enforce more strict + requirements on compatibility between Pods. + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/pods-feature-agent-for-kubernetes.md b/doc/architecture/blueprints/pods/pods-feature-agent-for-kubernetes.md new file mode 100644 index 00000000000..f390c751b8b --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-agent-for-kubernetes.md @@ -0,0 +1,29 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: Agent for Kubernetes' +--- + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: Agent for Kubernetes + +> TL;DR + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/pods-feature-ci-runners.md b/doc/architecture/blueprints/pods/pods-feature-ci-runners.md new file mode 100644 index 00000000000..b75515a916f --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-ci-runners.md @@ -0,0 +1,169 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: CI Runners' +--- + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: CI Runners + +GitLab in order to execute CI jobs [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner/), +very often managed by customer in their infrastructure. + +All CI jobs created as part of CI pipeline are run in a context of project +it poses a challenge how to manage GitLab Runners. + +## 1. Definition + +There are 3 different types of runners: + +- instance-wide: runners that are registered globally with specific tags (selection criteria) +- group runners: runners that execute jobs from a given top-level group or subprojects of that group +- project runners: runners that execute jobs from projects or many projects: some runners might + have projects assigned from projects in different top-level groups. + +This alongside with existing data structure where `ci_runners` is a table describing +all types of runners poses a challenge how the `ci_runners` should be managed in a Pods environment. + +## 2. Data flow + +GitLab Runners use a set of globally scoped endpoints to: + +- registration of a new runner via registration token `https://gitlab.com/api/v4/runners` + ([subject for removal](../runner_tokens/index.md)) (`registration token`) +- requests jobs via an authenticated `https://gitlab.com/api/v4/jobs/request` endpoint (`runner token`) +- upload job status via `https://gitlab.com/api/v4/jobs/:job_id` (`build token`) +- upload trace via `https://gitlab.com/api/v4/jobs/:job_id/trace` (`build token`) +- download and upload artifacts via `https://gitlab.com/api/v4/jobs/:job_id/artifacts` (`build token`) + +Currently three types of authentication tokens are used: + +- runner registration token ([subject for removal](../runner_tokens/index.md)) +- runner token representing an registered runner in a system with specific configuration (`tags`, `locked`, etc.) +- build token representing an ephemeral token giving a limited access to updating a specific + job, uploading artifacts, downloading dependent artifacts, downloading and uploading + container registry images + +Each of those endpoints do receive an authentication token via header (`JOB-TOKEN` for `/trace`) +or body parameter (`token` all other endpoints). + +Since the CI pipeline would be created in a context of a specific Pod it would be required +that pick of a build would have to be processed by that particular Pod. This requires +that build picking depending on a solution would have to be either: + +- routed to correct Pod for a first time +- be made to be two phase: request build from global pool, claim build on a specific Pod using a Pod specific URL + +## 3. Proposal + +This section describes various proposals. Reader should consider that those +proposals do describe solutions for different problems. Many or some aspects +of those proposals might be the solution to the stated problem. + +### 3.1. Authentication tokens + +Even though the paths for CI Runners are not routable they can be made routable with +those two possible solutions: + +- The `https://gitlab.com/api/v4/jobs/request` uses a long polling mechanism with + a ticketing mechanism (based on `X-GitLab-Last-Update` header). Runner when first + starts sends a request to GitLab to which GitLab responds with either a build to pick + by runner. This value is completely controlled by GitLab. This allows GitLab + to use JWT or any other means to encode `pod` identifier that could be easily + decodable by Router. +- The majority of communication (in terms of volume) is using `build token` making it + the easiest target to change since GitLab is sole owner of the token that Runner later + uses for specific job. There were prior discussions about not storing `build token` + but rather using `JWT` token with defined scopes. Such token could encode the `pod` + to which router could easily route all requests. + +### 3.2. Request body + +- The most of used endpoints pass authentication token in request body. It might be desired + to use HTTP Headers as an easier way to access this information by Router without + a need to proxy requests. + +### 3.3. Instance-wide are Pod local + +We can pick a design where all runners are always registered and local to a given Pod: + +- Each Pod has it's own set of instance-wide runners that are updated at it's own pace +- The project runners can only be linked to projects from the same organization + creating strong isolation. +- In this model the `ci_runners` table is local to the Pod. +- In this model we would require the above endpoints to be scoped to a Pod in some way + or made routable. It might be via prefixing them, adding additional Pod parameter, + or providing much more robust way to decode runner token and match it to Pod. +- If routable token is used, we could move away from cryptographic random stored in + database to rather prefer to use JWT tokens that would encode +- The Admin Area showing registered Runners would have to be scoped to a Pod + +This model might be desired since it provides strong isolation guarantees. +This model does significantly increase maintenance overhead since each Pod is managed +separately. + +This model may require adjustments to runner tags feature so that projects have consistent runner experience across pods. + +### 3.4. Instance-wide are cluster-wide + +Contrary to proposal where all runners are Pod local, we can consider that runners +are global, or just instance-wide runners are global. + +However, this requires significant overhaul of system and to change the following aspects: + +- `ci_runners` table would likely have to be split decomposed into `ci_instance_runners`, ... +- all interfaces would have to be adopted to use correct table +- build queuing would have to be reworked to be two phase where each Pod would know of all pending + and running builds, but the actual claim of a build would happen against a Pod containing data +- likely `ci_pending_builds` and `ci_running_builds` would have to be made `cluster-wide` tables + increasing likelihood of creating hotspots in a system related to CI queueing + +This model makes it complex to implement from engineering side. Does make some data being shared +between Pods. Creates hotspots / scalability issues in a system (ex. during abuse) that +might impact experience of organizations on other Pods. + +### 3.5. GitLab CI Daemon + +Another potential solution to explore is to have a dedicated service responsible for builds queueing +owning it's database and working in a model of either sharded or podded service. There were prior +discussions about [CI/CD Daemon](https://gitlab.com/gitlab-org/gitlab/-/issues/19435). + +If the service would be sharded: + +- depending on a model if runners are cluster-wide or pod-local this service would have to fetch + data from all Pods +- if the sharded service would be used we could adapt a model of either sharing database containing + `ci_pending_builds/ci_running_builds` with the service +- if the sharded service would be used we could consider a push model where each Pod pushes to CI/CD Daemon + builds that should be picked by Runner +- the sharded service would be aware which Pod is responsible for processing the given build and could + route processing requests to designated Pod + +If the service would be podded: + +- all expectations of routable endpoints are still valid + +In general usage of CI Daemon does not help significantly with the stated problem. However, this offers +a few upsides related to more efficient processing and decoupling model: push model and it opens a way +to offer stateful communication with GitLab Runners (ex. gRPC or Websockets). + +## 4. Evaluation + +Considering all solutions it appears that solution giving the most promise is: + +- use "instance-wide are Pod local" +- refine endpoints to have routable identities (either via specific paths, or better tokens) + +Other potential upsides is to get rid of `ci_builds.token` and rather use a `JWT token` +that can much better and easier encode wider set of scopes allowed by CI runner. + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/pods-feature-container-registry.md b/doc/architecture/blueprints/pods/pods-feature-container-registry.md new file mode 100644 index 00000000000..d47913fbc2a --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-container-registry.md @@ -0,0 +1,131 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: Container Registry' +--- + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: Container Registry + +GitLab Container Registry is a feature allowing to store Docker Container Images +in GitLab. You can read about GitLab integration [here](../../../user/packages/container_registry/index.md). + +## 1. Definition + +GitLab Container Registry is a complex service requiring usage of PostgreSQL, Redis +and Object Storage dependencies. Right now there's undergoing work to introduce +[Container Registry Metadata](../container_registry_metadata_database/index.md) +to optimize data storage and image retention policies of Container Registry. + +GitLab Container Registry is serving as a container for stored data, +but on it's own does not authenticate `docker login`. The `docker login` +is executed with user credentials (can be `personal access token`) +or CI build credentials (ephemeral `ci_builds.token`). + +Container Registry uses data deduplication. It means that the same blob +(image layer) that is shared between many projects is stored only once. +Each layer is hashed by `sha256`. + +The `docker login` does request JWT time-limited authentication token that +is signed by GitLab, but validated by Container Registry service. The JWT +token does store all authorized scopes (`container repository images`) +and operation types (`push` or `pull`). A single JWT authentication token +can be have many authorized scopes. This allows container registry and client +to mount existing blobs from another scopes. GitLab responds only with +authorized scopes. Then it is up to GitLab Container Registry to validate +if the given operation can be performed. + +The GitLab.com pages are always scoped to project. Each project can have many +container registry images attached. + +Currently in case of GitLab.com the actual registry service is served +via `https://registry.gitlab.com`. + +The main identifiable problems are: + +- the authentication request (`https://gitlab.com/jwt/auth`) that is processed by GitLab.com +- the `https://registry.gitlab.com` that is run by external service and uses it's own data store +- the data deduplication, the Pods architecture with registry run in a Pod would reduce + efficiency of data storage + +## 2. Data flow + +### 2.1. Authorization request that is send by `docker login` + +```shell +curl \ + --user "username:password" \ + "https://gitlab/jwt/auth?client_id=docker&offline_token=true&service=container_registry&scope=repository:gitlab-org/gitlab-build-images:push,pull" +``` + +Result is encoded and signed JWT token. Second base64 encoded string (split by `.`) contains JSON with authorized scopes. + +```json +{"auth_type":"none","access":[{"type":"repository","name":"gitlab-org/gitlab-build-images","actions":["pull"]}],"jti":"61ca2459-091c-4496-a3cf-01bac51d4dc8","aud":"container_registry","iss":"omnibus-gitlab-issuer","iat":1669309469,"nbf":166} +``` + +### 2.2. Docker client fetching tags + +```shell +curl \ + -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + -H "Authorization: Bearer token" \ + https://registry.gitlab.com/v2/gitlab-org/gitlab-build-images/tags/list + +curl \ + -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + -H "Authorization: Bearer token" \ + https://registry.gitlab.com/v2/gitlab-org/gitlab-build-images/manifests/danger-ruby-2.6.6 +``` + +### 2.3. Docker client fetching blobs and manifests + +```shell +curl \ + -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + -H "Authorization: Bearer token" \ + https://registry.gitlab.com/v2/gitlab-org/gitlab-build-images/blobs/sha256:a3f2e1afa377d20897e08a85cae089393daa0ec019feab3851d592248674b416 +``` + +## 3. Proposal + +### 3.1. Shard Container Registry separately to Pods architecture + +Due to it's architecture it extensive architecture and in general highly scalable +horizontal architecture it should be evaluated if the GitLab Container Registry +should be run not in Pod, but in a Cluster and be scaled independently. + +This might be easier, but would definitely not offer the same amount of data isolation. + +### 3.2. Run Container Registry within a Pod + +It appears that except `/jwt/auth` which would likely have to be processed by Router +(to decode `scope`) the container registry could be run as a local service of a Pod. + +The actual data at least in case of GitLab.com is not forwarded via registry, +but rather served directly from Object Storage / CDN. + +Its design encodes container repository image in a URL that is easily routable. +It appears that we could re-use the same stateless Router service in front of Container Registry +to serve manifests and blobs redirect. + +The only downside is increased complexity of managing standalone registry for each Pod, +but this might be desired approach. + +## 4. Evaluation + +There do not seem any theoretical problems with running GitLab Container Registry in a Pod. +Service seems that can be easily made routable to work well. + +The practical complexities are around managing complex service from infrastructure side. + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/pods-feature-contributions-forks.md b/doc/architecture/blueprints/pods/pods-feature-contributions-forks.md new file mode 100644 index 00000000000..566ae50ec49 --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-contributions-forks.md @@ -0,0 +1,120 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: Contributions: Forks' +--- + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: Contributions: Forks + +[Forking workflow](../../../user/project/repository/forking_workflow.md) allows users +to copy existing project sources into their own namespace of choice (personal or group). + +## 1. Definition + +[Forking workflow](../../../user/project/repository/forking_workflow.md) is common workflow +with various usage patterns: + +- allows users to contribute back to upstream project +- persist repositories into their personal namespace +- copy to make changes and release as modified project + +Forks allow users not having write access to parent project to make changes. The forking workflow +is especially important for the Open Source community which is able to contribute back +to public projects. However, it is equally important in some companies which prefer the strong split +of responsibilites and tighter access control. The access to project is restricted +to designated list of developers. + +Forks enable: + +- tigther control of who can modify the upstream project +- split of the responsibilites: parent project might use CI configuration connecting to production systems +- run CI pipelines in context of fork in much more restrictive environment +- consider all forks to be unveted which reduces risks of leaking secrets, or any other information + tied with the project + +The forking model is problematic in Pods architecture for following reasons: + +- Forks are clones of existing repositories, forks could be created across different organizations, Pods and Gitaly shards. +- User can create merge request and contribute back to upstream project, this upstream project might in a different organization and Pod. +- The merge request CI pipeline is to executed in a context of source project, but presented in a context of target project. + +## 2. Data flow + +## 3. Proposals + +### 3.1. Intra-Cluster forks + +This proposal makes us to implement forks as a intra-ClusterPod forks where communication is done via API +between all trusted Pods of a cluster: + +- Forks when created, they are created always in context of user choice of group. +- Forks are isolated from Organization. +- Organization or group owner could disable forking across organizations or forking in general. +- When a Merge Request is created it is created in context of target project, referencing + external project on another Pod. +- To target project the merge reference is transfered that is used for presenting information + in context of target project. +- CI pipeline is fetched in context of source project as it-is today, the result is fetched into + Merge Request of target project. +- The Pod holding target project internally uses GraphQL to fetch status of source project + and include in context of the information for merge request. + +Upsides: + +- All existing forks continue to work as-is, as they are treated as intra-Cluster forks. + +Downsides: + +- The purpose of Organizations is to provide strong isolation between organizations + allowing to fork across does break security boundaries. +- However, this is no different to ability of users today to clone repository to local computer + and push it to any repository of choice. +- Access control of source project can be lower than those of target project. System today + requires that in order to contribute back the access level needs to be the same for fork and upstream. + +### 3.2. Forks are created in a personal namespace of the current organization + +Instead of creating projects across organizations, the forks are created in a user personal namespace +tied with the organization. Example: + +- Each user that is part of organization receives their personal namespace. For example for `GitLab Inc.` + it could be `gitlab.com/organization/gitlab-inc/@ayufan`. +- The user has to fork into it's own personal namespace of the organization. +- The user has that many personal namespaces as many organizations it belongs to. +- The personal namespace behaves similar to currently offered personal namespace. +- The user can manage and create projects within a personal namespace. +- The organization can prevent or disable usage of personal namespaces disallowing forks. +- All current forks are migrated into personal namespace of user in Organization. +- All forks are part of to the organization. +- The forks are not federated features. +- The personal namespace and forked project do not share configuration with parent project. + +### 3.3. Forks are created as internal projects under current project + +Instead of creating projects across organizations, the forks are attachments to existing projects. +Each user forking a project receives their unique project. Example: + +- For project: `gitlab.com/gitlab-org/gitlab`, forks would be created in `gitlab.com/gitlab-org/gitlab/@kamil-gitlab`. +- Forks are created in a context of current organization, they do not cross organization boundaries + and are managed by the organization. +- Tied to the user (or any other user-provided name of the fork). +- The forks are not federated features. + +Downsides: + +- Does not answer how to handle and migrate all exisiting forks. +- Might share current group / project settings - breaking some security boundaries. + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/pods-feature-dashboard.md b/doc/architecture/blueprints/pods/pods-feature-dashboard.md new file mode 100644 index 00000000000..e63d912b4c9 --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-dashboard.md @@ -0,0 +1,29 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: Dashboard' +--- + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: Dashboard + +> TL;DR + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/pods-feature-data-migration.md b/doc/architecture/blueprints/pods/pods-feature-data-migration.md index fad6bca45fa..fbe97316dcc 100644 --- a/doc/architecture/blueprints/pods/pods-feature-data-migration.md +++ b/doc/architecture/blueprints/pods/pods-feature-data-migration.md @@ -26,6 +26,24 @@ we can document the reasons for not choosing this approach. It is essential for Pods architecture to provide a way to migrate data out of big Pods into smaller ones. This describes various approaches to provide this type of split. +We also need to handle for cases where data is already violating the expected +isolation constraints of Pods (ie. references cannot span multiple +organizations). We know that existing features like linked issues allowed users +to link issues across any projects regardless of their hierarchy. There are many +similar features. All of this data will need to be migrated in some way before +it can be split across different pods. This may mean some data needs to be +deleted, or the feature changed and modelled slightly differently before we can +properly split or migrate the organizations between pods. + +Having schema deviations across different Pods, which is a necessary +consequence of different databases, will also impact our ability to migrate +data between pods. Different schemas impact our ability to reliably replicate +data across pods and especially impact our ability to validate that the data is +correctly replicated. It might force us to only be able to move data between +pods when the schemas are all in sync (slowing down deployments and the +rebalancing process) or possibly only migrate from newer to older schemas which +would be complex. + ## 1. Definition ## 2. Data flow @@ -48,13 +66,35 @@ physical replication, etc. 1. The data of Pod 0 is live replicated to as many Pods it needs to be split. 1. Once consensus is achieved between Pod 0 and N-Pods the organizations to be migrated away are marked as read-only cluster-wide. -1. The `routes` is updated on for all organizations to be split to indicate an authorative +1. The `routes` is updated on for all organizations to be split to indicate an authoritative Pod holding the most recent data, like `gitlab-org` on `pod-100`. 1. The data for `gitlab-org` on Pod 0, and on other non-authoritative N-Pods are dormant and will be removed in the future. 1. All accesses to `gitlab-org` on a given Pod are validated about `pod_id` of `routes` to ensure that given Pod is authoritative to handle the data. +#### More challenges of this proposal + +1. There is no streaming replication capability for Elasticsearch, but you could + snapshot the whole Elasticsearch index and recreate, but this takes hours. + It could be handled by pausing Elasticsearch indexing on the initial pod during + the migration as indexing downtime is not a big issue, but this still needs + to be coordinated with the migration process +1. Syncing Redis, Gitaly, CI Postgres, Main Postgres, registry Postgres, other + new data stores snapshots in an online system would likely lead to gaps + without a long downtime. You need to choose a sync point and at the sync + point you need to stop writes to perform the migration. The more data stores + there are to migrate at the same time the longer the write downtime for the + failover. We would also need to find a reliable place in the application to + actually block updates to all these systems with a high degree of + confidence. In the past we've only been confident by shutting down all rails + services because any rails process could write directly to any of these at + any time due to async workloads or other surprising code paths. +1. How to efficiently delete all the orphaned data. Locating all `ci_builds` + associated with half the organizations would be very expensive if we have to + do joins. We haven't yet determined if we'd want to store an `organization_id` + column on every table, but this is the kind of thing it would be helpful for. + ### 3.2. Migrate organization from an existing Pod This is different to split, as we intend to perform logical and selective replication @@ -75,6 +115,14 @@ which Pod is authoritative for this organization. 1. It likely will require a full database structure analysis (more robust than project import/export) to perform selective PostgreSQL logical replication. +#### More challenges of this proposal + +1. Logical replication is still not performant enough to keep up with our + scale. Even if we could use logical replication we still don't have an + efficient way to filter data related to a single organization without + joining all the way to the `organizations` table which will slow down + logical replication dramatically. + ## 4. Evaluation ## 4.1. Pros diff --git a/doc/architecture/blueprints/pods/pods-feature-git-access.md b/doc/architecture/blueprints/pods/pods-feature-git-access.md index ae996281d46..9bda2d1de9c 100644 --- a/doc/architecture/blueprints/pods/pods-feature-git-access.md +++ b/doc/architecture/blueprints/pods/pods-feature-git-access.md @@ -15,7 +15,7 @@ we can document the reasons for not choosing this approach. # Pods: Git Access This document describes impact of Pods architecture on all Git access (over HTTPS and SSH) -patterns providing explanantion of how potentially those features should be changed +patterns providing explanation of how potentially those features should be changed to work well with Pods. ## 1. Definition @@ -130,7 +130,7 @@ sequenceDiagram ## 3. Proposal -The Pods stateless router proposal requires that any ambigious path (that is not routable) +The Pods stateless router proposal requires that any ambiguous path (that is not routable) will be made to be routable. It means that at least the following paths will have to be updated do introduce a routable entity (project, group, or organization). diff --git a/doc/architecture/blueprints/pods/pods-feature-gitlab-pages.md b/doc/architecture/blueprints/pods/pods-feature-gitlab-pages.md new file mode 100644 index 00000000000..932f996d8ba --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-gitlab-pages.md @@ -0,0 +1,29 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: GitLab Pages' +--- + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: GitLab Pages + +> TL;DR + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/pods-feature-global-search.md b/doc/architecture/blueprints/pods/pods-feature-global-search.md new file mode 100644 index 00000000000..5ea863ac646 --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-global-search.md @@ -0,0 +1,47 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: Global search' +--- + +DISCLAIMER: +This page may contain information related to upcoming products, features and +functionality. It is important to note that the information presented is for +informational purposes only, so please do not rely on the information for +purchasing or planning purposes. Just like with all projects, the items +mentioned on the page are subject to change or delay, and the development, +release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: Global search + +When we introduce multiple Pods we intend to isolate all services related to +those Pods. This will include Elasticsearch which means our current global +search functionality will not work. It may be possible to implement aggregated +search across all pods, but it is unlikely to be performant to do fan-out +searches across all pods especially once you start to do pagination which +requires setting the correct offset and page number for each search. + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +Likely first versions of Pods will simply not support global searches and then +we may later consider if building global searches to support popular use cases +is worthwhile. + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/pods-feature-graphql.md b/doc/architecture/blueprints/pods/pods-feature-graphql.md index 5f8a39c0b3f..87c8391fbb3 100644 --- a/doc/architecture/blueprints/pods/pods-feature-graphql.md +++ b/doc/architecture/blueprints/pods/pods-feature-graphql.md @@ -23,7 +23,7 @@ we can document the reasons for not choosing this approach. # Pods: GraphQL -GitLab exensively uses GraphQL to perform efficient data query operations. +GitLab extensively uses GraphQL to perform efficient data query operations. GraphQL due to it's nature is not directly routable. The way how GitLab uses it calls the `/api/graphql` endpoint, and only query or mutation of body request might define where the data can be accessed. diff --git a/doc/architecture/blueprints/pods/pods-feature-personal-namespaces.md b/doc/architecture/blueprints/pods/pods-feature-personal-namespaces.md new file mode 100644 index 00000000000..f78044bb551 --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-personal-namespaces.md @@ -0,0 +1,29 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: Personal Namespaces' +--- + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: Personal Namespaces + +> TL;DR + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/pods-feature-router-endpoints-classification.md b/doc/architecture/blueprints/pods/pods-feature-router-endpoints-classification.md index c672342fff9..bf0969fcb38 100644 --- a/doc/architecture/blueprints/pods/pods-feature-router-endpoints-classification.md +++ b/doc/architecture/blueprints/pods/pods-feature-router-endpoints-classification.md @@ -29,7 +29,7 @@ hitting load balancer of a GitLab installation to a Pod that can serve it. Each Pod should be able to decode each request and classify for which Pod it belongs to. -GitLab currently implements houndreds of endpoints. This document tries +GitLab currently implements hundreds of endpoints. This document tries to describe various techniques that can be implemented to allow the Rails to provide this information efficiently. diff --git a/doc/architecture/blueprints/pods/pods-feature-schema-changes.md b/doc/architecture/blueprints/pods/pods-feature-schema-changes.md new file mode 100644 index 00000000000..ae7c288028d --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-schema-changes.md @@ -0,0 +1,55 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: Schema changes' +--- + +DISCLAIMER: +This page may contain information related to upcoming products, features and +functionality. It is important to note that the information presented is for +informational purposes only, so please do not rely on the information for +purchasing or planning purposes. Just like with all projects, the items +mentioned on the page are subject to change or delay, and the development, +release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: Schema changes + +When we introduce multiple Pods that own their own databases this will +complicate the process of making schema changes to Postgres and Elasticsearch. +Today we already need to be careful to make changes comply with our zero +downtime deployments. For example, +[when removing a column we need to make changes over 3 separate deployments](../../../development/database/avoiding_downtime_in_migrations.md#dropping-columns). +We have tooling like `post_migrate` that helps with these kinds of changes to +reduce the number of merge requests needed, but these will be complicated when +we are dealing with deploying multiple rails applications that will be at +different versions at any one time. This problem will be particularly tricky to +solve for shared databases like our plan to share the `users` related tables +among all Pods. + +A key benefit of Pods may be that it allows us to run different +customers on different versions of GitLab. We may choose to update our own pod +before all our customers giving us even more flexibility than our current +canary architecture. But doing this means that schema changes need to have even +more versions of backward compatibility support which could slow down +development as we need extra steps to make schema changes. + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/pods-feature-snippets.md b/doc/architecture/blueprints/pods/pods-feature-snippets.md new file mode 100644 index 00000000000..1bb866ca958 --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-snippets.md @@ -0,0 +1,29 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: Snippets' +--- + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: Snippets + +> TL;DR + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/pods-feature-uploads.md b/doc/architecture/blueprints/pods/pods-feature-uploads.md new file mode 100644 index 00000000000..634f3ef9560 --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-uploads.md @@ -0,0 +1,29 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: Uploads' +--- + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: Uploads + +> TL;DR + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/proposal-stateless-router-with-buffering-requests.md b/doc/architecture/blueprints/pods/proposal-stateless-router-with-buffering-requests.md index 21aa72273fe..ab19b652f93 100644 --- a/doc/architecture/blueprints/pods/proposal-stateless-router-with-buffering-requests.md +++ b/doc/architecture/blueprints/pods/proposal-stateless-router-with-buffering-requests.md @@ -26,7 +26,7 @@ monolith. This architecture also supports regions by allowing for low traffic databases to be replicated across regions. Users are not directly exposed to the concept of Pods but instead they see -different data dependent on their currently chosen "organization". +different data dependent on their chosen "organization". [Organizations](index.md#organizations) will be a new model introduced to enforce isolation in the application and allow us to decide which request route to which pod, since an organization can only be on a single pod. @@ -166,7 +166,7 @@ graph TD; 1. A new column `routes.pod_id` is added to `routes` table 1. A new Router service exists to choose which pod to route a request to. 1. A new concept will be introduced in GitLab called an organization and a user can select a "default organization" and this will be a user level setting. The default organization is used to redirect users away from ambiguous routes like `/dashboard` to organization scoped routes like `/organizations/my-organization/-/dashboard`. Legacy users will have a special default organization that allows them to keep using global resources on `Pod US0`. All existing namespaces will initially move to this public organization. -1. If a pod receives a request for a `routes.pod_id` that it does not own it returns a `302` with `X-Gitlab-Pod-Redirect` header so that the router can send the request to the correct pod. The correct pod can also set a header `X-Gitlab-Pod-Cache` which contains information about how this request should be cached to remember the pod. For example if the request was `/gitlab-org/gitlab` then the header would encode `/gitlab-org/* => Pod US0` (ie. any requests starting with `/gitlab-org/` can always be routed to `Pod US0` +1. If a pod receives a request for a `routes.pod_id` that it does not own it returns a `302` with `X-Gitlab-Pod-Redirect` header so that the router can send the request to the correct pod. The correct pod can also set a header `X-Gitlab-Pod-Cache` which contains information about how this request should be cached to remember the pod. For example if the request was `/gitlab-org/gitlab` then the header would encode `/gitlab-org/* => Pod US0` (for example, any requests starting with `/gitlab-org/` can always be routed to `Pod US0` 1. When the pod does not know (from the cache) which pod to send a request to it just picks a random pod within it's region 1. Writes to `gitlab_users` and `gitlab_routes` are sent to a primary PostgreSQL server in our `US` region but reads can come from replicas in the same region. This will add latency for these writes but we expect they are infrequent relative to the rest of GitLab. @@ -176,7 +176,7 @@ All users will get a new column `users.default_organization` which they can control in user settings. We will introduce a concept of the `GitLab.com Public` organization. This will be set as the default organization for all existing users. This organization will allow the user to see data from all namespaces in -`Pod US0` (ie. our original GitLab.com instance). This behavior can be invisible to +`Pod US0` (for example, our original GitLab.com instance). This behavior can be invisible to existing users such that they don't even get told when they are viewing a global page like `/dashboard` that it's even scoped to an organization. @@ -195,7 +195,7 @@ frustrating and painful so to avoid this we will decompose and share all Admin A settings in the `gitlab_admin` schema. This should be safe (similar to other shared schemas) because these receive very little write traffic. -In cases where different pods need different settings (eg. the +In cases where different pods need different settings (for example, the Elasticsearch URL), we will either decide to use a templated format in the relevant `application_settings` row which allows it to be dynamic per pod. Alternatively if that proves difficult we'll introduce a new table @@ -241,7 +241,7 @@ keeping settings in sync for all pods. 1. Data in `gitlab_users` and `gitlab_routes` databases must be replicated in all regions which may be an issue for certain types of compliance. 1. The router cache may need to be very large if we get a wide variety of URLs - (ie. long tail). In such a case we may need to implement a 2nd level of + (for example, long tail). In such a case we may need to implement a 2nd level of caching in user cookies so their frequently accessed pages always go to the right pod the first time. 1. Having shared database access for `gitlab_users` and `gitlab_routes` @@ -363,7 +363,7 @@ sequenceDiagram 1. User is in Europe so DNS resolves to the router in Europe 1. The router does not have `/my-company/*` cached yet so it chooses randomly `Pod EU1` 1. `Pod EU1` redirects them through a login flow -1. Stil they request `/my-company/my-project` without the router cache, so the router chooses a random pod `Pod EU1` +1. Still they request `/my-company/my-project` without the router cache, so the router chooses a random pod `Pod EU1` 1. `Pod EU1` does not have `/my-company`, but it knows that it lives in `Pod EU0` so it redirects the router to `Pod EU0` 1. `Pod EU0` returns the correct response as well as setting the cache headers for the router `/my-company/* => Pod EU0` 1. The router now caches and remembers any request paths matching `/my-company/*` should go to `Pod EU0` @@ -499,7 +499,7 @@ allowed to use legacy global functionality like `/dashboard` to see data across namespaces located on `Pod US0`. The rails backend also knows that the default pod to render any ambiguous routes like `/dashboard` is `Pod US0`. Lastly the user will be allowed to navigate to organizations on another pod like `/my-organization` but when they do the -user will see a message indicating that some data may be missing (eg. the +user will see a message indicating that some data may be missing (for example, the MRs/Issues/Todos) counts. #### Navigates to `/gitlab-org/gitlab` while not logged in @@ -615,9 +615,9 @@ Migrating data between pods will need to factor all data stores: ### Is it still possible to leak the existence of private groups via a timing attack? If you have router in EU, and you know that EU router by default redirects -to EU located Pods, you know their latency (lets assume 10ms). Now, if your +to EU located Pods, you know their latency (lets assume 10 ms). Now, if your request is bounced back and redirected to US which has different latency -(lets assume that roundtrip will be around 60ms) you can deduce that 404 was +(lets assume that roundtrip will be around 60 ms) you can deduce that 404 was returned by US Pod and know that your 404 is in fact 403. We may defer this until we actually implement a pod in a different region. Such timing attacks are already theoretically possible with the way we do permission checks today but the timing difference is probably too small to be able to detect. diff --git a/doc/architecture/blueprints/pods/proposal-stateless-router-with-routes-learning.md b/doc/architecture/blueprints/pods/proposal-stateless-router-with-routes-learning.md index e7520f3d6a8..c99b02a35e9 100644 --- a/doc/architecture/blueprints/pods/proposal-stateless-router-with-routes-learning.md +++ b/doc/architecture/blueprints/pods/proposal-stateless-router-with-routes-learning.md @@ -26,7 +26,7 @@ monolith. This architecture also supports regions by allowing for low traffic databases to be replicated across regions. Users are not directly exposed to the concept of Pods but instead they see -different data dependent on their currently chosen "organization". +different data dependent on their chosen "organization". [Organizations](index.md#organizations) will be a new model introduced to enforce isolation in the application and allow us to decide which request route to which pod, since an organization can only be on a single pod. @@ -639,9 +639,9 @@ Migrating data between pods will need to factor all data stores: ### Is it still possible to leak the existence of private groups via a timing attack? If you have router in EU, and you know that EU router by default redirects -to EU located Pods, you know their latency (lets assume 10ms). Now, if your +to EU located Pods, you know their latency (lets assume 10 ms). Now, if your request is bounced back and redirected to US which has different latency -(lets assume that roundtrip will be around 60ms) you can deduce that 404 was +(lets assume that roundtrip will be around 60 ms) you can deduce that 404 was returned by US Pod and know that your 404 is in fact 403. We may defer this until we actually implement a pod in a different region. Such timing attacks are already theoretically possible with the way we do permission checks today but the timing difference is probably too small to be able to detect. diff --git a/doc/architecture/blueprints/rate_limiting/index.md b/doc/architecture/blueprints/rate_limiting/index.md index ffe0712d69b..22709a90cee 100644 --- a/doc/architecture/blueprints/rate_limiting/index.md +++ b/doc/architecture/blueprints/rate_limiting/index.md @@ -50,7 +50,7 @@ vision of our next rate limiting and policies enforcement architecture. - Finding what limits are defined requires performing a codebase audit. - We don't have a good way to expose limits to satellite services like Registry. - We enforce a number of different policies via opaque external systems - (Pipeline Validation Service, Bouncer, Watchtower, Cloudflare, Haproxy). + (Pipeline Validation Service, Bouncer, Watchtower, Cloudflare, HAProxy). - There is not standardized way to define policies in a way consistent with defining limits. - It is difficult to understand when a user is approaching a limit threshold. - There is no way to automatically notify a user when they are approaching thresholds. @@ -103,16 +103,16 @@ quota and by a policy. risks to performance, stability, and security. - _Example:_ API calls per second for a given IP address - _Example:_ `git clone` events per minute for a given user - - _Example:_ maximum artifact upload size of 1GB + - _Example:_ maximum artifact upload size of 1 GB - **Quota:** A global constraint in application usage that is aggregated across an entire namespace over the duration of their billing cycle. - _Example:_ 400 CI/CD minutes per namespace per month - - _Example:_ 10GB transfer per namespace per month + - _Example:_ 10 GB transfer per namespace per month - **Policy:** A representation of business logic that is decoupled from application code. Decoupled policy definitions allow logic to be shared across multiple services and/or "hot-loaded" at runtime without releasing a new version of the application. - _Example:_ decode and verify a JWT, determine whether the user has access to the - given resource based on the JWT's scopes and claims + given resource based on the JWT scopes and claims - _Example:_ deny access based on group-level constraints (such as IP allowlist, SSO, and 2FA) across all services @@ -286,7 +286,7 @@ The GitLab Policy Service might be used in two different ways: 1. The policy service feature will be used as a backend to store policies defined by users. These are two slightly different use-cases: first one is about using -internally-defined policies to ensure the stability / availably of a GitLab +internally-defined policies to ensure the stability / availability of a GitLab instance (GitLab.com or self-managed instance). The second use-case is about making GitLab Policy Service a feature that users will be able to build on top of. @@ -303,7 +303,7 @@ the sections of this document above. It is possible that GitLab Policy Service and Decoupled Limits Service can actually be the same thing. It, however, depends on the implementation details that we can't predict yet, and the decision about merging these services -together will need to be informed by subsequent interations' feedback. +together will need to be informed by subsequent iterations' feedback. ## Hierarchical limits @@ -362,7 +362,7 @@ hierarchy. Choosing a proper solution will require a thoughtful research. b. Develop YAML model for limits. c. Build Rails SDK. d. Create examples showcasing usage of the new rate limits SDK. -**Phase 3**: Team Fanout of Rails SDK - Stage Groups +**Phase 3**: Team fan out of Rails SDK - Stage Groups a. Individual stage groups begin using the SDK built in Phase 2 for new limit and policies. b. Stage groups begin replacing historical adhoc limit implementations with the SDK. c. Provides means to monitor and observe the progress of the replacement effort. Ideally this is broken down to the `feature_category` level to drive group-level buy-in -- Owning Team. @@ -373,7 +373,7 @@ hierarchy. Choosing a proper solution will require a thoughtful research. **Phase 5**: SDK for Satellite Services - Owning Team a. Build Golang SDK. c. Create examples showcasing usage of the new rate limits SDK. -**Phase 6**: Team Fanout for Satellite Services - Stage Groups +**Phase 6**: Team fan out for Satellite Services - Stage Groups a. Individual stage groups being using the SDK built in Phase 5 for new limit and policies. b. Stage groups begin replacing historical adhoc limit implementations with the SDK. diff --git a/doc/architecture/blueprints/remote_development/img/remote_dev_15_7.png b/doc/architecture/blueprints/remote_development/img/remote_dev_15_7.png Binary files differnew file mode 100644 index 00000000000..d0849ded94f --- /dev/null +++ b/doc/architecture/blueprints/remote_development/img/remote_dev_15_7.png diff --git a/doc/architecture/blueprints/remote_development/img/remote_dev_15_7_1.png b/doc/architecture/blueprints/remote_development/img/remote_dev_15_7_1.png Binary files differnew file mode 100644 index 00000000000..330873380d4 --- /dev/null +++ b/doc/architecture/blueprints/remote_development/img/remote_dev_15_7_1.png diff --git a/doc/architecture/blueprints/remote_development/index.md b/doc/architecture/blueprints/remote_development/index.md new file mode 100644 index 00000000000..39ea2fb2948 --- /dev/null +++ b/doc/architecture/blueprints/remote_development/index.md @@ -0,0 +1,315 @@ +--- +status: proposed +creation-date: "2022-11-15" +authors: [ "@vtak" ] +coach: "@grzesiek" +approvers: [ "@ericschurter", "@oregand" ] +owning-stage: "~devops::create" +participating-stages: [] +--- + +# Remote Development + +## Summary + +Remote Development is a new architecture for our software-as-a-service platform that provides a more consistent user experience writing code hosted in GitLab. It may also provide additional features in the future, such as a purely browser-based workspace and the ability to connect to an already running VM/Container or to use a GitLab-hosted VM/Container. + +## Web IDE and Remote Development + +It is important to note that `Remote Development !== Web IDE`, and this is something we want to be explicit about in this document as the terms can become conflated when they shouldn't. Our new Web IDE is a separate ongoing effort that is running in parallel to Remote Development. + +These two separate categories do have some overlap as it is a goal to allow a user to connect a running workspace to the Web IDE, **but** this does not mean the two are dependent on one another. + +You can use the [Web IDE](../../../user/project/web_ide/index.md) to commit changes to a project directly from your web browser without installing any dependencies or cloning any repositories. The Web IDE, however, lacks a native runtime environment on which you would compile code, run tests, or generate real-time feedback in the IDE. For a more complete IDE experience, you can pair the Web IDE with a Remote Development workspace that has been properly configured to run as a host. + + + +## Long-term vision + +As a [new Software Developer to a team such as Sasha](https://about.gitlab.com/handbook/product/personas/#sasha-software-developer) with no local development environment, I should be able to: + +- Navigate to a repository on GitLab.com or self-managed. +- Click a button that will provide a list of current workspaces for this repository. +- Click a button that will create a new workspace or select an existing workspace from a list. +- Go through a configuration wizard that will let me select various options for my workspace (memory/CPU). +- Start up a workspace from the Web IDE and within a minute have a fully interactive terminal panel at my disposal. +- Make code changes, run tests, troubleshoot based on the terminal output, and commit new changes. +- Submit MRs of any kind without having to clone the repository locally or to manually update a local development environment. + +## User Flow Diagram + + + +## Terminology + +We use the following terms to describe components and properties of the Remote Development architecture. + +### Remote Development + +Remote Development allows you to use a secure development environment in the cloud that you can connect to from your local machine through a web browser or a client-based solution with the purpose of developing a software product there. + +#### Remote Development properties + +- Separate your development environment to avoid impacting your local machine configuration. +- Make it easy for new contributors to get started and keep everyone on a consistent environment. +- Use tools or runtimes not available on your local OS or manage multiple versions of them. +- Access an existing development environment from multiple machines or locations. + +Discouraged synonyms: VS Code for web, Remote Development Extension, browser-only WebIDE, Client only WebIDE + +### Workspace + +Container/VM-based developer machines providing all the tools and dependencies needed to code, build, test, run, and debug applications. + +#### Workspace properties + +- Workspaces should be isolated from each other by default and are responsible for managing the lifecycle of their components. This isolation can be multi-layered: namespace isolation, network isolation, resources isolation, node isolation, sandboxing containers, etc. ([reference](https://kubernetes.io/docs/concepts/security/multi-tenancy/)). +- A workspace should contain project components as well as editor components. +- A workspace should be a combination of resources that support cloud-based development environment. +- Workspaces are constrained by the amount of resources provided to them. + +### Legacy Web IDE + +The current production [Web IDE](../../../user/project/web_ide/index.md). + +#### Legacy Web IDE properties + +An advanced editor with commit staging that currently supports: + +- [Live Preview](../../../user/project/web_ide/index.md#live-preview) +- [Interactive Web Terminals](../../../user/project/web_ide/index.md#interactive-web-terminals-for-the-web-ide) + +### Web IDE + +VS Code for web - replacement of our current legacy Web IDE. + +#### Web IDE properties + +A package for bootstrapping GitLab context-aware Web IDE that: + +- Is built on top of Microsoft's VS Code. We customize and add VS Code features in the [GitLab fork of the VS Code project](https://gitlab.com/gitlab-org/gitlab-web-ide-vscode-fork). +- Can be configured in a way that it connects to the workspace rather than only using the browser. When connected to a workspace, a user should be able to do the following from the Web IDE: + - Edit, build, or debug on a different OS than they are running locally. + - Make use of larger or more specialized hardware than their local machine for development. + - Separate developer environments to avoid conflicts, improve security, and speed up onboarding. + +### Remote Development Extension for Desktop + +Something that plugs into the desktop IDE and connects you to the workspace. + +#### Remote Development Extension for Desktop properties + +- Allows you to open any folder in a workspace. +- Should be desktop IDE agnostic. +- Should have access to local files or APIs. + +## Goals + +### A consistent experience + +Organizations should have the same user experience on our SaaS platform as they do on a self-managed GitLab instance. We want to abstract away the user's development environment to avoid impacting their local machine configuration. We also want to provide support for developing on the same operating system you deploy to or use larger or more specialized hardware. + +A major goal is that each member of a development team should have the same development experience minus any specialized local configuration. This will also make it easy for new contributors to get started and keep everyone on a consistent environment. + +### Increased availability + +A workspace should allow access to an existing development environment from multiple machines and locations across a single or multiple teams. It should also allow a user to make use of tools or runtimes not available on their local OS or manage multiple versions of them. + +Additionally, Remote Development workspaces could provide a way to implement disaster recovery if we are able to leverage the capabilities of [Pods](../../../architecture/blueprints/pods/index.md). + +### Scalability + +As an organization begins to scale, they quickly realize the need to support additional types of projects that might require extensive workflows. Remote Development workspaces aim to solve that issue by abstracting away the burden of complex machine configuration, dependency management, and possible data-seeding issues. + +To facilitate working on different features across different projects, Remote Development should allow each user to provision multiple workspaces to enable quick context switching. + +Eventually, we should be able to allow users to vertically scale their workspaces with more compute cores, memory, and other resources. If a user is currently working against a 2 CPU and 4 GB RAM workspace but comes to find they need more CPU, they should be able to upgrade their compute layer to something more suitable with a click or CLI command within the workspace. + +### Provide built-in security and enterprise readiness + +As Remote Development becomes a viable replacement for Virtual Desktop Infrastructure solutions, they must be secure and support enterprise requirements, such as role-based access control and the ability to remove all source code from developer machines. + +### Accelerate project and developer onboarding + +As a zero-install development environment that runs in your browser, Remote Development makes it easy for anyone to join your team and contribute to a project. + +### Regions + +GitLab.com is only hosted within the United States of America. Organizations located in other regions have voiced demand for local SaaS offerings. BYO infrastructure helps work in conjunction with [GitLab Regions](https://gitlab.com/groups/gitlab-org/-/epics/6037) because a user's workspace may be deployed within different geographies. The ability to deploy workspaces to different geographies might also help to solve data residency and compliance problems. + +## High-level architecture problems to solve + +A number of technical issues need to be resolved to implement a stable Remote Development offering. This section will be expanded. + +- Who is our main persona for BYO infrastructure? +- How do users authenticate? +- How do we support more than one IDE? +- How are workspaces provisioned? +- How can workspaces implement disaster recovery capabilities? +- If we cannot use SSH, what are the viable alternatives for establishing a secure WebSocket connection? +- Are we running into any limitations in functionality with the Web IDE by not having it running in the container itself? For example, are we going to get code completion, linting, and language server type features to work with our approach? +- How will our environments be provisioned, managed, created, destroyed, etc.? +- To what extent do we need to provide the user with a UI to interact with the provisioned environments? +- How will the files inside the workspace get live updated based on changes in the Web IDE? Are we going to use a [CRDT](https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type)-like setup to patch files in a container? Are we going to generate a diff and send it though a WebSocket connection? + +## Iteration plan + +We can't ship the entire Remote Development architecture in one go - it is too large. Instead, we are adopting an iteration plan that provides value along the way. + +- Use GitLab Agent for Kubernetes Remote Development Module. +- Integrate Remote Development with the UI and Web IDE. +- Improve security and usability. + +### High-level approach + +The nuts and bolts are being worked out at [Remote Development GA4K Architecture](https://gitlab.com/gitlab-org/remote-development/gitlab-remote-development-docs/-/blob/main/doc/architecture.md) to keep a SSoT. Once we have hammered out the details, we'll replace this section with the diagram in the above repository. + +### Iteration 0: [GitLab Agent for Kubernetes Remote Development Module (plumbing)](https://gitlab.com/groups/gitlab-org/-/epics/9138) + +#### Goals + +- Use the [GitLab Agent](../../../user/clusters/agent/index.md) integration. +- Create a workspace in a Kubernetes cluster based on a `devfile` in a public repository. +- Install the IDE and dependencies as defined. +- Report the status of the environment (via the terminal or through an endpoint). +- Connect to an IDE in the workspace. + +#### Requirements + +- Remote environment running on a Kubernetes cluster based on a `devfile` in a repo. + +These are **not** part of Iteration 0: + +- Authentication/authorization with GitLab and a user. +- Integration of Remote Development with the GitLab UI and Web IDE. +- Using GA4K instead of an Ingress controller. + +#### Assumptions + +- We will use [`devworkspace-operator` v0.17.0 (latest version)](https://github.com/devfile/devworkspace-operator/releases/tag/v0.17.0). A prerequisite is [`cert-manager`](https://github.com/devfile/devworkspace-operator#with-yaml-resources). +- We have an Ingress controller ([Ingress-NGINX](https://github.com/kubernetes/ingress-nginx)), which is accessible over the network. +- The initial server is stubbed. + +#### Success criteria + +- Using GA4K to communicate with the Kubernetes API from the `remote_dev` agent module. +- All calls to the Kubernetes API are done through GA4K. +- A workspace in a Kubernetes cluster created using DevWorkspace Operator. + +### Iteration 1: [Rails endpoints, authentication, and authorization](https://gitlab.com/groups/gitlab-org/-/epics/9323) + +#### Goals + +- Add endpoints in Rails to accept work from a user. +- Poll Rails for work from KAS. +- Add authentication and authorization to the workspaces created in the Kubernetes cluster. +- Extend the GA4K `remote_dev` agent module to accept more types of work (get details of a workspace, list workspaces for a user, etc). +- Build an editor injector for the GitLab fork of VS Code. + +#### Requirements + +- [GitLab Agent for Kubernetes Remote Development Module (plumbing)](https://gitlab.com/groups/gitlab-org/-/epics/9138) is complete. + +These are **not** part of Iteration 1: + +- Integration of Remote Development with the GitLab UI and Web IDE. +- Using GA4K instead of an Ingress controller. + +#### Assumptions + +- TBA + +#### Success criteria + +- Poll Rails for work from KAS. +- Rails endpoints to create/delete/get/list workspaces. +- All requests are correctly authenticated and authorized except where the user has requested the traffic to be public (for example, opening a server while developing and making it public). +- A user can create a workspace, start a server on that workspace, and have that traffic become private/internal/public. +- We are using the GitLab fork of VS Code as an editor. + +### Iteration 2: [Integrate Remote Development with the UI and Web IDE](https://gitlab.com/groups/gitlab-org/-/epics/9169) + +#### Goals + +- Allow users full control of their workspaces via the GitLab UI. + +#### Requirements + +- [GitLab Agent for Kubernetes Remote Development Module](https://gitlab.com/groups/gitlab-org/-/epics/9138). + +These are **not** part of Iteration 2: + +- Usability improvements +- Security improvements + +#### Success criteria + +- Be able to list/create/delete/stop/start/restart workspaces from the UI. +- Be able to create workspaces for the user in the Web IDE. +- Allow the Web IDE terminal to connect to different containers in the workspace. +- Configure DevWorkspace Operator for user-expected configuration (30-minute workspace timeout, a separate persistent volume for each workspace that is deleted when the workspace is deleted, etc.). + +### Iteration 3: [Improve security and usability](https://gitlab.com/groups/gitlab-org/-/epics/9170) + +#### Goals + +- Improve security and usability of our Remote Development solution. + +#### Requirements + +- [Integrate Remote Development with the UI and Web IDE](https://gitlab.com/groups/gitlab-org/-/epics/9169) is complete. + +#### Assumptions + +- We are allowing for internal feedback and closed/early customer feedback that can be iterated on. +- We have explored or are exploring the feasibility of using GA4K with Ingresses in [Solving Ingress problems for Remote Development](https://gitlab.com/gitlab-org/gitlab/-/issues/378998). +- We have explored or are exploring Kata containers for providing root access to workspace users in [Investigate Kata Containers / Firecracker / gVisor](https://gitlab.com/gitlab-org/gitlab/-/issues/367043). +- We have explored or are exploring how Ingress/Egress requests cannot be misused from [resources within or outside the cluster](https://gitlab.com/gitlab-org/remote-development/gitlab-remote-development-docs/-/blob/main/doc/securing-the-workspace.md) (security hardening). + +#### Success criteria + +Add options to: + +- Create different classes of workspaces (1gb-2cpu, 4gb-8cpu, etc.). +- Vertically scale up workspace resources. +- Inject secrets from a GitLab user/group/repository. +- Configure timeouts of workspaces at multiple levels. +- Allow users to expose endpoints in their workspace (for example, not allow anyone in the organization to expose any endpoint publicly). + +## Market analysis + +We have conducted a market analysis to understand the broader market and what others can offer us by way of open-source libraries, integrations, or partnership opportunities. We have broken down the effort into a set of issues where we investigate each potential competitor/pathway/partnership as a spike. + +- [Market analysis](https://gitlab.com/groups/gitlab-org/-/epics/8131) +- [YouTube results](https://www.youtube.com/playlist?list=PL05JrBw4t0KrRQhnSYRNh1s1mEUypx67-) + +### Next Steps + +While our spike proved fruitful, we have paused this investigation until we reach our goals in [Viable Maturity](https://gitlab.com/groups/gitlab-org/-/epics/9190). + +## Che versus a custom-built solution + +After an investigation into using [Che](https://gitlab.com/gitlab-org/gitlab/-/issues/366052) as our backend to accelerate Remote Development, we ultimately opted to [write our own custom-built solution](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/97449#note_1131215629). + +Some advantages of us opting to write our own custom-built solution are: + +- We can still use the core DevWorkspace Operator and build on top of it. +- It is easier to add support for other configurations apart from `devfile` in the future if the need arises. +- We have the ability to choose which tech stack to use (for example, instead of using Traefik which is used in Che, explore NGINX itself or use GitLab Agent for Kubernetes). + +## Links + +- [Remote Development presentation](https://docs.google.com/presentation/d/1XHH_ZilZPufQoWVWViv3evipI-BnAvRQrdvzlhBuumw/edit#slide=id.g131f2bb72e4_0_8) +- [Category Strategy epic](https://gitlab.com/groups/gitlab-org/-/epics/7419) +- [Minimal Maturity epic](https://gitlab.com/groups/gitlab-org/-/epics/9189) +- [Viable Maturity epic](https://gitlab.com/groups/gitlab-org/-/epics/9190) +- [Complete Maturity epic](https://gitlab.com/groups/gitlab-org/-/epics/9191) +- [Bi-weekly sync](https://docs.google.com/document/d/1hWVvksIc7VzZjG-0iSlzBnLpyr-OjwBVCYMxsBB3h_E/edit#) +- [Market analysis and architecture](https://gitlab.com/groups/gitlab-org/-/epics/8131) +- [GA4K Architecture](https://gitlab.com/gitlab-org/remote-development/gitlab-remote-development-docs/-/blob/main/doc/architecture.md) +- [BYO infrastructure](https://gitlab.com/groups/gitlab-org/-/epics/8290) +- [Browser runtime](https://gitlab.com/groups/gitlab-org/-/epics/8291) +- [GitLab-hosted infrastructure](https://gitlab.com/groups/gitlab-org/-/epics/8292) +- [Browser runtime spike](https://gitlab.com/gitlab-org/gitlab-web-ide/-/merge_requests/58). +- [Remote Development direction](https://about.gitlab.com/direction/create/editor/remote_development) +- [Ideal user journey](https://about.gitlab.com/direction/create/editor/remote_development/#ideal-user-journey) diff --git a/doc/architecture/blueprints/runner_scaling/index.md b/doc/architecture/blueprints/runner_scaling/index.md index 24c6820f94a..8eb6bfd2551 100644 --- a/doc/architecture/blueprints/runner_scaling/index.md +++ b/doc/architecture/blueprints/runner_scaling/index.md @@ -234,7 +234,7 @@ them each separately. etc... This information is very provider specific. - **VM lifecycle management**. Multiple machines will be created and a system must keep track of which machines belong to this executor. Typically - a cloud provider will have a way to manage a set of homogenous machines. + a cloud provider will have a way to manage a set of homogeneous machines. E.g. GCE Instance Group. The basic operations are increase, decrease and usually delete a specific machine. - **VM autoscaling**. In addition to low-level lifecycle management, @@ -273,7 +273,7 @@ interfaces. Within the `docker+autoscaling` executor the [`machineExecutor`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/executors/docker/machine/machine.go#L19) type has a [`Machine`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/helpers/docker/machine.go#L7) -interface which it uses to aquire a VM during the common [`Prepare`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/executors/docker/machine/machine.go#L71) +interface which it uses to acquire a VM during the common [`Prepare`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/executors/docker/machine/machine.go#L71) phase. This abstraction primarily creates, accesses and deletes VMs. There is no current abstraction for the VM autoscaling logic. It is tightly @@ -372,7 +372,7 @@ provide a context and an environment in which a build will be executed by one of the Custom Executors. There are multiple solutions to implementing a custom provider abstraction. We -can use raw Go plugins, Hashcorp's Go Plugin, HTTP interface or gRPC based +can use raw Go plugins, HashiCorp's Go Plugin, HTTP interface or gRPC based facade service. There are many solutions, and we want to choose the most optimal one. In order to do that, we will describe the solutions in a separate document, define requirements and score the solution accordingly. This will @@ -390,18 +390,18 @@ Rationale: [Description of the Custom Executor Provider proposal](https://gitlab We can introduce a more simple version of the `Machine` abstraction in the form of a "Fleeting" interface. Fleeting provides a low-level interface to -a homogenous VM group which allows increasing and decreasing the set size +a homogeneous VM group which allows increasing and decreasing the set size as well as consuming a VM from within the set. Plugins for cloud providers and other VM sources are implemented via the -Hashicorp go-plugin library. This is in practice gRPC over STDIN/STDOUT +HashiCorp go-plugin library. This is in practice gRPC over STDIN/STDOUT but other wire protocols can be used also. In order to make use of the new interface, the autoscaling logic is pulled out of the Docker Executor and placed into a new Taskscaler library. This places the concerns of VM lifecycle, VM shape and job routing within -the plugin. It also places the conern of VM autoscaling into a separate +the plugin. It also places the concern of VM autoscaling into a separate component so it can be used by multiple Runner Executors (not just `docker+autoscaling`). Rationale: [Description of the InstanceGroup / Fleeting proposal](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/28848#note_823430883) diff --git a/doc/architecture/blueprints/runner_tokens/index.md b/doc/architecture/blueprints/runner_tokens/index.md index 3f8a27e503d..7a282649c5c 100644 --- a/doc/architecture/blueprints/runner_tokens/index.md +++ b/doc/architecture/blueprints/runner_tokens/index.md @@ -14,7 +14,7 @@ CI/CD jobs in a reliable and concurrent environment. Ever since the beginnings of the service as a Ruby program, runners are registered in a GitLab instance with a registration token - a randomly generated string of text. The registration token is unique for its given scope (instance, group, or project). The registration token proves that the party that registers the runner has -administrator access to the instance, group, or project to which the runner is registered. +administrative access to the instance, group, or project to which the runner is registered. This approach has worked well in the initial years, but some major known issues started to become apparent as the target audience grew: @@ -37,49 +37,122 @@ We call this new mechanism the "next GitLab Runner Token architecture". The proposal addresses the issues of a _single token per scope_ and _token storage_ by eliminating the need for a registration token. Runner creation happens -in the GitLab Runners settings page for the given scope, in the context of the logged-in user -, which provides traceability. The page provides instructions to configure the newly-created -runner in supported environments. +in the GitLab Runners settings page for the given scope, in the context of the logged-in user, +which provides traceability. The page provides instructions to configure the newly-created +runner in supported environments using the existing `gitlab-runner register` command. -The runner configuration will be generated through a new `deploy` command, which will leverage -the `/runners/verify` REST endpoint to ensure the validity of the authentication token. The remaining concerns become non-issues due to the elimination of the registration token. -The configuration can be applied across many machines by reusing the same instructions. -A unique system identifier will be generated automatically if a value is missing from -the runner entry in the `config.toml` file. This allows differentiating systems sharing the same -runner token (for example, in auto-scaling scenarios), and is crucial for the proper functioning of our -long-polling mechanism when the same authentication token is shared across two or more runner managers. +### Using the authentication token in place of the registration token + +<!-- vale gitlab.Spelling = NO --> +In this proposal, runners created in the GitLab UI are assigned authentication tokens prefixed with +`glrt-` (**G**it**L**ab **R**unner **T**oken). +<!-- vale gitlab.Spelling = YES --> +The prefix allows the existing `register` command to use the authentication token _in lieu_ +of the current registration token (`--registration-token`), requiring minimal adjustments in +existing workflows. +The authentication token is shown to the user only once - after completing the creation flow - to +discourage unintended reuse. + +Given that the runner is pre-created through the GitLab UI, the `register` command fails if +provided with arguments that are exposed in the runner creation form. +Some examples are `--tag-list`, `--run-untagged`, `--locked`, or `--access-level` as these are +sensitive parameters that should be decided at creation time by an administrator/owner. +The runner configuration is generated through the existing `register` command, which can behave in +two different ways depending on whether it is supplied a registration token or an authentication +token in the `--registration-token` argument: + +| Token type | Behavior | +| ---------- | -------- | +| Registration token | Leverages the `POST /api/v4/runners` REST endpoint to create a new runner, creating a new entry in `config.toml`. | +| Authentication token | Leverages the `POST /api/v4/runners/verify` REST endpoint to ensure the validity of the authentication token. Creates an entry in `config.toml` file and a `system_id` value in a sidecar file if missing (`.runner_system_id`). | + +### Transition period + +During a transition period, legacy tokens ("registration tokens") continue to be shown on the +GitLab Runners settings page and to be accepted by the `gitlab-runner register` command. +The legacy workflow is nevertheless discouraged in the UI. +Users are steered towards the new flow consisting of creating the runner in the UI and using the +resulting authentication token with the `gitlab-runner register` command as they do today. +This approach reduces disruption to users responsible for deploying runners. + +### Reusing the runner authentication token across many machines + +In the existing model, a new runner is created whenever a new worker is required. This +has led to many situations where runners are left behind and become stale. + +In the proposed model, a `ci_runners` table entry describes a configuration that the user can reuse +across multiple machines. +A unique system identifier is [generated automatically](#generating-a-system_id-value) whenever the +runner application starts up or the configuration is saved. +This allows differentiating the context in which the runner is being used. + +The `system_id` value complements the short runner token that is currently used to identify a +runner in command line output, CI job logs, and GitLab UI. Given that the creation of runners involves user interaction, it should be possible to eventually lower the per-plan limit of CI runners that can be registered per scope. -### Auto-scaling scenarios (for example Helm chart) +#### Generating a `system_id` value -In the existing model, a new runner is created whenever a new worker is required. This -has led to many situations where runners are left behind and become stale. +We ensure that a unique system identifier is assigned at all times to a `gitlab-runner` +installation. +The ID is derived from an existing machine identifier such as `/etc/machine-id` (on Linux) and +hashed for privacy, in which case it is prefixed with `s_`. +If an ID is not available, a random string is used instead, in which case it is prefixed with `r_`. -In the proposed model, a `ci_runners` table entry describes a configuration, -which the runner could reuse across multiple machines. This allows differentiating the context in -which the runner is being used. In situations where we must differentiate between runners -that reuse the same configuration, we can use the unique system identifier to track all -unique "runners" that are executed in context of a single `ci_runners` model. This unique -system identifier would be present in the Runner's `config.toml` configuration file and -initially set when generating the new `[[runners]]` configuration by means of the `deploy` command. -Legacy files that miss values for unique system identifiers will get rewritten automatically with new values. +This unique ID identifies the `gitlab-runner` process and is sent +on `POST /api/v4/jobs` requests for all runners in the `config.toml` file. + +The ID is generated and saved both at `gitlab-runner` startup and whenever the configuration is +saved to disk. +Instead of saving the ID at the root of `config.toml` though, we save it to a new file that lives +next to it - `.runner_system_id`. The goal for this new file is to make it less likely that IDs +get reused due to manual copying of the `config.toml` file + +```plain +s_cpwhDr7zFz4xBJujFeEM +``` ### Runner identification in CI jobs -For users to identify the machine where the job was executed, the unique identifier will need to be visible in CI job contexts. +For users to identify the machine where the job was executed, the unique identifier needs to be +visible in CI job contexts. As a first iteration, GitLab Runner will include the unique system identifier in the build logs, wherever it publishes the short token SHA. -Given that the runner will potentially be reused with different unique system identifiers, -we can store the unique system ID. This ensures the unique system ID maps to a GitLab Runner's `config.toml` entry with -the runner token. The `ci_runner_machines` would hold information about each unique runner machine, -with information when runner last connected, and what type of runner it was. The relevant fields -will be moved from the `ci_runners`. -The `ci_builds_runner_session` (or `ci_builds` or `ci_builds_metadata`) will reference +Given that the runner can potentially be reused with different unique system identifiers, +we should store the unique system ID in the database. +This ensures the unique system ID maps to a GitLab Runner's `system_id` value with the runner token. +A new `ci_runner_machines` table holds information about each unique runner machine, +with information regarding when the runner last connected, and what type of runner it was. + +In the long term, the relevant fields are to be moved from the `ci_runners` into +`ci_runner_machines`. +Until the removal milestone though, they should be kept in the `ci_runners` as a fallback when a +matching `ci_runner_machines` record does not exist. +An expected scenario is the case when the table is created but the runner hasn't pinged the GitLab +instance (for example if the runner is offline). + +In addition, we should add the following columns to `ci_runners`: + +- a `user_id` column to keep track of who created a runner; +- a `registration_type` enum column to `ci_runners` to signal whether a runner has been created + using the legacy `register` method, or the new UI-based method. + Possible values are `:registration_token` and `:authenticated_user`. + This allows the stale runner cleanup service to determine which runners to clean up, and allows + future uses that may not be apparent. + +```sql +CREATE TABLE ci_runner ( + ... + user_id bigint + registration_type int8 +) +``` + +The `ci_builds_runner_session` (or `ci_builds` or `ci_builds_metadata`) shall reference `ci_runner_machines`. We might consider a more efficient way to store `contacted_at` than updating the existing record. @@ -110,7 +183,8 @@ CREATE TABLE ci_runner_machines ( - Runners can always be traced back to the user who created it, using the audit log; - The claims of a CI runner are known at creation time, and cannot be changed from the runner (for example, changing the `access_level`/`protected` flag). Authenticated users - may however still edit these settings through the GitLab UI. + may however still edit these settings through the GitLab UI; +- Easier cleanup of stale runners, which doesn't touch the `ci_runner` table. ## Details @@ -121,53 +195,95 @@ token to register new runners. The new workflow looks as follows: - 1. The user opens the Runners settings page; + 1. The user opens the Runners settings page (instance, group, or project level); 1. The user fills in the details regarding the new desired runner, namely description, tags, protected, locked, etc.; 1. The user clicks `Create`. That results in the following: - 1. Creates a new runner in the `ci_runners` table (and corresponding authentication token); + 1. Creates a new runner in the `ci_runners` table (and corresponding `glrt-` prefixed authentication token); 1. Presents the user with instructions on how to configure this new runner on a machine, with possibilities for different supported deployment scenarios (e.g. shell, `docker-compose`, Helm chart, etc.) - This information contains a token which will only be available to the user once, and the UI - will make it clear to the user that the value will not be shown again, as registering the same runner multiple times + This information contains a token which is available to the user only once, and the UI + makes it clear to the user that the value shall not be shown again, as registering the same runner multiple times is discouraged (though not impossible). - 1. The user copies and pastes the instructions for the intended deployment scenario (a `deploy` command), leading to the following actions: + 1. The user copies and pastes the instructions for the intended deployment scenario (a `register` command), leading to the following actions: - 1. Upon executing the new `gitlab-runner deploy` command in the instructions, `gitlab-runner` will perform - a call to the `POST /runners/verify` with the given runner token; - 1. If the `POST /runners/verify` GitLab endpoint validates the token, the `config.toml` file will be populated with the configuration. + 1. Upon executing the new `gitlab-runner register` command in the instructions, `gitlab-runner` performs + a call to the `POST /api/v4/runners/verify` with the given runner token; + 1. If the `POST /api/v4/runners/verify` GitLab endpoint validates the token, the `config.toml` + file is populated with the configuration; + 1. Whenever a runner pings for a job, the respective `ci_runner_machines` record is + ["upserted"](https://en.wiktionary.org/wiki/upsert) with the latest information about the + runner (with Redis cache in front of it like we do for Runner heartbeats). - The `gitlab-runner deploy` will also accept executor-specific arguments - currently present in the `register` command. +As part of the transition period, we provide admins and top-level group owners with an +instance/group-level setting (`allow_runner_registration_token`) to disable the legacy registration +token functionality and enforce using only the new workflow. +Any attempt by a `gitlab-runner register` command to hit the `POST /api/v4/runners` endpoint +to register a new runner with a registration token results in a `HTTP 410 Gone` status code. -As part of the transition period, we will provide admins and top-level group owners with a instance/group-level setting to disable -the legacy registration token functionality and enforce using only the new workflow. -Any attempt by a `gitlab-runner register` command to hit the `POST /runners` endpoint to register a new runner -will result in a `HTTP 410 - Gone` status code. The instance setting is inherited by the groups -, which means that if the legacy registration method is disabled at the instance method, the descendant groups/projects will also mandatorily -prevent the legacy registration method. +The instance setting is inherited by the groups. This means that if the legacy registration method +is disabled at the instance method, the descendant groups/projects mandatorily prevents the legacy +registration method. The registration token workflow is to be deprecated (with a deprecation notice printed by the `gitlab-runner register` command) and removed at a future major release after the concept is proven stable and customers have migrated to the new workflow. ### Handling of legacy runners -Legacy versions of GitLab Runner will not send the unique system identifier in its requests, and we +Legacy versions of GitLab Runner do not send the unique system identifier in its requests, and we will not change logic in Workhorse to handle unique system IDs. This can be improved upon in the -future once the legacy registration system is removed, and runners have been upgraded to newer +future after the legacy registration system is removed, and runners have been upgraded to newer versions. -Not using the unique system ID means that all connected runners with the same token will be +Job pings from such legacy runners results in a `ci_runner_machines` record containing a +`<legacy>` `machine_id` field value. + +Not using the unique system ID means that all connected runners with the same token are notified, instead of just the runner matching the exact system identifier. While not ideal, this is not an issue per-se. -### Helm chart +### `ci_runner_machines` record lifetime + +New records are created when the runner pings the GitLab instance for new jobs, if a record matching +the `token`+`system_id` does not already exist. + +Due to the time-decaying nature of the `ci_runner_machines` records, they are automatically +cleaned after 7 days after the last contact from the respective runner. + +### Required adaptations -The `runnerRegistrationToken` entry in the [`values.yaml` file](https://gitlab.com/gitlab-org/charts/gitlab-runner/-/blob/a70bc29a903b79d5675bb0c45d981adf8b7a8659/values.yaml#L52) -will be retired. The `runnerRegistrationToken` entry will be replaced by the existing `runnerToken` value, which will be passed -to the new `gitlab-runner deploy` command in [`configmap.yaml`](https://gitlab.com/gitlab-org/charts/gitlab-runner/-/blob/a70bc29a903b79d5675bb0c45d981adf8b7a8659/templates/configmap.yaml#L116). +#### Migration to `ci_runner_machines` table + +When details from `ci_runner_machines` are needed, we need to fall back to the existing fields in +`ci_runner` if a match is not found in `ci_runner_machines`. + +#### REST API + +API endpoints receiving runner tokens should be changed to also take an optional +`system_id` parameter, sent alongside with the runner token (most often as a JSON parameter on the +request body). + +#### GraphQL `CiRunner` type + +The [`CiRunner` type](../../../api/graphql/reference/index.md#cirunner) closely reflects the +`ci_runners` model. This means that machine information such as `ipAddress`, `architectureName`, +and `executorName` among others are no longer singular values in the proposed approach. +We can live with that fact for the time being and start returning lists of unique values, separated +by commas. +The respective `CiRunner` fields must return the values for the `ci_runner_machines` entries +(falling back to `ci_runner` record if non-existent). + +#### Stale runner cleanup + +The functionality to +[clean up stale runners](../../../ci/runners/configure_runners.md#clean-up-stale-runners) needs +to be adapted to clean up `ci_runner_machines` records instead of `ci_runners` records. + +At some point after the removal of the registration token support, we'll want to create a background +migration to clean up stale runners that have been created with a registration token (leveraging the +enum column created in the `ci_runners` table. ### Runner creation through API @@ -176,21 +292,66 @@ using PAT tokens for example - such that every runner is associated with an owne ## Implementation plan +### Stage 1 - Deprecations + +| Component | Milestone | Changes | +|------------------|----------:|---------| +| GitLab Rails app | `15.6` | Deprecate `POST /api/v4/runners` endpoint for `17.0`. This hinges on a [proposal](https://gitlab.com/gitlab-org/gitlab/-/issues/373774) to allow deprecating REST API endpoints for security reasons. | +| GitLab Runner | `15.6` | Add deprecation notice for `register` command for `17.0`. | +| GitLab Runner Helm Chart | `15.6` | Add deprecation notice for `runnerRegistrationToken` command for `17.0`. | +| GitLab Runner Operator | `15.6` | Add deprecation notice for `runner-registration-token` command for `17.0`. | +| GitLab Runner / GitLab Rails app | `15.7` | Add deprecation notice for registration token reset for `17.0`. | + +### Stage 2 - Prepare `gitlab-runner` for `system_id` + +| Component | Milestone | Changes | +|------------------|----------:|---------| +| GitLab Runner | `15.x` | Ensure a sidecar TOML file exists with a `system_id` value.<br/>Log new system ID values with `INFO` level as they get assigned. | +| GitLab Runner | `15.x` | Log unique system ID in the build logs. | +| GitLab Runner | `15.x` | Label Prometheus metrics with unique system ID. | +| GitLab Runner | `15.x` | Prepare `register` command to fail if runner server-side configuration options are passed together with a new `glrt-` token. | + +### Stage 3 - Database changes + +| Component | Milestone | Changes | +|------------------|----------:|---------| +| GitLab Rails app | | Create database migration to add columns to `ci_runners` table. | +| GitLab Rails app | | Create database migration to add `ci_runner_machines` table. | +| GitLab Rails app | | Create database migration to add `ci_runner_machines.machine_id` foreign key to `ci_builds_runner_session` table. | +| GitLab Rails app | | Create database migrations to add `allow_runner_registration_token` setting to `application_settings` and `namespace_settings` tables (default: `true`). | +| GitLab Runner | | Use runner token + `system_id` JSON parameters in `POST /jobs/request` request in the [heartbeat request](https://gitlab.com/gitlab-org/gitlab/blob/c73c96a8ffd515295842d72a3635a8ae873d688c/lib/api/ci/helpers/runner.rb#L14-20) to update the `ci_runner_machines` cache/table. | +| GitLab Runner | | Start sending `system_id` value in `POST /jobs/request` request and other follow-up requests that require identifying the unique system. | +| GitLab Rails app | | Create service similar to `StaleGroupRunnersPruneCronWorker` service to clean up `ci_runner_machines` records instead of `ci_runners` records.<br/>Existing service continues to exist but focuses only on legacy runners. | + +### Stage 4 - New UI + +| Component | Milestone | Changes | +|------------------|----------:|---------| +| GitLab Runner | | Implement new GraphQL user-authenticated API to create a new runner. | +| GitLab Runner | | [Add prefix to newly generated runner authentication tokens](https://gitlab.com/gitlab-org/gitlab/-/issues/383198). | +| GitLab Rails app | | Implement UI to create new runner. | +| GitLab Rails app | | GraphQL changes to `CiRunner` type. | +| GitLab Rails app | | UI changes to runner details view (listing of platform, architecture, IP address, etc.) (?) | + +### Stage 5 - Optional disabling of registration token + +| Component | Milestone | Changes | +|------------------|----------:|---------| +| GitLab Rails app | | Add UI to allow disabling use of registration tokens at project or group level. | +| GitLab Rails app | `16.0` | Introduce `:disable_runner_registration_tokens` feature flag (enabled by default) to control whether use of registration tokens is allowed. | +| GitLab Rails app | | Make [`POST /api/v4/runners` endpoint](../../../api/runners.md#register-a-new-runner) permanently return `HTTP 410 Gone` if either `allow_runner_registration_token` setting or `:disable_runner_registration_tokens` feature flag disables registration tokens.<br/>A future v5 version of the API should return `HTTP 404 Not Found`. | +| GitLab Rails app | | Start refusing job requests that don't include a unique ID, if either `allow_runner_registration_token` setting or `:disable_runner_registration_tokens` feature flag disables registration tokens. | +| GitLab Rails app | | Hide legacy UI showing registration with a registration token, if `:disable_runner_registration_tokens` feature flag disables registration tokens. | + +### Stage 6 - Removals + | Component | Milestone | Changes | -|------------------|-----------|---------| -| GitLab Rails app | `15.x` (latest at `15.6`) | Deprecate `POST /api/v4/runners` endpoint for `16.0`. This hinges on a [proposal](https://gitlab.com/gitlab-org/gitlab/-/issues/373774) to allow deprecating REST API endpoints for security reasons. | -| GitLab Runner | `15.x` (latest at `15.8`) | Add deprecation notice for `register` command for `16.0`. | -| GitLab Runner | `15.x` | Ensure all runner entries in `config.toml` have unique system identifier values assigned. Log new system ID values with `INFO` level as they get created. | -| GitLab Runner | `15.x` | Start additionally logging unique system ID anywhere we log the runner short SHA. | -| GitLab Rails app | `15.x` | Create database migrations to add settings from `application_settings` and `namaspace_settings` tables. | -| GitLab Runner | `15.x` | Start sending `unique_id` value in `POST /jobs/request` request and other follow-up requests that require identifying the unique system. | -| GitLab Runner | `15.x` | Implement new user-authenticated API (REST and GraphQL) to create a new runner. | -| GitLab Rails app | `15.x` | Implement UI to create new runner. | -| GitLab Runner | `16.0` | Remove `register` command and support for `POST /runners` endpoint. | -| GitLab Rails app | `16.0` | Remove legacy UI showing registration with a registration token. | -| GitLab Rails app | `16.0` | Create database migrations to remove settings from `application_settings` and `namaspace_settings` tables. | -| GitLab Rails app | `16.0` | Make [`POST /api/v4/runners` endpoint](../../../api/runners.md#register-a-new-runner-deprecated) permanently return `410 Gone`. A future v5 version of the API would return `404 Not Found`. | -| GitLab Rails app | `16.0` | Start refusing job requests that don't include a unique ID. | +|------------------|----------:|---------| +| GitLab Rails app | `17.0` | Remove legacy UI showing registration with a registration token. | +| GitLab Runner | `17.0` | Remove runner model arguments from `register` command (for example `--run-untagged`, `--tag-list`, etc.) | +| GitLab Rails app | `17.0` | Create database migrations to drop `allow_runner_registration_token` setting columns from `application_settings` and `namespace_settings` tables. | +| GitLab Rails app | `17.0` | Create database migrations to drop:<br/>- `runners_registration_token`/`runners_registration_token_encrypted` columns from `application_settings`;<br/>- `runners_token`/`runners_token_encrypted` from `namespaces` table;<br/>- `runners_token`/`runners_token_encrypted` from `projects` table. | +| GitLab Rails app | `17.0` | Remove `:disable_runner_registration_tokens` feature flag. | ## Status diff --git a/doc/architecture/blueprints/work_items/index.md b/doc/architecture/blueprints/work_items/index.md index 75a9d8d76ad..101fdbf4c2d 100644 --- a/doc/architecture/blueprints/work_items/index.md +++ b/doc/architecture/blueprints/work_items/index.md @@ -60,8 +60,8 @@ All Work Item types share the same pool of predefined widgets and are customized | assignees | | | description | | | hierarchy | | -| [iteration](https://gitlab.com/gitlab-org/gitlab/-/issues/367456) | work_items_mvc_2 | -| [milestone](https://gitlab.com/gitlab-org/gitlab/-/issues/367463) | work_items_mvc_2 | +| [iteration](https://gitlab.com/gitlab-org/gitlab/-/issues/367456) | | +| [milestone](https://gitlab.com/gitlab-org/gitlab/-/issues/367463) | | | labels | | | start and due date | | | status\* | | diff --git a/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md b/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md index a0665e1c054..ceb56b01dcd 100644 --- a/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md +++ b/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md @@ -122,7 +122,7 @@ To use GitLab CI/CD with a Bitbucket Cloud repository: 1. In Bitbucket, create a `.gitlab-ci.yml` file to use the script to push pipeline success and failures to Bitbucket. Similar to the script added above, - this file is copied to the GitLab repo as part of the mirroring process. + this file is copied to the GitLab repository as part of the mirroring process. ```yaml stages: diff --git a/doc/ci/cloud_services/azure/index.md b/doc/ci/cloud_services/azure/index.md index b2f78648be9..b846ee4b792 100644 --- a/doc/ci/cloud_services/azure/index.md +++ b/doc/ci/cloud_services/azure/index.md @@ -17,7 +17,7 @@ Prerequisites: - Access to an existing Azure Subscription with `Owner` access level. - Access to the corresponding Azure Active Directory Tenant with at least the `Application Developer` access level. - A local installation of the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli). - Alternatively, you can follow all the steps below with the [Azure Cloud Shell](https://shell.azure.com/). + Alternatively, you can follow all the steps below with the [Azure Cloud Shell](https://portal.azure.com/#cloudshell/). - A GitLab project. To complete this tutorial: diff --git a/doc/ci/directed_acyclic_graph/index.md b/doc/ci/directed_acyclic_graph/index.md index 39f45471021..49bce75e183 100644 --- a/doc/ci/directed_acyclic_graph/index.md +++ b/doc/ci/directed_acyclic_graph/index.md @@ -10,7 +10,7 @@ type: reference > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/47063) in GitLab 12.2. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/206902) in GitLab 12.10. -A [directed acyclic graph](https://www.techopedia.com/definition/5739/directed-acyclic-graph-dag) can be +A [directed acyclic graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) can be used in the context of a CI/CD pipeline to build relationships between jobs such that execution is performed in the quickest possible manner, regardless how stages may be set up. @@ -38,10 +38,10 @@ It has a pipeline that looks like the following: | build | test | deploy | | ----- | ---- | ------ | -| build_a | test_a | deploy_a | -| build_b | test_b | deploy_b | -| build_c | test_c | deploy_c | -| build_d | test_d | deploy_d | +| `build_a` | `test_a` | `deploy_a` | +| `build_b` | `test_b` | `deploy_b` | +| `build_c` | `test_c` | `deploy_c` | +| `build_d` | `test_d` | `deploy_d` | Using a DAG, you can relate the `_a` jobs to each other separately from the `_b` jobs, and even if service `a` takes a very long time to build, service `b` doesn't diff --git a/doc/ci/environments/deployment_approvals.md b/doc/ci/environments/deployment_approvals.md index d7fa31b583b..a4815a85bc1 100644 --- a/doc/ci/environments/deployment_approvals.md +++ b/doc/ci/environments/deployment_approvals.md @@ -134,6 +134,8 @@ To approve or reject a deployment to a protected environment using the UI: 1. On the left sidebar, select **Deployments > Environments**. 1. Select the environment's name. 1. In the deployment's row, select **Approval options** (**{thumb-up}**). + Before approving or rejecting the deployment, you can view the number of approvals granted and + remaining, also who has approved or rejected it. 1. Optional. Add a comment which describes your reason for approving or rejecting the deployment. 1. Select **Approve** or **Reject**. @@ -154,6 +156,29 @@ curl --data "status=approved&comment=Looks good to me" \ --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/deployments/1/approval" ``` +### View the approval details of a deployment + +Prerequisites: + +- Permission to deploy to the protected environment. + +A deployment to a protected environment can only proceed after all required approvals have been +granted. + +To view the approval details of a deployment: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Deployments > Environments**. +1. Select the environment's name. +1. In the deployment's row, select **Approval options** (**{thumb-up}**). + +The approval status details are shown: + +- Eligible approvers +- Number of approvals granted, and number of approvals required +- Users who have granted approval +- History of approvals or rejections + ## How to see blocked deployments ### Using the UI diff --git a/doc/ci/environments/external_deployment_tools.md b/doc/ci/environments/external_deployment_tools.md new file mode 100644 index 00000000000..ff3172f0e02 --- /dev/null +++ b/doc/ci/environments/external_deployment_tools.md @@ -0,0 +1,88 @@ +--- +stage: Release +group: Release +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: reference +--- + +# Track deployments of an external deployment tool **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22513) in GitLab 12.5. + +While GitLab offers a [built-in deployment solution](index.md), you might prefer to use an external deployment tool, such as Heroku or ArgoCD. +GitLab can receive deployment events from these external tools and allows you to track the deployments within GitLab. +For example, the following features are available by setting up tracking: + +- [See when an merge request has been deployed, and to which environment](../../user/project/merge_requests/widgets.md#post-merge-pipeline-status). +- [Filter merge requests by environment or deployment date](../../user/project/merge_requests/index.md#filter-merge-requests-by-environment-or-deployment-date). +- [DevOps Research and Assessment (DORA) metrics](../../user/analytics/dora_metrics.md). +- [View environments and deployments](index.md#view-environments-and-deployments). +- [Track newly included merge requests per deployment](index.md#track-newly-included-merge-requests-per-deployment). + +NOTE: +Some of the features are not available because GitLab can't authorize and leverage those external deployments, including +[Protected Environments](protected_environments.md), [Deployment Approvals](deployment_approvals.md), [Deployment safety](deployment_safety.md), and [Environment rollback](index.md#environment-rollback). + +## How to set up deployment tracking + +External deployment tools usually offer a [webhook](https://en.wikipedia.org/wiki/Webhook) to execute an additional API request when deployment state is changed. +You can configure your tool to make a request to the GitLab [Deployment API](../../api/deployments.md). Here is an overview of the event and API request flow: + +- When a deployment starts running, [create a deployment with `running` status](../../api/deployments.md#create-a-deployment). +- When a deployment succeeds, [update the deployment status to `success`](../../api/deployments.md#update-a-deployment). +- When a deployment fails, [update the deployment status to `failed`](../../api/deployments.md#update-a-deployment). + +NOTE: +You can create a [project access token](../../user/project/settings/project_access_tokens.md) for the GitLab API authentication. + +### Example: Track deployments of ArgoCD + +You can use [ArgoCD webhook](https://argocd-notifications.readthedocs.io/en/stable/services/webhook/) to send deployment events to GitLab Deployment API. +Here is an example setup that creates a `success` deployment record in GitLab when ArgoCD successfully deploys a new revision: + +1. Create a new webhook. You can save the following manifest file and apply it by `kubectl apply -n argocd -f <manifiest-file-path>`: + + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: argocd-notifications-cm + data: + trigger.on-deployed: | + - description: Application is synced and healthy. Triggered once per commit. + oncePer: app.status.sync.revision + send: + - gitlab-deployment-status + when: app.status.operationState.phase in ['Succeeded'] and app.status.health.status == 'Healthy' + template.gitlab-deployment-status: | + webhook: + gitlab: + method: POST + path: /projects/<your-project-id>/deployments + body: | + { + "status": "success", + "environment": "production", + "sha": "{{.app.status.operationState.operation.sync.revision}}", + "ref": "main", + "tag": "false" + } + service.webhook.gitlab: | + url: https://gitlab.com/api/v4 + headers: + - name: PRIVATE-TOKEN + value: <your-access-token> + - name: Content-type + value: application/json + ``` + +1. Create a new subscription in your application: + + ```shell + kubectl patch app <your-app-name> -n argocd -p '{"metadata": {"annotations": {"notifications.argoproj.io/subscribe.on-deployed.gitlab":""}}}' --type merge + ``` + +NOTE: +If a deployment wasn't created as expected, you can troubleshoot with [`argocd-notifications` tool](https://argocd-notifications.readthedocs.io/en/stable/troubleshooting/). +For example, `argocd-notifications template notify gitlab-deployment-status <your-app-name> --recipient gitlab:argocd-notifications` +triggers API request immediately and renders an error message from GitLab API server if any. diff --git a/doc/ci/environments/index.md b/doc/ci/environments/index.md index c4672b9dc7e..0c412c85e47 100644 --- a/doc/ci/environments/index.md +++ b/doc/ci/environments/index.md @@ -45,15 +45,20 @@ Deployments show up in this list only after a deployment job has created them. ## Search environments -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10754) in GitLab 15.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10754) in GitLab 15.5. +> - [Searching environments within a folder](https://gitlab.com/gitlab-org/gitlab/-/issues/373850) was introduced in GitLab 15.7 with [Feature flag `enable_environments_search_within_folder`](https://gitlab.com/gitlab-org/gitlab/-/issues/382108). Enabled by default. To search environments by name: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Deployments > Environments**. -1. In the search bar, enter your search term. Matching applies from the - beginning of the environment name. For example, `devel` matches the - environment name `development`, but `elop` does not. +1. In the search bar, enter your search term. + - The length of your **search term should be 3 or more characters**. + - Matching applies from the beginning of the environment name. + - For example, `devel` matches the environment name `development`, but `elop` does not. + - For environments with a folder name format, matching applies after the base folder name. + - For example when the name is `review/test-app`, search term `test` matches `review/test-app`. + - Also searching with the folder name prefixed like `review/test` matches `review/test-app`. ## Types of environments @@ -340,7 +345,7 @@ and displayed at a post-merge pipeline in [merge request pages](../../user/proje To activate this tracking, your environment must be configured in the following: -- [Environment name](../yaml/index.md#environmentname) is not foldered with `/` (that is, top-level/long-lived environments), _OR_ +- [Environment name](../yaml/index.md#environmentname) is not using folders with `/` (that is, top-level/long-lived environments), _OR_ - [Environment tier](#deployment-tier-of-environments) is either `production` or `staging`. Here are the example setups of [`environment` keyword](../yaml/index.md#environment) in `.gitlab-ci.yml`: @@ -609,6 +614,20 @@ Because `stop_review_app` is set to `auto_stop_in: 1 week`, if a merge request is inactive for more than a week, GitLab automatically triggers the `stop_review_app` job to stop the environment. +#### Stop an environment through the UI + +NOTE: +To trigger an `on_stop` action and manually stop an environment from the +Environments view, the stop and deploy jobs must be in the same +[`resource_group`](../yaml/index.md#resource_group). + +To stop an environment in the GitLab UI: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Deployments > Environments**. +1. Next to the environment you want to stop, select **Stop**. +1. On the confirmation dialog box, select **Stop environment**. + #### Multiple stop actions for an environment > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22456) in GitLab 14.10 [with a flag](../../administration/feature_flags.md) named `environment_multiple_stop_actions`. Disabled by default. @@ -977,6 +996,7 @@ Instead, you need to delete the old environment and create a new one: - [Environments Dashboard](../environments/environments_dashboard.md): View a summary of each environment's operational health. **(PREMIUM)** - [Deployment safety](deployment_safety.md#restrict-write-access-to-a-critical-environment): Secure your deployments. +- [Track deployments of an external deployment tool](external_deployment_tools.md): Use an external deployment tool instead of built-in deployment solution. ## Troubleshooting diff --git a/doc/ci/environments/protected_environments.md b/doc/ci/environments/protected_environments.md index 5c120da32a0..638bcf77967 100644 --- a/doc/ci/environments/protected_environments.md +++ b/doc/ci/environments/protected_environments.md @@ -132,13 +132,13 @@ they have the following privileges: Users granted access to a protected environment, but not push or merge access to the branch deployed to it, are only granted access to deploy the environment. -[Invited groups](../../user/project/members/share_project_with_groups.md#share-a-project-with-a-group-of-users) added +[Invited groups](../../user/project/members/share_project_with_groups.md#share-a-project-with-a-group) added to the project with [Reporter role](../../user/permissions.md#project-members-permissions), appear in the dropdown list for deployment-only access. To add deployment-only access: 1. Create a group with members who are granted to access to the protected environment, if it doesn't exist yet. -1. [Invite the group](../../user/project/members/share_project_with_groups.md#share-a-project-with-a-group-of-users) to the project with the Reporter role. +1. [Invite the group](../../user/project/members/share_project_with_groups.md#share-a-project-with-a-group) to the project with the Reporter role. 1. Follow the steps in [Protecting Environments](#protecting-environments). ## Modifying and unprotecting environments @@ -200,7 +200,7 @@ To maximize the effectiveness of group-level protected environments, [group-level memberships](../../user/group/index.md) must be correctly configured: -- Operators should be given at least the Owner role +- Operators should be given the Owner role for the top-level group. They can maintain CI/CD configurations for the higher environments (such as production) in the group-level settings page, which includes group-level protected environments, diff --git a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md index 7208caaccae..125ae3650c9 100644 --- a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md +++ b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md @@ -90,7 +90,7 @@ The JWT is encoded by using RS256 and signed with a dedicated private key. The e You can use this JWT and your instance's JWKS endpoint (`https://gitlab.example.com/-/jwks`) to authenticate with a Vault server that is configured to allow the JWT Authentication method for authentication. -When configuring roles in Vault, you can use [bound_claims](https://developer.hashicorp.com/vault/docs/auth/jwt#bound-claims) to match against the JWT's claims and restrict which secrets each CI job has access to. +When configuring roles in Vault, you can use [bound claims](https://developer.hashicorp.com/vault/docs/auth/jwt#bound-claims) to match against the JWT claims and restrict which secrets each CI/CD job has access to. To communicate with Vault, you can use either its CLI client or perform API requests (using `curl` or another client). @@ -180,10 +180,35 @@ $ vault write auth/jwt/role/myproject-production - <<EOF EOF ``` -This example uses [bound_claims](https://developer.hashicorp.com/vault/api-docs/auth/jwt#bound_claims) to specify that only a JWT with matching values for the specified claims is allowed to authenticate. +This example uses [bound claims](https://developer.hashicorp.com/vault/api-docs/auth/jwt#bound_claims) to specify that only a JWT with matching values for the specified claims is allowed to authenticate. Combined with [protected branches](../../../user/project/protected_branches.md), you can restrict who is able to authenticate and read the secrets. +To use the same policy for a list of projects, use `namespace_id`: + +```json +"bound_claims": { + "namespace_id": ["12", "22", "37"] +} +``` + +Any of the claims [included in the JWT](#how-it-works) can be matched against a list of values +in the bound claims. For example: + +```json +"bound_claims": { + "user_login": ["alice", "bob", "mallory"] +} + +"bound_claims": { + "ref": ["main", "develop", "test"] +} + +"bound_claims": { + "project_id": ["12", "22", "37"] +} +``` + [`token_explicit_max_ttl`](https://developer.hashicorp.com/vault/api-docs/auth/jwt#token_explicit_max_ttl) specifies that the token issued by Vault, upon successful authentication, has a hard lifetime limit of 60 seconds. [`user_claim`](https://developer.hashicorp.com/vault/api-docs/auth/jwt#user_claim) specifies the name for the Identity alias created by Vault upon a successful login. @@ -225,7 +250,7 @@ $ vault write auth/jwt/config \ bound_issuer="gitlab.example.com" ``` -[bound_issuer](https://developer.hashicorp.com/vault/api-docs/auth/jwt#bound_issuer) specifies that only a JWT with the issuer (that is, the `iss` claim) set to `gitlab.example.com` can use this method to authenticate, and that the JWKS endpoint (`https://gitlab.example.com/-/jwks`) should be used to validate the token. +[`bound_issuer`](https://developer.hashicorp.com/vault/api-docs/auth/jwt#bound_issuer) specifies that only a JWT with the issuer (that is, the `iss` claim) set to `gitlab.example.com` can use this method to authenticate, and that the JWKS endpoint (`https://gitlab.example.com/-/jwks`) should be used to validate the token. For the full list of available configuration options, see Vault's [API documentation](https://developer.hashicorp.com/vault/api-docs/auth/jwt#configure). @@ -256,7 +281,7 @@ NOTE: If you're using a Vault instance provided by HashiCorp Cloud Platform, you need to export the `VAULT_NAMESPACE` variable. Its default value is `admin`. - + The following job is able to authenticate using the `myproject-production` role and read secrets under `/secret/myproject/production/`: @@ -279,14 +304,14 @@ read_secrets: - echo $PASSWORD ``` - + ### Limit token access to Vault secrets You can control `CI_JOB_JWT` access to Vault secrets by using Vault protections and GitLab features. For example, restrict the token by: -- Using Vault [bound_claims](https://developer.hashicorp.com/vault/docs/auth/jwt#bound-claims) +- Using Vault [bound claims](https://developer.hashicorp.com/vault/docs/auth/jwt#bound-claims) for specific groups using `group_claim`. - Hard coding values for Vault bound claims based on the `user_login` and `user_email` of specific users. diff --git a/doc/ci/examples/deployment/composer-npm-deploy.md b/doc/ci/examples/deployment/composer-npm-deploy.md index a603207aef7..533b6519d9a 100644 --- a/doc/ci/examples/deployment/composer-npm-deploy.md +++ b/doc/ci/examples/deployment/composer-npm-deploy.md @@ -120,7 +120,7 @@ Therefore, for a production environment we use additional steps to ensure that a ## Where to go next -Since this was a WordPress project, I gave real life code snippets. Some further ideas you can pursue: +Since this was a WordPress project, it includes real code snippets. Some further ideas you can pursue: - Having a slightly different script for the default branch allows you to deploy to a production server from that branch and to a stage server from any other branches. - Instead of pushing it live, you can push it to WordPress official repository. diff --git a/doc/ci/examples/index.md b/doc/ci/examples/index.md index 361061d0d75..c8ad653e41f 100644 --- a/doc/ci/examples/index.md +++ b/doc/ci/examples/index.md @@ -33,7 +33,7 @@ The following table lists examples with step-by-step tutorials that are containe | npm with semantic-release | [Publish npm packages to the GitLab Package Registry using semantic-release](semantic-release.md). | | PHP with Laravel, Envoy | [Test and deploy Laravel applications with GitLab CI/CD and Envoy](laravel_with_gitlab_and_envoy/index.md). | | PHP with npm, SCP | [Running Composer and npm scripts with deployment via SCP in GitLab CI/CD](deployment/composer-npm-deploy.md). | -| PHP with PHPunit, `atoum` | [Testing PHP projects](php.md). | +| PHP with PHPUnit, `atoum` | [Testing PHP projects](php.md). | | Secrets management with Vault | [Authenticating and Reading Secrets With HashiCorp Vault](authenticating-with-hashicorp-vault/index.md). | ### Contributed examples diff --git a/doc/ci/examples/php.md b/doc/ci/examples/php.md index 2d0c6382dd8..a83bcf69491 100644 --- a/doc/ci/examples/php.md +++ b/doc/ci/examples/php.md @@ -246,7 +246,7 @@ the [SSH keys](../ssh_keys/index.md) to be able to clone it. ## Use databases or other services Most of the time, you need a running database for your tests to be able to -run. If you're using the Docker executor, you can leverage Docker's ability to +run. If you're using the Docker executor, you can leverage Docker to link to other containers. With GitLab Runner, this can be achieved by defining a `service`. diff --git a/doc/ci/examples/semantic-release.md b/doc/ci/examples/semantic-release.md index 8f0321517ab..1fa526e787a 100644 --- a/doc/ci/examples/semantic-release.md +++ b/doc/ci/examples/semantic-release.md @@ -13,7 +13,7 @@ You can also view or fork the complete [example source](https://gitlab.com/gitla ## Initialize the module 1. Open a terminal and navigate to the project's repository. -1. Run `npm init`. Name the module according to [the Package Registry's naming conventions](../../user/packages/npm_registry/index.md#package-naming-convention). For example, if the project's path is `gitlab-examples/semantic-release-npm`, name the module `@gitlab-examples/semantic-release-npm`. +1. Run `npm init`. Name the module according to [the Package Registry's naming conventions](../../user/packages/npm_registry/index.md#naming-convention). For example, if the project's path is `gitlab-examples/semantic-release-npm`, name the module `@gitlab-examples/semantic-release-npm`. 1. Install the following npm packages: @@ -89,6 +89,7 @@ The default `before_script` generates a temporary `.npmrc` that is used to authe As part of publishing a package, semantic-release increases the version number in `package.json`. For semantic-release to commit this change and push it back to GitLab, the pipeline requires a custom CI/CD variable named `GITLAB_TOKEN`. To create this variable: <!-- markdownlint-disable MD044 --> + 1. On the top bar, on the top right, select your avatar. 1. On the left sidebar, select **Access Tokens**. 1. In the **Token name** box, enter a token name. diff --git a/doc/ci/git_submodules.md b/doc/ci/git_submodules.md index 68cbff21fae..0f206b3fceb 100644 --- a/doc/ci/git_submodules.md +++ b/doc/ci/git_submodules.md @@ -68,7 +68,7 @@ To make submodules work correctly in CI/CD jobs: ``` 1. You can filter or exclude specific submodules to control which submodules will be synced using - [`GIT_SUBMODULE_PATHS`](runners/configure_runners.md#git-submodule-paths). + [`GIT_SUBMODULE_PATHS`](runners/configure_runners.md#sync-or-exclude-specific-submodules-from-ci-jobs). ```yaml variables: @@ -83,14 +83,6 @@ To make submodules work correctly in CI/CD jobs: GIT_SUBMODULE_STRATEGY: recursive GIT_SUBMODULE_UPDATE_FLAGS: --jobs 4 ``` - -1. You can set the [GIT_SUBMODULE_PATHS](runners/configure_runners.md#sync-or-exclude-specific-submodules-from-ci-jobs) to explicitly ignore submodules during cloning: - - ```yaml - variables: - GIT_SUBMODULE_STRATEGY: recursive - GIT_SUBMODULE_PATHS: ':(exclude)submodule' - ``` If you use the [`CI_JOB_TOKEN`](jobs/ci_job_token.md) to clone a submodule in a pipeline job, the user executing the job must be assigned to a role that has diff --git a/doc/ci/jobs/ci_job_token.md b/doc/ci/jobs/ci_job_token.md index 1e7b389c84a..d95451a67dc 100644 --- a/doc/ci/jobs/ci_job_token.md +++ b/doc/ci/jobs/ci_job_token.md @@ -54,7 +54,7 @@ To make sure that this token doesn't leak, GitLab: To make sure that this token doesn't leak, you should also configure your [runners](../runners/index.md) to be secure. Avoid: -- Using Docker's `privileged` mode if the machines are re-used. +- Using Docker `privileged` mode if the machines are re-used. - Using the [`shell` executor](https://docs.gitlab.com/runner/executors/shell.html) when jobs run on the same machine. diff --git a/doc/ci/jobs/index.md b/doc/ci/jobs/index.md index 4eb8952dd73..15ec92a896e 100644 --- a/doc/ci/jobs/index.md +++ b/doc/ci/jobs/index.md @@ -110,6 +110,7 @@ You can't use these keywords as job names: - `true` - `false` - `nil` +- `pages:deploy` configured for a `deploy` stage Job names must be 255 characters or fewer. diff --git a/doc/ci/large_repositories/index.md b/doc/ci/large_repositories/index.md index c7ecc25dc44..5a553228ba1 100644 --- a/doc/ci/large_repositories/index.md +++ b/doc/ci/large_repositories/index.md @@ -196,18 +196,14 @@ Our pipeline is most performant if we use the following `.gitlab-ci.yml`: ```yaml variables: - GIT_DEPTH: 10 GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_CONCURRENT_ID/$CI_PROJECT_NAME build: script: ls -al ``` -The above configures a: - -- Shallow clone of 10, to speed up subsequent `git fetch` commands. -- Custom clone path to make it possible to re-use worktrees between parent project and all forks - because we use the same clone path for all forks. +This YAML setting configures a custom clone path. This path makes it possible to re-use worktrees +between the parent project and forks because we use the same clone path for all forks. Why use `$CI_CONCURRENT_ID`? The main reason is to ensure that worktrees used are not conflicting between projects. The `$CI_CONCURRENT_ID` represents a unique identifier within the given executor. @@ -264,4 +260,4 @@ For very active repositories with a large number of references and files, you ca seeding the repository data also helps avoid `429 Too many requests` errors from Cloudflare. This error can occur if you have many runners behind a single, - NAT'ed IP address that pulls from GitLab.com. + IP address using NAT, that pulls from GitLab.com. diff --git a/doc/ci/migration/jenkins.md b/doc/ci/migration/jenkins.md index 4ba59e14811..235dd0e80ca 100644 --- a/doc/ci/migration/jenkins.md +++ b/doc/ci/migration/jenkins.md @@ -8,9 +8,9 @@ type: index, howto # Migrating from Jenkins **(FREE)** -A lot of GitLab users have successfully migrated to GitLab CI/CD from Jenkins. To make this -easier if you're just getting started, we've collected several resources here that you might find useful -before diving in. Think of this page as a "GitLab CI/CD for Jenkins Users" guide. +A lot of GitLab users have successfully migrated to GitLab CI/CD from Jenkins. +We've collected several resources here that you might find informative if you're just getting started. +Think of this page as a "GitLab CI/CD for Jenkins Users" guide. The following list of recommended steps was created after observing organizations that were able to quickly complete this migration: @@ -29,9 +29,10 @@ that were able to quickly complete this migration: 1. Check the [pipeline efficiency documentation](../pipelines/pipeline_efficiency.md) to learn how to make your GitLab CI/CD pipelines faster and more efficient. -For an example of how to convert a Jenkins pipeline into a GitLab CI/CD pipeline, -or how to use Auto DevOps to test your code automatically, watch the -[Migrating from Jenkins to GitLab](https://www.youtube.com/watch?v=RlEVGOpYF5Y) video. +Watch the [Migrating from Jenkins to GitLab](https://www.youtube.com/watch?v=RlEVGOpYF5Y) video for examples of how to: + +- Convert a Jenkins pipeline into a GitLab CI/CD pipeline. +- Use Auto DevOps to test your code automatically. Otherwise, read on for important information that helps you get the ball rolling. Welcome to GitLab! @@ -42,8 +43,8 @@ can be a great resource. ## Manage organizational transition An important part of transitioning from Jenkins to GitLab is the cultural and organizational -changes that come with the move, and successfully managing them. There are a few -things we have found that help this: +changes that come with the move, and successfully managing them. A few +things we have found that help this are: - Setting and communicating a clear vision of what your migration goals are helps your users understand why the effort is worth it. The value is clear when @@ -67,11 +68,11 @@ of transition, by letting you delay the migration of less urgent pipelines for a If you are interested in helping GitLab test the wrapper, join our [public testing issue](https://gitlab.com/gitlab-org/gitlab/-/issues/215675) for instructions and to provide your feedback. NOTE: -If you have a paid GitLab subscription, note that the JenkinsFile Wrapper is not packaged as part of GitLab, and falls outside of the scope of support. For more information, see the [Statement of Support](https://about.gitlab.com/support/statement-of-support/). +If you have a paid GitLab subscription, the JenkinsFile Wrapper is not packaged with GitLab and falls outside of the scope of support. For more information, see the [Statement of Support](https://about.gitlab.com/support/statement-of-support/). ## Important product differences -There are some high level differences between the products worth mentioning: +Some high level differences between the products worth mentioning are: - With GitLab you don't need a root `pipeline` keyword to wrap everything. - The way pipelines are triggered and [trigger other pipelines](../yaml/index.md#trigger) @@ -93,14 +94,13 @@ There are some high level differences between the products worth mentioning: contain scripts or other reusable code. - You can also use the [`extends` keyword](../yaml/index.md#extends) to reuse configuration in a single pipeline configuration. -- All jobs in a single stage always run in parallel, and all stages run in sequence. We are planning - to allow certain jobs to break this sequencing as needed with our [directed acyclic graph](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/47063) +- All jobs in a single stage always run in parallel, and all stages run in sequence. + Certain jobs might break this sequencing as needed with our [directed acyclic graph](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/47063) feature. - The [`parallel`](../yaml/index.md#parallel) keyword can automatically parallelize tasks, like tests that support parallelization. - Normally all jobs in a single stage run in parallel, and all stages run in sequence. - There are different [pipeline architectures](../pipelines/pipeline_architectures.md) - that allow you to change this behavior. + Different [pipeline architectures](../pipelines/pipeline_architectures.md) allow you to change this behavior. - The new [`rules` syntax](../yaml/index.md#rules) is the recommended method of controlling when different jobs run. It is more powerful than the `only/except` syntax. - One important difference is that jobs run independently of each other and have a @@ -114,10 +114,10 @@ There are some high level differences between the products worth mentioning: - Manual approvals or gates can be set up as [`when:manual` jobs](../jobs/job_control.md#create-a-job-that-must-be-run-manually). These can also leverage [`protected environments`](../jobs/job_control.md#run-a-job-after-a-delay) to control who is able to approve them. -- GitLab comes with a [container registry](../../user/packages/container_registry/index.md), and we recommend using +- GitLab comes with a [container registry](../../user/packages/container_registry/index.md), so you can use container images to set up your build environment. For example, set up one pipeline that builds your build environment itself and publish that to the container registry. Then, have your pipelines use this instead of each building their - own environment, which is slower and may be less consistent. We have extensive docs on [how to use the Container Registry](../../user/packages/container_registry/index.md). + own environment, which is slower and may be less consistent. We have extensive documentation on [how to use the Container Registry](../../user/packages/container_registry/index.md). - A central utilities repository can be a great place to put assorted scheduled jobs or other manual jobs that function like utilities. Jenkins installations tend to have a few of these. @@ -129,7 +129,7 @@ Jenkins agent, uninstall it and then [install and register the runner](../runner Runners do not require much overhead, so you can size them similarly to the Jenkins agents you were using. -There are some important differences in the way runners work in comparison to agents: +Some important differences in the way runners work in comparison to agents are: - Runners can be set up as [shared across an instance, be added at the group level, or set up at the project level](../runners/runners_scope.md). They self-select jobs from the scopes you've defined automatically. @@ -137,8 +137,8 @@ There are some important differences in the way runners work in comparison to ag associate runners with specific jobs. For example, you can use a tag for jobs that require dedicated, more powerful, or specific hardware. - GitLab has [autoscaling for runners](https://docs.gitlab.com/runner/configuration/autoscale.html). - Use autoscaling to provision runners only when needed, and scale down when not needed. - This is similar to ephemeral agents in Jenkins. + Use autoscaling to provision runners only when needed and scale down when not needed, + similar to ephemeral agents in Jenkins. If you are using `gitlab.com`, you can take advantage of our [shared runner fleet](../runners/index.md) to run jobs without provisioning your own runners. We are investigating making them @@ -148,14 +148,14 @@ as well. ## Groovy vs. YAML Jenkins Pipelines are based on [Groovy](https://groovy-lang.org/), so the pipeline specification is written as code. -GitLab works a bit differently, we use the more highly structured [YAML](https://yaml.org/) format, which -places scripting elements inside of `script` blocks separate from the pipeline specification itself. +GitLab works a bit differently, using the more highly structured [YAML](https://yaml.org/) format. +The scripting elements are in `script` blocks separate from the pipeline specification itself. -This is a strength of GitLab, in that it helps keep the learning curve much simpler to get up and running -and avoids some of the problem of unconstrained complexity which can make your Jenkinsfile hard to understand +Using YAML is a strength of GitLab, in that it helps keep the learning curve much simpler to get up and running. +It also avoids some of the problem of unconstrained complexity which can make your Jenkinsfile hard to understand and manage. -That said, we do of course still value DRY (don't repeat yourself) principles and want to ensure that +We do of course still value DRY (don't repeat yourself) principles. We want to ensure that behaviors of your jobs can be codified once and applied as needed. You can use the `extends` syntax to [reuse configuration in your jobs](../yaml/index.md#extends), and `include` can be used to [reuse pipeline configurations](../yaml/index.md#include) in pipelines @@ -197,13 +197,13 @@ can leverage. You can see the complete list of packaging features in the ## Integrated features -Where you may have used plugins to get things like code quality, unit tests, and security scanning working in Jenkins, +You may have used plugins to get things like code quality, unit tests, and security scanning working in Jenkins. GitLab takes advantage of our connected ecosystem to automatically pull these kinds of results into your merge requests, pipeline details pages, and other locations. You may find that you actually don't need to configure anything to have these appear. -If they aren't working as expected, or if you'd like to see what's available, our [CI/CD feature index](../index.md#features) has the full list -of bundled features and links to the documentation for each. +Our [CI/CD feature index](../index.md#features) has the full list of bundled features and links to the documentation for each. +Refer to this index if these features aren't working as expected, or if you'd like to see what's available. ### Templates @@ -213,7 +213,7 @@ as well as encourage inner sourcing. In self-managed GitLab instances, you can build an [Instance Template Repository](../../user/admin_area/settings/instance_template_repository.md). Development teams across the whole organization can select templates from a dropdown list. A group maintainer or a group owner is able to set a group to use as the source for the -[custom project templates](../../user/admin_area/custom_project_templates.md), which can +[custom project templates](../../user/admin_area/custom_project_templates.md). This can be used by all projects in the group. An instance administrator can set a group as the source for [instance project templates](../../user/group/custom_project_templates.md), which can be used by projects in that instance. @@ -221,7 +221,7 @@ which can be used by projects in that instance. ## Convert a declarative Jenkinsfile A declarative Jenkinsfile contains "Sections" and "Directives" which are used to control the behavior of your -pipelines. There are equivalents for all of these in GitLab, which we've documented below. +pipelines. Equivalents for all of these exist in GitLab, which we've documented below. This section is based on the [Jenkinsfile syntax documentation](https://www.jenkins.io/doc/book/pipeline/syntax/) and is meant to be a mapping of concepts there to concepts in GitLab. @@ -231,8 +231,8 @@ and is meant to be a mapping of concepts there to concepts in GitLab. #### `agent` The agent section is used to define how a pipeline executes. For GitLab, we use [runners](../runners/index.md) -to provide this capability. You can configure your own runners in Kubernetes or on any host, or take advantage -of our shared runner fleet (note that the shared runner fleet is only available for GitLab.com users). +to provide this capability. You can configure your own runners in Kubernetes or on any host. You can also take advantage +of our shared runner fleet (the shared runner fleet is only available for GitLab.com users). We also support using [tags](../runners/configure_runners.md#use-tags-to-control-which-jobs-a-runner-can-run) to direct different jobs to different runners (execution agents). @@ -272,11 +272,11 @@ default: #### `stages` GitLab CI/CD also lets you define stages, but is a little bit more free-form to configure. The GitLab [`stages` keyword](../yaml/index.md#stages) -is a top level setting that enumerates the list of stages, but you are not required to nest individual jobs underneath +is a top level setting that enumerates the list of stages. You are not required to nest individual jobs underneath the `stages` section. Any job defined in the `.gitlab-ci.yml` can be made a part of any stage through use of the [`stage` keyword](../yaml/index.md#stage). -Note that, unless otherwise specified, every pipeline is instantiated with a `build`, `test`, and `deploy` stage +Unless otherwise specified, every pipeline is instantiated with a `build`, `test`, and `deploy` stage which are run in that order. Jobs that have no `stage` defined are placed by default in the `test` stage. Of course, each job that refers to a stage must refer to a stage that exists in the pipeline configuration. @@ -292,8 +292,8 @@ my_job: #### `steps` -The `steps` section is equivalent to the [`script` section](../yaml/index.md#script) of an individual job. This is -a simple YAML array with each line representing an individual command to be run: +The `steps` section is equivalent to the [`script` section](../yaml/index.md#script) of an individual job. The `steps` section is a YAML array +with each line representing an individual command to be run: ```yaml my_job: @@ -308,7 +308,7 @@ my_job: In GitLab, we use the [`variables` keyword](../yaml/index.md#variables) to define different variables at runtime. These can also be set up through the GitLab UI, under CI/CD settings. See also our [general documentation on variables](../variables/index.md), -including the section on [protected variables](../variables/index.md#protected-cicd-variables) which can be used +including the section on [protected variables](../variables/index.md#protected-cicd-variables). This can be used to limit access to certain variables to certain environments or runners: ```yaml @@ -330,18 +330,18 @@ can provide any variables they like. #### `triggers` / `cron` -Because GitLab is integrated tightly with Git, SCM polling options for triggers are not needed. We support an easy to use +Because GitLab is integrated tightly with Git, SCM polling options for triggers are not needed. We support a [syntax for scheduling pipelines](../pipelines/schedules.md). #### `tools` GitLab does not support a separate `tools` directive. Our best-practice recommendation is to use pre-built -container images, which can be cached, and can be built to already contain the tools you need for your pipelines. Pipelines can +container images. These images can be cached and can be built to already contain the tools you need for your pipelines. Pipelines can be set up to automatically build these images as needed and deploy them to the [container registry](../../user/packages/container_registry/index.md). -If you're not using container images with Docker/Kubernetes, for example on Mac or FreeBSD, then the `shell` executor does require you to -set up your environment either in advance or as part of the jobs. You could create a `before_script` -action that handles this for you. +If you don't use container images with Docker or Kubernetes, but use the `shell` executor on your own system, +you must set up your environment. You can set up the environment in advance, or as part of the jobs +with a `before_script` action that handles this for you. #### `input` @@ -351,7 +351,7 @@ variable entry. #### `when` GitLab does support a [`when` keyword](../yaml/index.md#when) which is used to indicate when a job should be -run in case of (or despite) failure, but most of the logic for controlling pipelines can be found in +run in case of (or despite) failure. Most of the logic for controlling pipelines can be found in our very powerful [`rules` system](../yaml/index.md#rules): ```yaml diff --git a/doc/ci/pipelines/cicd_minutes.md b/doc/ci/pipelines/cicd_minutes.md index 772a06980af..d9ad224ab95 100644 --- a/doc/ci/pipelines/cicd_minutes.md +++ b/doc/ci/pipelines/cicd_minutes.md @@ -126,7 +126,7 @@ These additional CI/CD minutes: - Are used only after the monthly quota included in your subscription runs out. - Are carried over to the next month, if any remain at the end of the month. -- Are valid for 12 months from date of purchase or until all minutes are consumed, whichever comes first. Expiry of minutes is not currently enforced. +- Are valid for 12 months from date of purchase or until all minutes are consumed, whichever comes first. Expiry of minutes is not enforced. For example, with a GitLab SaaS Premium license: @@ -137,7 +137,7 @@ For example, with a GitLab SaaS Premium license: If you use `13,000` minutes during the month, the next month your additional minutes become `2,000`. If you use `9,000` minutes during the month, your additional minutes remain the same. -If you bought additional CI/CD minutes while on a trial subscription those minutes will be available after the trial ends or you upgrade to a paid plan. +If you bought additional CI/CD minutes while on a trial subscription, those minutes are available after the trial ends or you upgrade to a paid plan. You can find pricing for additional CI/CD minutes on the [GitLab Pricing page](https://about.gitlab.com/pricing/). diff --git a/doc/ci/pipelines/downstream_pipelines.md b/doc/ci/pipelines/downstream_pipelines.md index c8fcd06da07..6c17c23552d 100644 --- a/doc/ci/pipelines/downstream_pipelines.md +++ b/doc/ci/pipelines/downstream_pipelines.md @@ -183,7 +183,7 @@ You can trigger a child pipeline from a YAML file generated in a job, instead of static file saved in your project. This technique can be very powerful for generating pipelines targeting content that changed or to build a matrix of targets and architectures. -The artifact containing the generated YAML file must not be [larger than 5MB](https://gitlab.com/gitlab-org/gitlab/-/issues/249140). +The artifact containing the generated YAML file must not be [larger than 5 MB](https://gitlab.com/gitlab-org/gitlab/-/issues/249140). <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For an overview, see [Create child pipelines using dynamically generated configurations](https://youtu.be/nMdfus2JWHM). diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index f1ca8afa62c..ab98bab022e 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -188,6 +188,30 @@ In this example: - `DEPLOY_ENVIRONMENT` is listed in the **Run pipeline** page, but with no value set. The user is expected to define the value each time the pipeline is run manually. +##### Configure a list of selectable values for a prefilled variable + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363660) in GitLab 15.5 [with a flag](../../administration/feature_flags.md) named `run_pipeline_graphql`. Disabled by default. +> - The `options` keyword was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105502) in GitLab 15.7. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106038) in GitLab 15.7. Feature flag `run_pipeline_graphql` removed. + +You can define an array of CI/CD variable values the user can select from when running a pipeline manually. +These values are in a dropdown list in the **Run pipeline** page. Add the list of +value options to `options` and set the default value with `value`. The string in `value` +must also be included in the `options` list. + +For example: + +```yaml +variables: + DEPLOY_ENVIRONMENT: + value: "staging" + options: + - "production" + - "staging" + - "canary" + description: "The deployment target. Set to 'staging' by default." +``` + ### Run a pipeline by using a URL query string > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24146) in GitLab 12.5. diff --git a/doc/ci/pipelines/multi_project_pipelines.md b/doc/ci/pipelines/multi_project_pipelines.md deleted file mode 100644 index 25ac9e13185..00000000000 --- a/doc/ci/pipelines/multi_project_pipelines.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'downstream_pipelines.md' -remove_date: '2022-11-30' ---- - -This document was moved to [another location](downstream_pipelines.md). - -<!-- This redirect file can be deleted after <2022-11-30>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/ci/pipelines/parent_child_pipelines.md b/doc/ci/pipelines/parent_child_pipelines.md deleted file mode 100644 index be8ed8ba6d7..00000000000 --- a/doc/ci/pipelines/parent_child_pipelines.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'downstream_pipelines.md' -remove_date: '2022-12-05' ---- - -This document was moved to [another location](downstream_pipelines.md). - -<!-- This redirect file can be deleted after <2022-12-05>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/ci/pipelines/pipeline_efficiency.md b/doc/ci/pipelines/pipeline_efficiency.md index 276a03cb480..0952fdf1f8f 100644 --- a/doc/ci/pipelines/pipeline_efficiency.md +++ b/doc/ci/pipelines/pipeline_efficiency.md @@ -219,7 +219,7 @@ that download and run faster. Try to use custom Docker images with the software pre-installed. It's usually much faster to download a larger pre-configured image than to use a common image and install -software on it each time. Docker's [Best practices for writing Dockerfiles](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/) +software on it each time. The Docker [Best practices for writing Dockerfiles article](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/) has more information about building efficient Docker images. Methods to reduce Docker image size: diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index a7f3cfb59d3..423ee31dec4 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -32,7 +32,7 @@ To change the visibility of your pipelines and related features: When it is selected, pipelines and related features are visible: - For [**Public**](../../user/public_access.md) projects, to everyone. - - For **Internal** projects, to all logged-in users except [external users](../../user/permissions.md#external-users). + - For **Internal** projects, to all logged-in users except [external users](../../user/admin_area/external_users.md). - For **Private** projects, to all project members (Guest or higher). When it is cleared: @@ -41,7 +41,7 @@ To change the visibility of your pipelines and related features: and the **CI/CD** menu items are visible only to project members (Reporter or higher). Other users, including guest users, can only view the status of pipelines and jobs, and only when viewing merge requests or commits. - - For **Internal** projects, pipelines are visible to all logged in users except [external users](../../user/permissions.md#external-users). + - For **Internal** projects, pipelines are visible to all logged in users except [external users](../../user/admin_area/external_users.md). Related features are visible only to project members (Reporter or higher). - For **Private** projects, pipelines and related features are visible to project members (Reporter or higher) only. @@ -284,9 +284,8 @@ when merging a merge request would cause the project's test coverage to decline. Follow these steps to enable the `Coverage-Check` MR approval rule: 1. Set up a [`coverage`](../yaml/index.md#coverage) regular expression for all jobs you want to include in the overall coverage value. -1. Go to your project and select **Settings > General**. -1. Expand **Merge request approvals**. -1. Select **Enable** next to the `Coverage-Check` approval rule. +1. Go to your project and select **Settings > Merge requests**. +1. Under **Merge request approvals**, select **Enable** next to the `Coverage-Check` approval rule. 1. Select the **Target branch**. 1. Set the number of **Approvals required** to greater than zero. 1. Select the users or groups to provide approval. diff --git a/doc/ci/runners/configure_runners.md b/doc/ci/runners/configure_runners.md index c675f7204ec..b61e11dd8bc 100644 --- a/doc/ci/runners/configure_runners.md +++ b/doc/ci/runners/configure_runners.md @@ -62,7 +62,7 @@ How this feature works: With some [runner executors](https://docs.gitlab.com/runner/executors/), if you can run a job on the runner, you can get full access to the file system, and thus any code it runs as well as the token of the runner. With shared runners, this means that anyone -that runs jobs on the runner, can access anyone else's code that runs on the +that runs jobs on the runner, can access another user's code that runs on the runner. In addition, because you can get access to the runner token, it is possible @@ -292,12 +292,12 @@ variables: A runner can have one of the following statuses. -| Status | Description | -|--------|-------------| -| **online** | The runner has contacted GitLab within the last 2 hours and is available to run jobs. | -| **offline** | The runner has not contacted GitLab in more than 2 hours and is not available to run jobs. Check the runner to see if you can bring it online. | -| **stale** | The runner has not contacted GitLab in more than 3 months. If the runner was created more than 3 months ago, but it never contacted the instance, it is also considered **stale**. | -| **never_contacted** | The runner has never contacted GitLab. To make the runner contact GitLab, run `gitlab-runner run`. | +| Status | Description | +|---------|-------------| +| `online` | The runner has contacted GitLab within the last 2 hours and is available to run jobs. | +| `offline` | The runner has not contacted GitLab in more than 2 hours and is not available to run jobs. Check the runner to see if you can bring it online. | +| `stale` | The runner has not contacted GitLab in more than 3 months. If the runner was created more than 3 months ago, but it never contacted the instance, it is also considered **stale**. | +| `never_contacted` | The runner has never contacted GitLab. To make the runner contact GitLab, run `gitlab-runner run`. | ## Configure runner behavior with variables @@ -306,11 +306,9 @@ globally or for individual jobs: - [`GIT_STRATEGY`](#git-strategy) - [`GIT_SUBMODULE_STRATEGY`](#git-submodule-strategy) -- [`GIT_SUBMODULE_PATHS`](#sync-or-exclude-specific-submodules-from-ci-jobs) - [`GIT_CHECKOUT`](#git-checkout) - [`GIT_CLEAN_FLAGS`](#git-clean-flags) - [`GIT_FETCH_EXTRA_FLAGS`](#git-fetch-extra-flags) -- [`GIT_SUBMODULE_PATHS`](#git-submodule-paths) - [`GIT_SUBMODULE_UPDATE_FLAGS`](#git-submodule-update-flags) - [`GIT_DEPTH`](#shallow-cloning) (shallow cloning) - [`GIT_SUBMODULE_DEPTH`](#git-submodule-depth) @@ -498,16 +496,13 @@ git fetch origin $REFSPECS --depth 50 --prune Where `$REFSPECS` is a value provided to the runner internally by GitLab. -### Git submodule paths +### Sync or exclude specific submodules from CI jobs > [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/2249) in GitLab Runner 14.0. Use the `GIT_SUBMODULE_PATHS` variable to control which submodules have to be synced or updated. You can set it globally or per-job in the [`variables`](../yaml/index.md#variables) section. -This variable can be very useful for projects which have a large number of submodules which not all of them -need to be synced or updated in all CI jobs. - The path syntax is the same as [`git submodule`](https://git-scm.com/docs/git-submodule#Documentation/git-submodule.txt-ltpathgt82308203): - To sync and update specific paths: @@ -524,6 +519,12 @@ The path syntax is the same as [`git submodule`](https://git-scm.com/docs/git-su GIT_SUBMODULE_PATHS: :(exclude)submoduleA :(exclude)submoduleB ``` +WARNING: +Git ignores nested paths. To ignore a nested submodule, exclude +the parent submodule and then manually clone it in the job's scripts. For example, + `git clone <repo> --recurse-submodules=':(exclude)nested-submodule'`. Make sure +to wrap the string in single quotes so the YAML can be parsed successfully. + ### Git submodule update flags > [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3192) in GitLab Runner 14.8. @@ -565,34 +566,6 @@ You should be aware of the implications for the security, stability, and reprodu your builds when using the `--remote` flag. In most cases, it is better to explicitly track submodule commits as designed, and update them using an auto-remediation/dependency bot. -### Sync or exclude specific submodules from CI jobs - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26495) in GitLab Runner 14.0. - -Some projects have a large number of submodules, and not all of them need to be -synced or updated in all CI jobs. Use the `GIT_SUBMODULE_PATHS` variable to control this behavior. -The path syntax is the same as [`git submodule`](https://git-scm.com/docs/git-submodule#Documentation/git-submodule.txt-ltpathgt82308203): - -- To sync and update specific paths: - - ```yaml - variables: - GIT_SUBMODULE_PATHS: 'submoduleA' - ``` - -- To exclude specific paths: - - ```yaml - variables: - GIT_SUBMODULE_PATHS: ':(exclude)submoduleA' - ``` - -WARNING: -Git ignores nested and multiple submodule paths. To ignore a nested submodule, exclude -the parent submodule and then manually clone it in the job's scripts. For example, - `git clone <repo> --recurse-submodules=':(exclude)nested-submodule'`. Make sure -to wrap the string in single quotes so the YAML can be parsed successfully. - ### Shallow cloning > Introduced in GitLab 8.9 as an experimental feature. @@ -930,7 +903,7 @@ The default is the number of CPUs available, but given the memory ramifications, setting. `FASTZIP_EXTRACTOR_CONCURRENCY` controls how many files are decompressed at once. Files from a zip archive can natively -be read from concurrency, so no additional memory is allocated in additional to what the decompressor requires. This +be read from concurrency, so no additional memory is allocated in addition to what the decompressor requires. This defaults to the number of CPUs available. ## Clean up stale runners **(ULTIMATE)** diff --git a/doc/ci/runners/saas/linux_saas_runner.md b/doc/ci/runners/saas/linux_saas_runner.md index df7d5570953..61421f86ff5 100644 --- a/doc/ci/runners/saas/linux_saas_runner.md +++ b/doc/ci/runners/saas/linux_saas_runner.md @@ -17,7 +17,7 @@ For Free, Premium, and Ultimate plan customers, jobs on these instances consume | | Small | Medium | Large | |-------------------|---------------------------|---------------------------|--------------------------| -| Specs | 1 vCPU, 3.75GB RAM | 2 vCPUs, 8GB RAM | 4 vCPUs, 16GB RAM | +| Specs | 1 vCPU, 3.75 GB RAM | 2 vCPUs, 8 GB RAM | 4 vCPUs, 16 GB RAM | | GitLab CI/CD tags | `saas-linux-small-amd64` | `saas-linux-medium-amd64` | `saas-linux-large-amd64` | | Subscription | Free, Premium, Ultimate | Free, Premium, Ultimate | Premium, Ultimate | @@ -73,12 +73,12 @@ Below are the settings for SaaS runners on Linux. | Setting | GitLab.com | Default | |-------------------------------------------------------------------------|------------------|---------| | Executor | `docker+machine` | - | -| Default Docker image | `ruby:2.5` | - | +| Default Docker image | `ruby:3.1` | - | | `privileged` (run [Docker in Docker](https://hub.docker.com/_/docker/)) | `true` | `false` | - **Cache**: These runners share a [distributed cache](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) - that's stored in a Google Cloud Storage (GCS) bucket. Cache contents not updated within + that's stored in a Google Cloud Storage (GCS) bucket. Cache contents not updated in the last 14 days are automatically removed, based on the [object lifecycle management policy](https://cloud.google.com/storage/docs/lifecycle). @@ -88,8 +88,8 @@ Below are the settings for SaaS runners on Linux. and [#4070](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/4070). NOTE: -The final disk space your jobs can use will be less than 25GB. Some disk space -allocated to the instance will be occupied by the operating system, the Docker image, +The final disk space your jobs can use is less than 25 GB. Some disk space +allocated to the instance is occupied by the operating system, the Docker image, and a copy of your cloned repository. ## Pre-clone script @@ -198,7 +198,7 @@ sentry_dsn = "X" ] limit = X [runners.docker] - image = "ruby:2.5" + image = "ruby:3.1" privileged = true volumes = [ "/certs/client", diff --git a/doc/ci/runners/saas/macos_saas_runner.md b/doc/ci/runners/saas/macos_saas_runner.md index cd40aac25bc..270f85e65e2 100644 --- a/doc/ci/runners/saas/macos_saas_runner.md +++ b/doc/ci/runners/saas/macos_saas_runner.md @@ -18,9 +18,11 @@ Jobs handled by macOS shared runners on GitLab.com **time out after 3 hours**, r ## Access request process -While in beta, to run CI jobs on the macOS runners, GitLab SaaS customer namespaces must be explicitly added to the macOS `allow-list`. +While in beta, to run CI jobs on the macOS runners, you must specify the GitLab SaaS customer personal or group [namespaces](../../../user/namespace/index.md) in the macOS `allow-list`. These are the namespaces that use the macOS runners. -After you have been added, you can use the macOS runners for any projects in your namespace. +When you specify a personal or group namespace, the top level group is not added unless you specify it. + +After you add your namespace, you can use the macOS runners for any projects under the namespace you included. To request access, open an [access request](https://gitlab.com/gitlab-com/runner-saas-macos-limited-availability/-/issues/new). The expected turnaround for activation is two business days. diff --git a/doc/ci/secrets/index.md b/doc/ci/secrets/index.md index a5082af89bc..bfc53210348 100644 --- a/doc/ci/secrets/index.md +++ b/doc/ci/secrets/index.md @@ -143,7 +143,7 @@ different policies together. If authentication is successful, these policies are attached to the resulting Vault token. [Bound claims](https://developer.hashicorp.com/vault/docs/auth/jwt#bound-claims) are predefined -values that are matched to the JWT's claims. With bounded claims, you can restrict access +values that are matched to the JWT claims. With bounded claims, you can restrict access to specific GitLab users, specific projects, or even jobs running for specific Git references. You can have as many bounded claims you need, but they must *all* match for authentication to be successful. diff --git a/doc/ci/secure_files/index.md b/doc/ci/secure_files/index.md index bba8a3e4c27..e63b671ad2b 100644 --- a/doc/ci/secure_files/index.md +++ b/doc/ci/secure_files/index.md @@ -29,7 +29,7 @@ plain text and binary file types. You can manage secure files in the project settings, or with the [secure files API](../../api/secure_files.md). Secure files can be [downloaded and used by CI/CD jobs](#use-secure-files-in-cicd-jobs) -by using the [load-secure-files](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/load-secure-files) +by using the [download-secure-files](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files) tool. NOTE: @@ -49,11 +49,11 @@ To add a secure file to a project: ## Use secure files in CI/CD jobs -To use your secure files in a CI/CD job, you must use the [`load-secure-files`](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/load-secure-files) +To use your secure files in a CI/CD job, you must use the [`download-secure-files`](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files) tool to download the files in the job. After they are downloaded, you can use them with your other script commands. -Add a command in the `script` section of your job to download the `load-secure-files` tool +Add a command in the `script` section of your job to download the `download-secure-files` tool and execute it. The files download into a `.secure_files` directory in the root of the project. To change the download location for the secure files, set the path in the `SECURE_FILES_DOWNLOAD_PATH` [CI/CD variable](../variables/index.md). @@ -65,5 +65,5 @@ test: variables: SECURE_FILES_DOWNLOAD_PATH: './where/files/should/go/' script: - - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/load-secure-files/-/raw/main/installer" | bash + - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash ``` diff --git a/doc/ci/services/index.md b/doc/ci/services/index.md index 830b9406c6e..beebaa6d03f 100644 --- a/doc/ci/services/index.md +++ b/doc/ci/services/index.md @@ -260,7 +260,7 @@ test: | Setting | Required | GitLab version | Description | |------------|----------|----------------| ----------- | | `name` | yes, when used with any other option | 9.4 | Full name of the image to use. If the full image name includes a registry hostname, use the `alias` option to define a shorter service access name. For more information, see [Accessing the services](#accessing-the-services). | -| `entrypoint` | no | 9.4 |Command or script to execute as the container's entrypoint. It's translated to Docker's `--entrypoint` option while creating the container. The syntax is similar to [`Dockerfile`'s `ENTRYPOINT`](https://docs.docker.com/engine/reference/builder/#entrypoint) directive, where each shell token is a separate string in the array. | +| `entrypoint` | no | 9.4 |Command or script to execute as the container's entrypoint. It's translated to the Docker `--entrypoint` option while creating the container. The syntax is similar to [`Dockerfile`'s `ENTRYPOINT`](https://docs.docker.com/engine/reference/builder/#entrypoint) directive, where each shell token is a separate string in the array. | | `command` | no | 9.4 |Command or script that should be used as the container's command. It's translated to arguments passed to Docker after the image's name. The syntax is similar to [`Dockerfile`'s `CMD`](https://docs.docker.com/engine/reference/builder/#cmd) directive, where each shell token is a separate string in the array. | | `alias` (1) | no | 9.4 | Additional alias that can be used to access the service from the job's container. Read [Accessing the services](#accessing-the-services) for more information. | | `variables` (2) | no | 14.5 | Additional environment variables that are passed exclusively to the service. The syntax is the same as [Job Variables](../variables/index.md). | @@ -394,6 +394,8 @@ time. ## Capturing service container logs +> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3680) in GitLab Runner 15.6. + Logs generated by applications running in service containers can be captured for subsequent examination and debugging. You might want to look at service container's logs when the service container has started successfully, but is not behaving es expected, leading to job failures. The logs can indicate missing or incorrect configuration of the service diff --git a/doc/ci/testing/code_quality.md b/doc/ci/testing/code_quality.md index d1ed28b79c0..8e1c3d72d3d 100644 --- a/doc/ci/testing/code_quality.md +++ b/doc/ci/testing/code_quality.md @@ -71,11 +71,12 @@ See also the Code Climate list of [Supported Languages for Maintainability](http > - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/284140) in GitLab 13.12. > - [Disabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/2526) in GitLab 14.0 due to [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/334116). > - [Inline annotation added](https://gitlab.com/gitlab-org/gitlab/-/issues/2526) and [feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/284140) in GitLab 14.1. +> - [Updated](https://gitlab.com/groups/gitlab-org/-/epics/8071) to handle multiple findings better in GitLab 15.7, disabled by default behind the `refactor_code_quality_inline_findings` [feature flag](../../administration/feature_flags.md). Changes to files in merge requests can cause Code Quality to fall if merged. In these cases, the merge request's diff view displays an indicator next to lines with new Code Quality violations. For example: - + ## Example configuration diff --git a/doc/ci/testing/fail_fast_testing.md b/doc/ci/testing/fail_fast_testing.md index 58471a626da..b1bc44a0a37 100644 --- a/doc/ci/testing/fail_fast_testing.md +++ b/doc/ci/testing/fail_fast_testing.md @@ -12,7 +12,7 @@ For applications that use RSpec for running tests, we've introduced the `Verify/ [template to run subsets of your test suite](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml), based on the changes in your merge request. -The template uses the [test_file_finder (`tff`) gem](https://gitlab.com/gitlab-org/ci-cd/test_file_finder/) +The template uses the [`test_file_finder` (`tff`) gem](https://gitlab.com/gitlab-org/ci-cd/test_file_finder/) that accepts a list of files as input, and returns a list of spec (test) files that it believes to be relevant to the input files. diff --git a/doc/ci/testing/img/code_quality_mr_diff_report_v14_2.png b/doc/ci/testing/img/code_quality_mr_diff_report_v14_2.png Binary files differdeleted file mode 100644 index c1e9aad24ac..00000000000 --- a/doc/ci/testing/img/code_quality_mr_diff_report_v14_2.png +++ /dev/null diff --git a/doc/ci/testing/img/code_quality_mr_diff_report_v15_7.png b/doc/ci/testing/img/code_quality_mr_diff_report_v15_7.png Binary files differnew file mode 100644 index 00000000000..b45124e0e5d --- /dev/null +++ b/doc/ci/testing/img/code_quality_mr_diff_report_v15_7.png diff --git a/doc/ci/testing/unit_test_report_examples.md b/doc/ci/testing/unit_test_report_examples.md index 5d4cfa88d88..87426fc8496 100644 --- a/doc/ci/testing/unit_test_report_examples.md +++ b/doc/ci/testing/unit_test_report_examples.md @@ -42,7 +42,7 @@ Use the following job in `.gitlab-ci.yml`: golang: stage: test script: - - go get gotest.tools/gotestsum + - go install gotest.tools/gotestsum@latest - gotestsum --junitfile report.xml --format testname artifacts: when: always diff --git a/doc/ci/troubleshooting.md b/doc/ci/troubleshooting.md index 87ebff74600..81e13192cef 100644 --- a/doc/ci/troubleshooting.md +++ b/doc/ci/troubleshooting.md @@ -40,29 +40,8 @@ is at: https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/editor/schema/ci.json. ``` -The schema rules for custom YAML tags like `!reference` are treated as invalid by your editor until the custom tags are -set in your editor settings. For example: - -- In VS Code, you can set `vscode-yaml` to parse `customTags` in your `settings.json` file: - - ```json - "yaml.customTags": [ - "!reference sequence" - ] - ``` - -- In Sublime Text, if you are using the `LSP-yaml` package, you can set `customTags` in your `LSP-yaml` user settings: - - ```json - { - "settings": { - "yaml.customTags": ["!reference sequence"] - } - } - ``` - To see the full list of custom tags covered by the CI/CD schema, check the -latest version of the schema, linked above. +latest version of the schema. ### Verify syntax with CI Lint tool @@ -116,7 +95,7 @@ if you are using that type: Troubleshooting guides are available for some CI/CD features and related topics: -- [Container Registry](../user/packages/container_registry/index.md#troubleshooting-the-gitlab-container-registry) +- [Container Registry](../user/packages/container_registry/troubleshoot_container_registry.md) - [GitLab Runner](https://docs.gitlab.com/runner/faq/) - [Merge Trains](pipelines/merge_trains.md#troubleshooting) - [Docker Build](docker/using_docker_build.md#troubleshooting) @@ -362,7 +341,7 @@ To [prevent duplicate pipelines](jobs/job_control.md#avoid-duplicate-pipelines), [`workflow: rules`](yaml/index.md#workflow) or rewrite your rules to control which pipelines can run. -### Console workaround if job using resource_group gets stuck **(FREE SELF)** +### Console workaround if job using `resource_group` gets stuck **(FREE SELF)** ```ruby # find resource group by name diff --git a/doc/ci/variables/index.md b/doc/ci/variables/index.md index 97707c603bd..10dfa0174a0 100644 --- a/doc/ci/variables/index.md +++ b/doc/ci/variables/index.md @@ -153,6 +153,8 @@ job: ### Add a CI/CD variable to a project +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362227) in GitLab 15.7, projects can define a maximum of 200 CI/CD variables. + You can add CI/CD variables to a project's settings. Only project members with the Maintainer role can add or update project CI/CD variables. To keep a CI/CD variable secret, put it @@ -191,7 +193,8 @@ The output is: ### Add a CI/CD variable to a group -> Support for environment scopes [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2874) in GitLab Premium 13.11 +> - Support for environment scopes [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2874) in GitLab Premium 13.11 +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362227) in GitLab 15.7, groups can define a maximum of 200 CI/CD variables. To make a CI/CD variable available to all projects in a group, define a group CI/CD variable. Only group owners can add or update group-level CI/CD variables. @@ -391,6 +394,21 @@ To mark a variable as protected: The variable is available for all subsequent pipelines. +### Expand CI/CD variables + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217309) in GitLab 13.7. + +Expanded variables treat values with the `$` character as a reference to another variable. CI/CD variables are expanded +by default. + +To treat variables with a `$` character as raw strings, disable variable expansion for the variable: + +1. In the project or group, go to **Settings > CI/CD**. +1. Expand the **Variables** section. +1. Next to the variable you want to do not want expanded, select **Edit**. +1. Clear the **Expand variable** checkbox. +1. Select **Update variable**. + ### CI/CD variable security Malicious code pushed to your `.gitlab-ci.yml` file could compromise your variables @@ -715,7 +733,7 @@ You can override the value of a CI/CD variable when you > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/295234) in GitLab 13.8. -You can grant permission to override variables to [maintainers](../../user/permissions.md#project-features) only. When other users try to run a pipeline +You can grant permission to override variables to users with the Maintainer role only. When other users try to run a pipeline with overridden variables, they receive the `Insufficient permissions to set pipeline variables` error message. @@ -757,7 +775,7 @@ You can configure [Auto DevOps](../../topics/autodevops/index.md) to pass CI/CD to a running application. To make a CI/CD variable available as an environment variable in the running application's container, -[prefix the variable key](../../topics/autodevops/customize.md#application-secret-variables) +[prefix the variable key](../../topics/autodevops/cicd_variables.md#configure-application-secret-variables) with `K8S_SECRET_`. CI/CD variables with multi-line values are not supported. @@ -776,7 +794,7 @@ or job scripts. Debug logging exposes job execution details that are usually hid by the runner and makes job logs more verbose. It also exposes all variables and secrets available to the job. -Before you enable debug logging, make sure only [team members](../../user/permissions.md#project-features) +Before you enable debug logging, make sure only team members can view job logs. You should also [delete job logs](../jobs/index.md#view-jobs-in-a-pipeline) with debug output before you make logs public again. diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index 852110a1415..d7f67b79416 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -40,7 +40,7 @@ as it can cause the pipeline to behave unexpectedly. | `CI_COMMIT_SHORT_SHA` | 11.7 | all | The first eight characters of `CI_COMMIT_SHA`. | | `CI_COMMIT_TAG` | 9.0 | 0.5 | The commit tag name. Available only in pipelines for tags. | | `CI_COMMIT_TAG_MESSAGE` | 15.5 | all | The commit tag message. Available only in pipelines for tags. | -| `CI_COMMIT_TIMESTAMP` | 13.4 | all | The timestamp of the commit in the ISO 8601 format. | +| `CI_COMMIT_TIMESTAMP` | 13.4 | all | The timestamp of the commit in the [ISO 8601](https://www.rfc-editor.org/rfc/rfc3339#appendix-A) format. | | `CI_COMMIT_TITLE` | 10.8 | all | The title of the commit. The full first line of the message. | | `CI_CONCURRENT_ID` | all | 11.10 | The unique ID of build execution in a single executor. | | `CI_CONCURRENT_PROJECT_ID` | all | 11.10 | The unique ID of build execution in a single executor and project. | @@ -75,6 +75,7 @@ as it can cause the pipeline to behave unexpectedly. | `CI_JOB_NAME_SLUG` | 15.4 | all | `CI_JOB_NAME_SLUG` in lowercase, shortened to 63 bytes, and with everything except `0-9` and `a-z` replaced with `-`. No leading / trailing `-`. Use in paths. | | `CI_JOB_STAGE` | 9.0 | 0.5 | The name of the job's stage. | | `CI_JOB_STATUS` | all | 13.5 | The status of the job as each runner stage is executed. Use with [`after_script`](../yaml/index.md#after_script). Can be `success`, `failed`, or `canceled`. | +| `CI_JOB_TIMEOUT` | 15.7 | 15.7 | The job timeout value. | | `CI_JOB_TOKEN` | 9.0 | 1.2 | A token to authenticate with [certain API endpoints](../jobs/ci_job_token.md). The token is valid as long as the job is running. | | `CI_JOB_URL` | 11.1 | 0.5 | The job details URL. | | `CI_JOB_STARTED_AT` | 13.10 | all | The UTC datetime when a job started, in [ISO 8601](https://www.rfc-editor.org/rfc/rfc3339#appendix-A) format. | @@ -94,6 +95,7 @@ as it can cause the pipeline to behave unexpectedly. | `CI_PROJECT_ID` | all | all | The ID of the current project. This ID is unique across all projects on the GitLab instance. | | `CI_PROJECT_NAME` | 8.10 | 0.5 | The name of the directory for the project. For example if the project URL is `gitlab.example.com/group-name/project-1`, `CI_PROJECT_NAME` is `project-1`. | | `CI_PROJECT_NAMESPACE` | 8.10 | 0.5 | The project namespace (username or group name) of the job. | +| `CI_PROJECT_NAMESPACE_ID` | 15.7 | 0.5 | The project namespace ID of the job. | | `CI_PROJECT_PATH_SLUG` | 9.3 | all | `$CI_PROJECT_PATH` in lowercase with characters that are not `a-z` or `0-9` replaced with `-` and shortened to 63 bytes. Use in URLs and domain names. | | `CI_PROJECT_PATH` | 8.10 | 0.5 | The project namespace with the project name included. | | `CI_PROJECT_REPOSITORY_LANGUAGES` | 12.3 | all | A comma-separated, lowercase list of the languages used in the repository. For example `ruby,javascript,html,css`. The maximum number of languages is limited to 5. An issue [proposes to increase the limit](https://gitlab.com/gitlab-org/gitlab/-/issues/368925). | diff --git a/doc/ci/variables/where_variables_can_be_used.md b/doc/ci/variables/where_variables_can_be_used.md index c8436fc044d..88d0c9a2454 100644 --- a/doc/ci/variables/where_variables_can_be_used.md +++ b/doc/ci/variables/where_variables_can_be_used.md @@ -28,16 +28,16 @@ There are two places defined variables can be used. On the: | [`artifacts:name`](../yaml/index.md#artifactsname) | yes | Runner | The variable expansion is made by GitLab Runner's shell environment. | | [`before_script`](../yaml/index.md#before_script) | yes | Script execution shell | The variable expansion is made by the [execution shell environment](#execution-shell-environment) | | [`cache:key`](../yaml/index.md#cachekey) | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | -| [`environment:name`](../yaml/index.md#environmentname) | yes | GitLab | Similar to `environment:url`, but the variables expansion doesn't support the following:<br/><br/>- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).<br/>- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).<br/>- [Persisted variables](#persisted-variables). | +| [`environment:name`](../yaml/index.md#environmentname) | yes | GitLab | Similar to `environment:url`, but the variables expansion doesn't support the following:<br/><br/>- `CI_ENVIRONMENT_*` variables.<br/>- [Persisted variables](#persisted-variables). | | [`environment:url`](../yaml/index.md#environmenturl) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab.<br/><br/>Supported are all variables defined for a job (project/group variables, variables from `.gitlab-ci.yml`, variables from triggers, variables from pipeline schedules).<br/><br/>Not supported are variables defined in the GitLab Runner `config.toml` and variables created in the job's `script`. | | [`environment:auto_stop_in`](../yaml/index.md#environmentauto_stop_in)| yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab.<br/><br/> The value of the variable being substituted should be a period of time in a human readable natural language form. See [possible inputs](../yaml/index.md#environmentauto_stop_in) for more information.| -| [`except:variables`](../yaml/index.md#onlyvariables--exceptvariables) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:<br/><br/>- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).<br/>- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).<br/>- [Persisted variables](#persisted-variables). | +| [`except:variables`](../yaml/index.md#onlyvariables--exceptvariables) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:<br/><br/>- `CI_ENVIRONMENT_*` variables, except `CI_ENVIRONMENT_NAME` which is supported.<br/>- [Persisted variables](#persisted-variables). | | [`image`](../yaml/index.md#image) | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | | [`include`](../yaml/index.md#include) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab. <br/><br/>See [Use variables with include](../yaml/includes.md#use-variables-with-include) for more information on supported variables. | -| [`only:variables`](../yaml/index.md#onlyvariables--exceptvariables) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:<br/><br/>- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).<br/>- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).<br/>- [Persisted variables](#persisted-variables). | +| [`only:variables`](../yaml/index.md#onlyvariables--exceptvariables) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:<br/><br/>- `CI_ENVIRONMENT_*` variables, except `CI_ENVIRONMENT_NAME` which is supported.<br/>- [Persisted variables](#persisted-variables). | | [`resource_group`](../yaml/index.md#resource_group) | yes | GitLab | Similar to `environment:url`, but the variables expansion doesn't support the following:<br/>- `CI_ENVIRONMENT_URL`<br/>- [Persisted variables](#persisted-variables). | | [`rules:exists`](../yaml/index.md#rulesexists) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab. | -| [`rules:if`](../yaml/index.md#rulesif) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:<br/><br/>- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).<br/>- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).<br/>- [Persisted variables](#persisted-variables). | +| [`rules:if`](../yaml/index.md#rulesif) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:<br/><br/>- `CI_ENVIRONMENT_*` variables, except `CI_ENVIRONMENT_NAME` which is supported.<br/>- [Persisted variables](#persisted-variables). | | [`script`](../yaml/index.md#script) | yes | Script execution shell | The variable expansion is made by the [execution shell environment](#execution-shell-environment). | | [`services:name`](../yaml/index.md#services) | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | | [`services`](../yaml/index.md#services) | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | diff --git a/doc/ci/yaml/artifacts_reports.md b/doc/ci/yaml/artifacts_reports.md index 9f5e4e919b0..e12786f06ce 100644 --- a/doc/ci/yaml/artifacts_reports.md +++ b/doc/ci/yaml/artifacts_reports.md @@ -107,19 +107,17 @@ GitLab can display the results of coverage report in the merge request ## `artifacts:reports:codequality` -> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212499) to GitLab Free in 13.2. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212499) to GitLab Free in 13.2. +> - [Added support for multiple reports in diff annotations and full pipeline report](https://gitlab.com/gitlab-org/gitlab/-/issues/9014) in 15.7. The `codequality` report collects [code quality issues](../testing/code_quality.md). The collected code quality report uploads to GitLab as an artifact. -GitLab can display the results of: +GitLab can display the results of one or more reports in: -- One or more reports in the merge request [code quality widget](../testing/code_quality.md#code-quality-widget). -- Only one report in: - - The merge request [diff annotations](../testing/code_quality.md#code-quality-in-diff-view). - Track progress on adding support for multiple reports in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/328257). - - The [full report](../testing/metrics_reports.md). Track progress on adding support for multiple reports in - [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/9014). +- The merge request [code quality widget](../testing/code_quality.md#code-quality-widget). +- The merge request [diff annotations](../testing/code_quality.md#code-quality-in-diff-view). +- The [full report](../testing/metrics_reports.md). ## `artifacts:reports:container_scanning` **(ULTIMATE)** @@ -152,16 +150,16 @@ GitLab can display the results of one or more reports in: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360766) in GitLab 15.3 This report is a Software Bill of Materials describing the components of a project -following the [cyclonedx](https://cyclonedx.org/docs/1.4) protocol format. +following the [CycloneDX](https://cyclonedx.org/docs/1.4) protocol format. -You can specify multiple cyclonedx reports per job. These can be either supplied +You can specify multiple CycloneDX reports per job. These can be either supplied as a list of filenames, a filename pattern, or both: - List of filenames: `cyclonedx: [gl-sbom-npm-npm.cdx.json, gl-sbom-bundler-gem.cdx.json]`. - A filename pattern: `cyclonedx: gl-sbom-*.json`. - Combination of both of the above: `cyclonedx: [gl-sbom-*.json, my-cyclonedx.json]`. -Below is an example of a job exposing cyclonedx artifacts: +Below is an example of a job exposing CycloneDX artifacts: ```yaml artifacts: diff --git a/doc/ci/yaml/includes.md b/doc/ci/yaml/includes.md index 2b9f42e1c75..daf2e653250 100644 --- a/doc/ci/yaml/includes.md +++ b/doc/ci/yaml/includes.md @@ -13,8 +13,8 @@ You can use [`include`](index.md#include) to include external YAML files in your To include a single configuration file, use either of these syntax options: -- `include` by itself with a single file, which is the same as - [`include:local`](index.md#includelocal): +- `include` by itself with a single file. If this is a local file, it is the same as [`include:local`](index.md#includelocal). + If this is a remote file, it is the same as [`include:remote`](index.md#includeremote). ```yaml include: '/templates/.after-script-template.yml' @@ -31,7 +31,8 @@ To include a single configuration file, use either of these syntax options: You can include an array of configuration files: -- If you do not specify an `include` type, the type defaults to [`include:local`](index.md#includelocal): +- If you do not specify an `include` type, each array item defaults to [`include:local`](index.md#includelocal) + or [`include:remote`](index.md#includeremote), as needed: ```yaml include: diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index 3392304775a..dffe409b193 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -86,6 +86,7 @@ of the listed keywords use the value defined in the `default` section. - [`artifacts`](#artifacts) - [`before_script`](#before_script) - [`cache`](#cache) +- [`hooks`](#hooks) - [`image`](#image) - [`interruptible`](#interruptible) - [`retry`](#retry) @@ -394,12 +395,12 @@ Use [`workflow`](workflow.md) to control pipeline behavior. #### `workflow:name` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/372538) in GitLab 15.5 [with a flag](../../administration/feature_flags.md) named `pipeline_name`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/372538) in GitLab 15.5 [with a flag](../../administration/feature_flags.md) named `pipeline_name`. Disabled by default. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/376095) in GitLab 15.7. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `pipeline_name`. -The feature is not ready for production use. +On self-managed GitLab, by default this feature is available. To hide the feature, +ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `pipeline_name`. You can use `name` in `workflow:` to define a name for pipelines. @@ -424,7 +425,7 @@ A configuration with different pipeline names depending on the pipeline conditio ```yaml variables: - PIPELINE_NAME: 'Default pipeline name' + PIPELINE_NAME: 'Default pipeline name' # A default is not required. workflow: name: '$PIPELINE_NAME' @@ -437,6 +438,11 @@ workflow: PIPELINE_NAME: 'Ruby 3 pipeline' ``` +**Additional details**: + +- If the name is an empty string, the pipeline is not assigned a name. A name consisting + of only CI/CD variables could evaluate to an empty string if all the variables are also empty. + #### `workflow:rules` The `rules` keyword in `workflow` is similar to [`rules` defined in jobs](#rules), @@ -1766,8 +1772,8 @@ deploy: **Additional details**: -- Enviroments created from this job definition are assigned a [tier](../environments/index.md#deployment-tier-of-environments) based on this value. -- Existing environments don't have their tier updated if this value is added later. Existing enviroments must have their tier updated via the [Environments API](../../api/environments.md#update-an-existing-environment). +- Environments created from this job definition are assigned a [tier](../environments/index.md#deployment-tier-of-environments) based on this value. +- Existing environments don't have their tier updated if this value is added later. Existing environments must have their tier updated via the [Environments API](../../api/environments.md#update-an-existing-environment). **Related topics**: @@ -1862,6 +1868,74 @@ rspec: - [Reuse configuration sections by using `extends`](yaml_optimization.md#use-extends-to-reuse-configuration-sections). - Use `extends` to reuse configuration from [included configuration files](yaml_optimization.md#use-extends-and-include-together). +### `hooks` + +> Introduced in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_hooks_pre_get_sources_script`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, +ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `ci_hooks_pre_get_sources_script`. +The feature is not ready for production use. + +Use `hooks` to specify lists of commands to execute on the runner +at certain stages of job execution, like before retrieving the Git repository. + +**Keyword type**: Job keyword. You can use it only as part of a job or in the +[`default` section](#default). + +**Possible inputs**: + +- A hash of hooks and their commands. Available hooks: `pre_get_sources_script`. + +#### `hooks:pre_get_sources_script` + +> Introduced in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_hooks_pre_get_sources_script`. Disabled by default. + +Use `hooks:pre_get_sources_script` to specify a list of commands to execute on the runner +before retrieving the Git repository and any submodules. You can use it +to adjust the Git client configuration first, for example. + +**Related topics**: + +- [GitLab Runner configuration](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) + +**Example of `hooks:pre_get_sources_script`**: + +```yaml +job1: + hooks: + pre_get_sources_script: + - echo 'hello job1 pre_get_sources_script' + script: echo 'hello job1 script' +``` + +### `id_tokens` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356986) in GitLab 15.7. + +Use `id_tokens` to create [JSON web tokens (JWT)](https://www.rfc-editor.org/rfc/rfc7519) to authenticate with third party services. All +JWTs created this way support OIDC authentication. The required `aud` sub-keyword is used to configure the `aud` claim for the JWT. + +**Possible inputs**: + +- Token names with their `aud` claims. `aud` can be a single string or as an array of strings. + +**Example of `id_tokens`**: + +```yaml +job_with_id_tokens: + id_tokens: + ID_TOKEN_1: + aud: https://gitlab.com + ID_TOKEN_2: + aud: + - https://gcp.com + - https://aws.com + script: + - command_to_authenticate_with_gitlab $ID_TOKEN_1 + - command_to_authenticate_with_aws $ID_TOKEN_2 +``` + ### `image` Use `image` to specify a Docker image that the job runs in. @@ -2210,6 +2284,9 @@ This example creates four paths of execution: in a job's `needs` section. - In GitLab 13.9 and older, if `needs` refers to a job that might not be added to a pipeline because of `only`, `except`, or `rules`, the pipeline might fail to create. In GitLab 13.10 and later, use the [`needs:optional`](#needsoptional) keyword to resolve a failed pipeline creation. +- If a pipeline has jobs with `needs: []` and jobs in the [`.pre`](#stage-pre) stage, they will + all start as soon as the pipeline is created. Jobs with `needs: []` start immediately, + and jobs in the `.pre` stage also start immediately. #### `needs:artifacts` @@ -2268,12 +2345,12 @@ In this example: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14311) in GitLab 12.7. Use `needs:project` to download artifacts from up to five jobs in other pipelines. -The artifacts are downloaded from the latest successful pipeline for the specified ref. +The artifacts are downloaded from the latest successful specified job for the specified ref. To specify multiple jobs, add each as separate array items under the `needs` keyword. -If there is a pipeline running for the specified ref, a job with `needs:project` -does not wait for the pipeline to complete. Instead, the job downloads the artifact -from the latest pipeline that completed successfully. +If there is a pipeline running for the ref, a job with `needs:project` +does not wait for the pipeline to complete. Instead, the artifacts are downloaded +from the latest successful run of the specified job. `needs:project` must be used with `job`, `ref`, and `artifacts`. @@ -2963,7 +3040,7 @@ job: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363024) in GitLab 15.3. Supported by `release-cli` v0.12.0 or later. -If the tag does not exist, the newly created tag is annotated with the message specifed by `tag_message`. +If the tag does not exist, the newly created tag is annotated with the message specified by `tag_message`. If omitted, a lightweight tag is created. **Keyword type**: Job keyword. You can use it only as part of a job. @@ -3670,7 +3747,7 @@ job: ### `services` -Use `services` to specify an additional Docker image to run scripts in. The [`services` image](../services/index.md) is linked +Use `services` to specify any additional Docker images that your scripts require to run successfully. The [`services` image](../services/index.md) is linked to the image specified in the [`image`](#image) keyword. **Keyword type**: Job keyword. You can use it only as part of a job or in the @@ -3706,9 +3783,11 @@ test: - bundle exec rake spec ``` -In this example, the job launches a Ruby container. Then, from that container, the job launches -another container that's running PostgreSQL. Then the job then runs scripts -in that container. +In this example, GitLab launches two containers for the job: + +- A Ruby container that runs the `script` commands. +- A PostgreSQL container. The `script` commands in the Ruby container can connect to + the PostgreSQL database at the `db-postgrest` hostname. **Related topics**: @@ -3882,6 +3961,12 @@ job2: - echo "This job runs in the test stage." ``` +**Additional details:** + +- If a pipeline has jobs with [`needs: []`](#needs) and jobs in the `.pre` stage, they will + all start as soon as the pipeline is created. Jobs with `needs: []` start immediately, + ignoring any stage configuration. + ### `tags` > - A limit of 50 tags per job [enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/338929) in GitLab 14.3. @@ -4228,7 +4313,8 @@ deploy_review_job: #### `variables:description` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30101) in GitLab 13.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30101) in GitLab 13.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363660) in GitLab 15.5, `variables:value` can contain an array of values. Use the `description` keyword to define a [pipeline-level (global) variable that is prefilled](../pipelines/index.md#prefill-variables-in-manual-pipelines) when [running a pipeline manually](../pipelines/index.md#run-a-pipeline-manually). @@ -4240,6 +4326,7 @@ If used with `value`, the variable value is also prefilled when running a pipeli **Possible inputs**: - A string. +- An array of strings. **Example of `variables:description`**: @@ -4254,10 +4341,17 @@ variables: - A global variable defined with `value` but no `description` behaves the same as [`variables`](#variables). +- `variables:value` can [contain an array of selectable values](../pipelines/index.md#configure-a-list-of-selectable-values-for-a-prefilled-variable). #### `variables:expand` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353991) in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_raw_variables_in_yaml_config`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353991) in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_raw_variables_in_yaml_config`. Disabled by default. +> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/375034) in GitLab 15.6. +> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/375034) in GitLab 15.7. + +FLAG: +On self-managed GitLab, by default this feature is available. To hide the feature per project, +ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `ci_raw_variables_in_yaml_config`. Use the `expand` keyword to configure a variable to be expandable or not. diff --git a/doc/ci/yaml/yaml_optimization.md b/doc/ci/yaml/yaml_optimization.md index f4774619713..5344a999b95 100644 --- a/doc/ci/yaml/yaml_optimization.md +++ b/doc/ci/yaml/yaml_optimization.md @@ -473,3 +473,27 @@ nested-references: ``` In this example, the `nested-references` job runs all three `echo` commands. + +### Configure your IDE to support `!reference` tags + +The [pipeline editor](../pipeline_editor/index.md) supports `!reference` tags. However, the schema rules for custom YAML +tags like `!reference` might be treated as invalid by your editor by default. +You can configure some editors to accept `!reference` tags. For example: + +- In VS Code, you can set `vscode-yaml` to parse `customTags` in your `settings.json` file: + + ```json + "yaml.customTags": [ + "!reference sequence" + ] + ``` + +- In Sublime Text, if you are using the `LSP-yaml` package, you can set `customTags` in your `LSP-yaml` user settings: + + ```json + { + "settings": { + "yaml.customTags": ["!reference sequence"] + } + } + ``` diff --git a/doc/development/adding_database_indexes.md b/doc/development/adding_database_indexes.md deleted file mode 100644 index 7ab846cce3e..00000000000 --- a/doc/development/adding_database_indexes.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/adding_database_indexes.md' -remove_date: '2022-11-05' ---- - -This document was moved to [another location](database/adding_database_indexes.md). - -<!-- This redirect file can be deleted after <2022-11-05>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/api_graphql_styleguide.md b/doc/development/api_graphql_styleguide.md index c19341a1404..e0db0d7e34d 100644 --- a/doc/development/api_graphql_styleguide.md +++ b/doc/development/api_graphql_styleguide.md @@ -2115,7 +2115,7 @@ end Authenticating a user with the `current_user:` argument for `post_graphql` generates more queries on the first request than on subsequent requests on that same user. If you are testing for N+1 queries using - [QueryRecorder](query_recorder.md), use a **different** user for each request. + [QueryRecorder](database/query_recorder.md), use a **different** user for each request. The below example shows how a test for avoiding N+1 queries should look: @@ -2223,7 +2223,7 @@ end [Adding field with resolver on a Type causes "Can't determine the return type " error on a different Type](https://github.com/rmosolgo/graphql-ruby/issues/3974). Unfortunately, the errors generated don't really indicate what the problem is. For example, - remove the quotes from the `Rspec.descrbe` in + remove the quotes from the `Rspec.describe` in [ee/spec/graphql/resolvers/compliance_management/merge_requests/compliance_violation_resolver_spec.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/spec/graphql/resolvers/compliance_management/merge_requests/compliance_violation_resolver_spec.rb). Then run `rspec ee/spec/graphql/resolvers/compliance_management/merge_requests/compliance_violation_resolver_spec.rb`. @@ -2263,7 +2263,7 @@ end 1. 95% of the resolver specs use arguments that are Ruby objects, as opposed to when using the GraphQL API only strings and integers are used. This works fine in most cases. - 1. If your resolver takes arguments that use a `prepare` proc, such as a resolver that accepts timeframe + 1. If your resolver takes arguments that use a `prepare` proc, such as a resolver that accepts time frame arguments (`TimeFrameArguments`), you must pass the `arg_style: :internal_prepared` parameter into the `resolve` method. This tells the code to convert the arguments into strings and integers and pass them through regular argument handling, ensuring that the `prepare` proc is called correctly. diff --git a/doc/development/api_styleguide.md b/doc/development/api_styleguide.md index f74ed8db50f..006d0a01abb 100644 --- a/doc/development/api_styleguide.md +++ b/doc/development/api_styleguide.md @@ -152,6 +152,34 @@ For non-200 HTTP responses, use the provided helpers in `lib/api/helpers.rb` to For `DELETE` requests, you should also generally use the `destroy_conditionally!` helper which by default returns a `204 No Content` response on success, or a `412 Precondition Failed` response if the given `If-Unmodified-Since` header is out of range. This helper calls `#destroy` on the passed resource, but you can also implement a custom deletion method by passing a block. +## Choosing HTTP verbs + +When defining a new [API route](https://github.com/ruby-grape/grape#routes), use +the correct [HTTP request method](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods). + +### Deciding between `PATCH` and `PUT` + +In a Rails application, both the `PATCH` and `PUT` request methods are routed to +the `update` method in controllers. With Grape, the framework we use to write +the GitLab API, you must explicitly set the `PATCH` or `PUT` HTTP verb for an +endpoint that does updates. + +If the endpoint updates *all* attributes of a given resource, use the +[`PUT`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/PUT) request +method. If the endpoint updates *some* attributes of a given resource, use the +[`PATCH`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/PATCH) +request method. + +Here is a good example for `PATCH`: [`PATCH /projects/:id/protected_branches/:name`](../api/protected_branches.md#update-a-protected-branch) +Here is a good example for `PUT`: [`PUT /projects/:id/merge_requests/:merge_request_iid/approve`](../api/merge_request_approvals.md#approve-merge-request) + +Often, a good `PUT` endpoint only has ids and a verb (in the example above, "approve"). +Or, they only have a single value and represent a key/value pair. + +The [Rails blog](https://rubyonrails.org/2012/2/26/edge-rails-patch-is-the-new-primary-http-method-for-updates) +has a detailed explanation of why `PATCH` is usually the most apt verb for web +API endpoints that perform an update. + ## Using API path helpers in GitLab Rails codebase Because we support [installing GitLab under a relative URL](../install/relative_url.md), one must take this @@ -271,7 +299,7 @@ which introduced the scope. When an API endpoint returns collections, always add a test to verify that the API endpoint does not have an N+1 problem, now and in the future. -We can do this using [`ActiveRecord::QueryRecorder`](query_recorder.md). +We can do this using [`ActiveRecord::QueryRecorder`](database/query_recorder.md). Example: diff --git a/doc/development/application_limits.md b/doc/development/application_limits.md index b64d25ccf64..b1efc11db62 100644 --- a/doc/development/application_limits.md +++ b/doc/development/application_limits.md @@ -38,7 +38,7 @@ It's recommended to create two separate migration script files. desired limit using `create_or_update_plan_limit` migration helper, such as: ```ruby - class InsertProjectHooksPlanLimits < Gitlab::Database::Migration[2.0] + class InsertProjectHooksPlanLimits < Gitlab::Database::Migration[2.1] def up create_or_update_plan_limit('project_hooks', 'default', 0) create_or_update_plan_limit('project_hooks', 'free', 10) diff --git a/doc/development/application_slis/rails_request_apdex.md b/doc/development/application_slis/rails_request_apdex.md index 2fa9f5f4869..8fcd725f74d 100644 --- a/doc/development/application_slis/rails_request_apdex.md +++ b/doc/development/application_slis/rails_request_apdex.md @@ -108,7 +108,7 @@ a case-by-case basis. Take the following into account: should try to keep as short as possible. 1. Traffic characteristics should also be taken into account. If the - traffic to the endpoint is bursty, like CI traffic spinning up a + traffic to the endpoint sometimes bursts, like CI traffic spinning up a big batch of jobs hitting the same endpoint, then having these endpoints take five seconds is unacceptable from an infrastructure point of view. We cannot scale up the fleet fast enough to accommodate for diff --git a/doc/development/approval_rules.md b/doc/development/approval_rules.md index ce774b1e8f9..312bf2b1bb7 100644 --- a/doc/development/approval_rules.md +++ b/doc/development/approval_rules.md @@ -1,6 +1,6 @@ --- stage: Create -group: Source Code +group: Code Review info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -221,7 +221,7 @@ It is responsible for parsing `approval_rules_attributes` parameter to: - Filter the group IDs whether they are visible to user. - Identify the `any_approver` rule. - Append hidden groups to it when specified. -- Append user defined inapplicable (rules that does not apply to MR's target +- Append user defined inapplicable (rules that do not apply to the merge request's target branch) approval rules. ## Flow diff --git a/doc/development/architecture.md b/doc/development/architecture.md index a731e661a80..5eb1dcc3208 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -592,8 +592,6 @@ Grafana is an open source, feature rich metrics dashboard and graph editor for G Jaeger, inspired by Dapper and OpenZipkin, is a distributed tracing system. It can be used for monitoring microservices-based distributed systems. -For monitoring deployed apps, see [Jaeger tracing documentation](../operations/tracing.md) - #### Logrotate - [Project page](https://github.com/logrotate/logrotate/blob/master/README.md) @@ -626,7 +624,7 @@ Mattermost is an open source, private cloud, Slack-alternative from <https://mat - Layer: Core Service (Data) - GitLab.com: [Storage Architecture](https://about.gitlab.com/handbook/engineering/infrastructure/production/architecture/#storage-architecture) -MinIO is an object storage server released under the GNU AGPL v3.0. It is compatible with Amazon S3 cloud storage service. It is best suited for storing unstructured data such as photos, videos, log files, backups, and container / VM images. Size of an object can range from a few KBs to a maximum of 5TB. +MinIO is an object storage server released under the GNU AGPL v3.0. It is compatible with Amazon S3 cloud storage service. It is best suited for storing unstructured data such as photos, videos, log files, backups, and container / VM images. Size of an object can range from a few KB to a maximum of 5 TB. #### NGINX diff --git a/doc/development/caching.md b/doc/development/caching.md index 36fbfc7010e..58ec7a77591 100644 --- a/doc/development/caching.md +++ b/doc/development/caching.md @@ -80,7 +80,7 @@ indicates we have plenty of headroom. - Generic data can be cached for everyone. - You must keep this in mind when building new features. 1. Try to preserve cache data as much as possible: - - Use nested caches to maintain as much cached data as possible across expiries. + - Use nested caches to maintain as much cached data as possible across expires. 1. Perform as few requests to the cache as possible: - This reduces variable latency caused by network issues. - Lower overhead for each read on the cache. @@ -166,7 +166,7 @@ Is the cache being added "worthy"? This can be hard to measure, but you can cons - Calling the same method multiple times but only calculating the value once. - Stored in Ruby memory. - `@article ||= Article.find(params[:id])` - - `strong_memoize { Article.find(params[:id]) }` + - `strong_memoize_attr :method_name` 1. Request caching: - Return the same value for a key for the duration of a web request. - `Gitlab::SafeRequestStore.fetch` @@ -252,7 +252,7 @@ All the time! ### When to use method caching -- Using instance variables, or [strong_memoize](utilities.md#strongmemoize) is something we all tend to do anyway. +- Use instance variables, or [`StrongMemoize`](utilities.md#strongmemoize). - Useful when the same value is needed multiple times in a request. - Can be used to prevent multiple cache calls for the same key. - Can cause issues with ActiveRecord objects where a value doesn't change until you call diff --git a/doc/development/cascading_settings.md b/doc/development/cascading_settings.md index 1a0f0ec5b5f..389623e68d8 100644 --- a/doc/development/cascading_settings.md +++ b/doc/development/cascading_settings.md @@ -38,7 +38,7 @@ Settings are not cascading by default. To define a cascading setting, take the f `application_settings`. ```ruby - class AddDelayedProjectRemovalCascadingSetting < Gitlab::Database::Migration[2.0] + class AddDelayedProjectRemovalCascadingSetting < Gitlab::Database::Migration[2.1] include Gitlab::Database::MigrationHelpers::CascadingNamespaceSettings enable_lock_retries! @@ -153,7 +153,7 @@ Renders the label for a checkbox setting. [`_setting_label_fieldset.html.haml`](https://gitlab.com/gitlab-org/gitlab/-/blob/c2736823b8e922e26fd35df4f0cd77019243c858/app/views/shared/namespaces/cascading_settings/_setting_label_fieldset.html.haml) -Renders the label for a fieldset setting. +Renders the label for a `fieldset` setting. | Local | Description | Type | Required (default value) | |:-----------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------|:-------------------------| diff --git a/doc/development/cicd/pipeline_wizard.md b/doc/development/cicd/pipeline_wizard.md index eb6d1c60bb1..213a9fe1762 100644 --- a/doc/development/cicd/pipeline_wizard.md +++ b/doc/development/cicd/pipeline_wizard.md @@ -127,7 +127,7 @@ is planned to add the ability to create a MR from here. ### Props -- `template` (required): The template content as an unparsed String. See +- `template` (required): The template content as an un-parsed string. See [Template file location](#template-file-location) for more information. - `project-path` (required): The full path of the project the final file should be committed to diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 90f33319365..30d9d671038 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -28,7 +28,7 @@ The reviewer can: - Give you a second opinion on the chosen solution and implementation. - Help look for bugs, logic problems, or uncovered edge cases. -If the merge request is trivial (for example, fixing a typo or a tiny refactor that doesn't change the behavior or any data), +If the merge request is trivial to review (for example, fixing a typo or a tiny refactor that doesn't change the behavior or any data), you can skip the reviewer step and directly ask a [maintainer](https://about.gitlab.com/handbook/engineering/workflow/code-review/#maintainer). Otherwise, a merge request should always be first reviewed by a reviewer in each [category (e.g. backend, database)](#approval-guidelines) @@ -124,8 +124,10 @@ page, with these behaviors: branch name (unless their out-of-office (`OOO`) status changes, as in point 1). It removes leading `ce-` and `ee-`, and trailing `-ce` and `-ee`, so that it can be stable for backport branches. +- People whose Slack or [GitLab status](../user/profile/index.md#set-your-current-status) emoji + is Ⓜ `:m:`are only suggested as reviewers on projects they are a maintainer of. -The [Roulette dashboard](https://gitlab-org.gitlab.io/gitlab-roulette) contains: +The [Roulette dashboard](https://gitlab-org.gitlab.io/gitlab-roulette/) contains: - Assignment events in the last 7 and 30 days. - Currently assigned merge requests per person. @@ -155,7 +157,7 @@ with [domain expertise](#domain-experts). | `~frontend` changes (*1*) | [Frontend maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_frontend). | | `~UX` user-facing changes (*3*) | [Product Designer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_reviewers_UX). Refer to the [design and user interface guidelines](contributing/design.md) for details. | | Adding a new JavaScript library (*1*) | - [Frontend foundations member](https://about.gitlab.com/direction/manage/foundations/) if the library significantly increases the [bundle size](https://gitlab.com/gitlab-org/frontend/playground/webpack-memory-metrics/-/blob/master/doc/report.md).<br/>- A [legal department member](https://about.gitlab.com/handbook/legal/) if the license used by the new library hasn't been approved for use in GitLab.<br/><br/>More information about license compatibility can be found in our [GitLab Licensing and Compatibility documentation](licensing.md). | -| A new dependency or a file system change | - [Distribution team member](https://about.gitlab.com/company/team/). See how to work with the [Distribution team](https://about.gitlab.com/handbook/engineering/development/enablement/systems/distribution/#how-to-work-with-distribution) for more details.<br/>- For Rubygems, request an [AppSec review](gemfile.md#request-an-appsec-review). | +| A new dependency or a file system change | - [Distribution team member](https://about.gitlab.com/company/team/). See how to work with the [Distribution team](https://about.gitlab.com/handbook/engineering/development/enablement/systems/distribution/#how-to-work-with-distribution) for more details.<br/>- For RubyGems, request an [AppSec review](gemfile.md#request-an-appsec-review). | | `~documentation` or `~UI text` changes | [Technical writer](https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments) based on assignments in the appropriate [DevOps stage group](https://about.gitlab.com/handbook/product/categories/#devops-stages). | | Changes to development guidelines | Follow the [review process](development_processes.md#development-guidelines-review) and get the approvals accordingly. | | End-to-end **and** non-end-to-end changes (*4*) | [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors). | @@ -185,49 +187,52 @@ Using checklists improves quality in software engineering. This checklist is a s See the [test engineering process](https://about.gitlab.com/handbook/engineering/quality/quality-engineering/test-engineering/) for further quality guidelines. -1. I have self-reviewed this MR per [code review guidelines](code_review.md). -1. For the code that this change impacts, I believe that the automated tests ([Testing Guide](testing_guide/index.md)) validate functionality that is highly important to users (including consideration of [all test levels](testing_guide/testing_levels.md)). -1. If the existing automated tests do not cover the above functionality, I have added the necessary additional tests or added an issue to describe the automation testing gap and linked it to this MR. -1. I have considered the technical aspects of this change's impact on GitLab.com hosted customers and self-managed customers. -1. I have considered the impact of this change on the frontend, backend, and database portions of the system where appropriate and applied the `~ux`, `~frontend`, `~backend`, and `~database` labels accordingly. -1. I have tested this MR in [all supported browsers](../install/requirements.md#supported-web-browsers), or determined that this testing is not needed. -1. I have confirmed that this change is [backwards compatible across updates](multi_version_compatibility.md), or I have decided that this does not apply. -1. I have properly separated EE content from FOSS, or this MR is FOSS only. +1. You have self-reviewed this MR per [code review guidelines](code_review.md). +1. For the code that this change impacts, you believe that the automated tests ([Testing Guide](testing_guide/index.md)) validate functionality that is highly important to users (including consideration of [all test levels](testing_guide/testing_levels.md)). +1. If the existing automated tests do not cover the above functionality, you have added the necessary additional tests or added an issue to describe the automation testing gap and linked it to this MR. +1. You have considered the technical aspects of this change's impact on GitLab.com hosted customers and self-managed customers. +1. You have considered the impact of this change on the frontend, backend, and database portions of the system where appropriate and applied the `~ux`, `~frontend`, `~backend`, and `~database` labels accordingly. +1. You have tested this MR in [all supported browsers](../install/requirements.md#supported-web-browsers), or determined that this testing is not needed. +1. You have confirmed that this change is [backwards compatible across updates](multi_version_compatibility.md), or you have decided that this does not apply. +1. You have properly separated EE content from FOSS, or this MR is FOSS only. - [Where should EE code go?](ee_features.md) -1. I have considered that existing data may be surprisingly varied. For example, a new model validation can break existing records. Consider making validation on existing data optional rather than required if you haven't confirmed that existing data will pass validation. +1. You have considered that existing data may be surprisingly varied. For example, a new model validation can break existing records. Consider making validation on existing data optional rather than required if you haven't confirmed that existing data will pass validation. ##### Performance, reliability, and availability -1. I am confident that this MR does not harm performance, or I have asked a reviewer to help assess the performance impact. ([Merge request performance guidelines](merge_request_performance_guidelines.md)) -1. I have added [information for database reviewers in the MR description](database_review.md#required), or I have decided that it is unnecessary. +1. You are confident that this MR does not harm performance, or you have asked a reviewer to help assess the performance impact. ([Merge request performance guidelines](merge_request_performance_guidelines.md)) +1. You have added [information for database reviewers in the MR description](database_review.md#required), or you have decided that it is unnecessary. - [Does this MR have database-related changes?](database_review.md) -1. I have considered the availability and reliability risks of this change. -1. I have considered the scalability risk based on future predicted growth. -1. I have considered the performance, reliability, and availability impacts of this change on large customers who may have significantly more data than the average customer. +1. You have considered the availability and reliability risks of this change. +1. You have considered the scalability risk based on future predicted growth. +1. You have considered the performance, reliability, and availability impacts of this change on large customers who may have significantly more data than the average customer. ##### Observability instrumentation -1. I have included enough instrumentation to facilitate debugging and proactive performance improvements through observability. +1. You have included enough instrumentation to facilitate debugging and proactive performance improvements through observability. See [example](https://gitlab.com/gitlab-org/gitlab/-/issues/346124#expectations) of adding feature flags, logging, and instrumentation. ##### Documentation -1. I have included changelog trailers, or I have decided that they are not needed. +1. You have included changelog trailers, or you have decided that they are not needed. - [Does this MR need a changelog?](changelog.md#what-warrants-a-changelog-entry) -1. I have added/updated documentation or decided that documentation changes are unnecessary for this MR. +1. You have added/updated documentation or decided that documentation changes are unnecessary for this MR. - [Is documentation required?](https://about.gitlab.com/handbook/product/ux/technical-writing/workflow/#when-documentation-is-required) ##### Security -1. I have confirmed that if this MR contains changes to processing or storing of credentials or tokens, authorization, and authentication methods, or other items described in [the security review guidelines](https://about.gitlab.com/handbook/security/#when-to-request-a-security-review), I have added the `~security` label and I have `@`-mentioned `@gitlab-com/gl-security/appsec`. -1. I have reviewed the documentation regarding [internal application security reviews](https://about.gitlab.com/handbook/security/#internal-application-security-reviews) for **when** and **how** to request a security review and requested a security review if this is warranted for this change. +1. You have confirmed that if this MR contains changes to processing or storing of credentials or tokens, authorization, and authentication methods, or other items described in [the security review guidelines](https://about.gitlab.com/handbook/security/#when-to-request-a-security-review), you have added the `~security` label and you have `@`-mentioned `@gitlab-com/gl-security/appsec`. +1. You have reviewed the documentation regarding [internal application security reviews](https://about.gitlab.com/handbook/security/#internal-application-security-reviews) for **when** and **how** to request a security review and requested a security review if this is warranted for this change. +1. If there are security scan results that are blocking the MR (due to the [scan result policies](https://gitlab.com/gitlab-com/gl-security/security-policies)): + - For true positive findings, they should be corrected before the merge request is merged. This will remove the AppSec approval required by the scan result policy. + - For false positive findings, something that should be discussed for risk acceptance, or anything questionable, please ping `@gitlab-com/gl-security/appsec`. ##### Deployment -1. I have considered using a feature flag for this change because the change may be high risk. -1. If I am using a feature flag, I plan to test the change in staging before I test it in production, and I have considered rolling it out to a subset of production customers before rolling it out to all customers. +1. You have considered using a feature flag for this change because the change may be high risk. +1. If you are using a feature flag, you plan to test the change in staging before you test it in production, and you have considered rolling it out to a subset of production customers before rolling it out to all customers. - [When to use a feature flag](https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/#when-to-use-feature-flags) -1. I have informed the Infrastructure department of a default setting or new setting change per [definition of done](contributing/merge_request_workflow.md#definition-of-done), or decided that this is unnecessary. +1. You have informed the Infrastructure department of a default setting or new setting change per [definition of done](contributing/merge_request_workflow.md#definition-of-done), or decided that this is unnecessary. ### The responsibility of the merge request author diff --git a/doc/development/contributing/design.md b/doc/development/contributing/design.md index e6b6b56cf73..b8d7a8eef39 100644 --- a/doc/development/contributing/design.md +++ b/doc/development/contributing/design.md @@ -24,8 +24,17 @@ screenshots (or videos) of your changes in the description, as explained in our [MR workflow](merge_request_workflow.md). These screenshots/videos are very helpful for all reviewers and can speed up the review process, especially if the changes are small. -- Attach the ~UX label to any merge request that impacts the user experience. This will enable Product Designers to [review](https://about.gitlab.com/handbook/product/ux/product-designer/mr-reviews/#stage-group-mrs/) any user facing changes. -- Assign the Product Designer suggested by Reviewer Roulette as the reviewer of your merge request. The reviewer does not have to be the domain expert unless this is a community contribution. +- Attach the ~UX label to any merge request that has any user facing changes. This will trigger our +Reviewer Roulette to suggest a UX [reviewer](https://about.gitlab.com/handbook/product/ux/product-designer/mr-reviews/#stage-group-mrs). + +If you are a **team member**: We recommend assigning the Product Designer suggested by the +[Reviewer Roulette](../code_review.md#reviewer-roulette) as reviewer. [This helps us](https://about.gitlab.com/handbook/product/ux/product-designer/mr-reviews/#benefits) spread work evenly, improve communication, and make our UI more +consistent. If you have a reason to choose a different reviewer, add a comment to mention you assigned +it to a Product Designer of your choice. + +If you are a **community contributor**: We favor choosing the Product Designer that is a +[domain expert](../code_review.md#domain-experts) in the area you are contributing, to regardless +of the Reviewer Roulette. ## Checklist diff --git a/doc/development/contributing/index.md b/doc/development/contributing/index.md index 8c0d18f877b..1a5b801a95a 100644 --- a/doc/development/contributing/index.md +++ b/doc/development/contributing/index.md @@ -100,7 +100,7 @@ If you have any questions or need help, visit [Getting Help](https://about.gitla communicate with the GitLab community. GitLab prefers [asynchronous communication](https://about.gitlab.com/handbook/communication/#internal-communication) over real-time communication. We do encourage you to connect and hang out with us. GitLab has a Gitter room dedicated for [contributors](https://gitter.im/gitlab/contributors), which is bridged with our -internal Slack. We actively monitor this channel. There is also a community-run [Discord server](http://discord.gg/gitlab) where you can +internal Slack. We actively monitor this channel. There is also a community-run [Discord server](https://discord.gg/gitlab) where you can find other contributors in the `#contributors` channel. Thanks for your contribution! @@ -229,4 +229,4 @@ For information on how to contribute documentation, see GitLab ## Getting an Enterprise Edition License If you need a license for contributing to an EE-feature, see -[relevant information](https://about.gitlab.com/handbook/marketing/community-relations/code-contributor-program/#contributing-to-the-gitlab-enterprise-edition-ee). +[relevant information](https://about.gitlab.com/handbook/marketing/community-relations/code-contributor-program/operations/#contributing-to-the-gitlab-enterprise-edition-ee). diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index f2c06e289c9..f06e8825660 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -213,6 +213,7 @@ To reach the definition of done, the merge request must create no regressions an - Verified as working in production on GitLab.com. - Verified as working for self-managed instances. +- Verified as supporting [Geo](../../administration/geo/index.md) via the [self-service framework](../geo/framework.md). To learn more see [here](../geo/framework.md#geo-is-a-requirement-in-the-definition-of-done). If a regression occurs, we prefer you revert the change. Your contribution is *incomplete* until you have made sure it meets all of these @@ -313,7 +314,7 @@ The following items are checked after the merge request has been merged: 1. Confirmed to be working in staging before implementing the change in production, where possible. 1. Confirmed to be working in the production with no new [Sentry](https://about.gitlab.com/handbook/engineering/monitoring/#sentry) errors after the contribution is deployed. 1. Confirmed that the [rollout plan](https://about.gitlab.com/handbook/engineering/development/processes/rollout-plans/) has been completed. -1. If there is a performance risk in the change, I have analyzed the performance of the system before and after the change. +1. If there is a performance risk in the change, you have analyzed the performance of the system before and after the change. 1. *If the merge request uses feature flags, per-project or per-group enablement, and a staged rollout:* - Confirmed to be working on GitLab projects. - Confirmed to be working at each stage for all projects added. diff --git a/doc/development/contributing/style_guides.md b/doc/development/contributing/style_guides.md index fe1aa8449c2..0a030d1f3bc 100644 --- a/doc/development/contributing/style_guides.md +++ b/doc/development/contributing/style_guides.md @@ -47,7 +47,7 @@ We were using Overcommit prior to Lefthook, so you may want to uninstall it firs bundle exec lefthook run pre-push ``` -This should return the lefthook version and the list of executable commands with output. +This should return the Lefthook version and the list of executable commands with output. ### Lefthook configuration diff --git a/doc/development/creating_enums.md b/doc/development/creating_enums.md deleted file mode 100644 index d3892c4c44e..00000000000 --- a/doc/development/creating_enums.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/creating_enums.md' -remove_date: '2022-11-06' ---- - -This document was moved to [another location](database/creating_enums.md). - -<!-- This redirect file can be deleted after <2022-11-06>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/dangerbot.md b/doc/development/dangerbot.md index 52503f2d9c8..b568809ea4e 100644 --- a/doc/development/dangerbot.md +++ b/doc/development/dangerbot.md @@ -132,7 +132,7 @@ have had the chance to add to `helper.labels_to_add`. #### Shared rules and plugins If the rule or plugin you implement can be useful for other projects, think about -upstreaming them to the [`gitlab-dangerfiles`](https://gitlab.com/gitlab-org/ruby/gems/gitlab-dangerfiles) project. +adding them upstream to the [`gitlab-dangerfiles`](https://gitlab.com/gitlab-org/ruby/gems/gitlab-dangerfiles) project. #### Enable Danger on a project diff --git a/doc/development/database/add_foreign_key_to_existing_column.md b/doc/development/database/add_foreign_key_to_existing_column.md index 1609e00531e..07fa8133496 100644 --- a/doc/development/database/add_foreign_key_to_existing_column.md +++ b/doc/development/database/add_foreign_key_to_existing_column.md @@ -69,7 +69,7 @@ In the example above, you'd be still able to update records in the `emails` tabl Migration file for adding `NOT VALID` foreign key: ```ruby -class AddNotValidForeignKeyToEmailsUser < Gitlab::Database::Migration[2.0] +class AddNotValidForeignKeyToEmailsUser < Gitlab::Database::Migration[2.1] def up add_concurrent_foreign_key :emails, :users, column: :user_id, on_delete: :cascade, validate: false end @@ -100,7 +100,7 @@ In case the data volume is higher (>1000 records), it's better to create a backg Example for cleaning up records in the `emails` table in a database migration: ```ruby -class RemoveRecordsWithoutUserFromEmailsTable < Gitlab::Database::Migration[2.0] +class RemoveRecordsWithoutUserFromEmailsTable < Gitlab::Database::Migration[2.1] disable_ddl_transaction! class Email < ActiveRecord::Base @@ -133,7 +133,7 @@ Migration file for validating the foreign key: ```ruby # frozen_string_literal: true -class ValidateForeignKeyOnEmailUsers < Gitlab::Database::Migration[2.0] +class ValidateForeignKeyOnEmailUsers < Gitlab::Database::Migration[2.1] def up validate_foreign_key :emails, :user_id end diff --git a/doc/development/database/adding_database_indexes.md b/doc/development/database/adding_database_indexes.md index d4cd807ef22..6a401c804f5 100644 --- a/doc/development/database/adding_database_indexes.md +++ b/doc/development/database/adding_database_indexes.md @@ -107,11 +107,10 @@ determining whether existing indexes are still required. More information on the meaning of the various columns can be found at <https://www.postgresql.org/docs/current/monitoring-stats.html>. -To determine if an index is still being used on production, use the following -Thanos query with your index name: +To determine if an index is still being used on production, use [Thanos](https://thanos-query.ops.gitlab.net/graph?g0.expr=sum%20by%20(type)(rate(pg_stat_user_indexes_idx_scan%7Benv%3D%22gprd%22%2C%20indexrelname%3D%22INSERT%20INDEX%20NAME%20HERE%22%7D%5B30d%5D))&g0.tab=1&g0.stacked=0&g0.range_input=1h&g0.max_source_resolution=0s&g0.deduplicate=1&g0.partial_response=0&g0.store_matches=%5B%5D): ```sql -sum(rate(pg_stat_user_indexes_idx_tup_read{env="gprd", indexrelname="index_ci_name", type="patroni-ci"}[5m])) +sum by (type)(rate(pg_stat_user_indexes_idx_scan{env="gprd", indexrelname="INSERT INDEX NAME HERE"}[30d])) ``` Because the query output relies on the actual usage of your database, it @@ -232,7 +231,7 @@ A Rails migration example: ```ruby # in db/post_migrate/ -class AddIndexToPartitionedTable < Gitlab::Database::Migration[2.0] +class AddIndexToPartitionedTable < Gitlab::Database::Migration[2.1] include Gitlab::Database::PartitioningMigrationHelpers disable_ddl_transaction! @@ -355,7 +354,7 @@ Use the asynchronous index helpers on your local environment to test changes for For very large tables, index destruction can be a challenge to manage. While `remove_concurrent_index` removes indexes in a way that does not block normal traffic, it can still be problematic if index destruction runs for -during autovacuum. Necessary database operations like `autovacuum` cannot run, and +during `autovacuum`. Necessary database operations like `autovacuum` cannot run, and the deployment process on GitLab.com is blocked while waiting for index destruction to finish. diff --git a/doc/development/database/avoiding_downtime_in_migrations.md b/doc/development/database/avoiding_downtime_in_migrations.md index 57f5a66a9ee..b34c0bbf728 100644 --- a/doc/development/database/avoiding_downtime_in_migrations.md +++ b/doc/development/database/avoiding_downtime_in_migrations.md @@ -82,7 +82,7 @@ to write a migration that removes a column: In this case, a **transactional migration** can be used. Something as simple as: ```ruby -class RemoveUsersUpdatedAtColumn < Gitlab::Database::Migration[2.0] +class RemoveUsersUpdatedAtColumn < Gitlab::Database::Migration[2.1] def up remove_column :users, :updated_at end @@ -103,7 +103,7 @@ If the `down` method requires adding back any dropped indexes or constraints, th be done within a transactional migration, then the migration would look like this: ```ruby -class RemoveUsersUpdatedAtColumn < Gitlab::Database::Migration[2.0] +class RemoveUsersUpdatedAtColumn < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -158,7 +158,7 @@ renaming. For example ```ruby # A regular migration in db/migrate -class RenameUsersUpdatedAtToUpdatedAtTimestamp < Gitlab::Database::Migration[2.0] +class RenameUsersUpdatedAtToUpdatedAtTimestamp < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -186,7 +186,7 @@ We can perform this cleanup using ```ruby # A post-deployment migration in db/post_migrate -class CleanupUsersUpdatedAtRename < Gitlab::Database::Migration[2.0] +class CleanupUsersUpdatedAtRename < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -233,7 +233,7 @@ as follows: ```ruby # A regular migration in db/migrate -class ChangeUsersUsernameStringToText < Gitlab::Database::Migration[2.0] +class ChangeUsersUsernameStringToText < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -252,7 +252,7 @@ Next we need to clean up our changes using a post-deployment migration: ```ruby # A post-deployment migration in db/post_migrate -class ChangeUsersUsernameStringToTextCleanup < Gitlab::Database::Migration[2.0] +class ChangeUsersUsernameStringToTextCleanup < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -319,6 +319,11 @@ This operation is safe as there's no code using the table just yet. Dropping tables can be done safely using a post-deployment migration, but only if the application no longer uses the table. +Add the table to `DELETED_TABLES` in +[gitlab_schema.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/database/gitlab_schema.rb), +along with its `gitlab_schema`. Even though the table is deleted, it is still +referenced in database migrations. + ## Renaming Tables Renaming tables requires downtime as an application may continue @@ -418,7 +423,7 @@ Check how the migration is performing while it's running. Multiple ways to do th #### High-level status of batched background migrations -See how to [check the status of batched background migrations](../../update/index.md#checking-for-background-migrations-before-upgrading). +See how to [check the status of batched background migrations](../../update/background_migrations.md). #### Query the database @@ -478,7 +483,7 @@ for batched background migration: To monitor the health of the database, use these additional metrics: -- [PostgreSQL Tuple Statistics](https://dashboards.gitlab.net/d/000000167/postgresql-tuple-statistics?orgId=1&refresh=1m): if you see high rate of updates for the tables being actively converted, or increasing percentage of dead tuples for this table, it might mean that autovacuum cannot keep up. +- [PostgreSQL Tuple Statistics](https://dashboards.gitlab.net/d/000000167/postgresql-tuple-statistics?orgId=1&refresh=1m): if you see high rate of updates for the tables being actively converted, or increasing percentage of dead tuples for this table, it might mean that `autovacuum` cannot keep up. - [PostgreSQL Overview](https://dashboards.gitlab.net/d/000000144/postgresql-overview?orgId=1): if you see high system usage or transactions per second (TPS) on the primary database server, it might mean that the migration is causing problems. ### Prometheus metrics @@ -499,8 +504,8 @@ If the migration has not completed, the subsequent steps fail anyway. By checkin aim to have more helpful error message. 1. Create indexes using the `bigint` columns that match the existing indexes using the `integer` column ([see an example](https://gitlab.com/gitlab-org/gitlab/-/blob/41fbe34a4725a4e357a83fda66afb382828767b2/db/post_migrate/20210707210916_finalize_ci_stages_bigint_conversion.rb#L28-34)). -1. Create foreign keys (FK) using the `bigint` columns that match the existing FKs using the -`integer` column. Do this both for FK referencing other tables, and FKs that reference the table +1. Create foreign keys (FK) using the `bigint` columns that match the existing FK using the +`integer` column. Do this both for FK referencing other tables, and FK that reference the table that is being migrated ([see an example](https://gitlab.com/gitlab-org/gitlab/-/blob/41fbe34a4725a4e357a83fda66afb382828767b2/db/post_migrate/20210707210916_finalize_ci_stages_bigint_conversion.rb#L36-43)). 1. Inside a transaction, swap the columns: 1. Lock the tables involved. To reduce the chance of hitting a deadlock, we recommended to do this in parent to child order ([see an example](https://gitlab.com/gitlab-org/gitlab/-/blob/41fbe34a4725a4e357a83fda66afb382828767b2/db/post_migrate/20210707210916_finalize_ci_stages_bigint_conversion.rb#L47)). @@ -509,7 +514,7 @@ that is being migrated ([see an example](https://gitlab.com/gitlab-org/gitlab/-/ 1. Swap the defaults ([see an example](https://gitlab.com/gitlab-org/gitlab/-/blob/41fbe34a4725a4e357a83fda66afb382828767b2/db/post_migrate/20210707210916_finalize_ci_stages_bigint_conversion.rb#L59-62)). 1. Swap the PK constraint (if any) ([see an example](https://gitlab.com/gitlab-org/gitlab/-/blob/41fbe34a4725a4e357a83fda66afb382828767b2/db/post_migrate/20210707210916_finalize_ci_stages_bigint_conversion.rb#L64-68)). 1. Remove old indexes and rename new ones ([see an example](https://gitlab.com/gitlab-org/gitlab/-/blob/41fbe34a4725a4e357a83fda66afb382828767b2/db/post_migrate/20210707210916_finalize_ci_stages_bigint_conversion.rb#L70-72)). - 1. Remove old FKs (if still present) and rename new ones ([see an example](https://gitlab.com/gitlab-org/gitlab/-/blob/41fbe34a4725a4e357a83fda66afb382828767b2/db/post_migrate/20210707210916_finalize_ci_stages_bigint_conversion.rb#L74)). + 1. Remove old foreign keys (if still present) and rename new ones ([see an example](https://gitlab.com/gitlab-org/gitlab/-/blob/41fbe34a4725a4e357a83fda66afb382828767b2/db/post_migrate/20210707210916_finalize_ci_stages_bigint_conversion.rb#L74)). See example [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66088), and [migration](https://gitlab.com/gitlab-org/gitlab/-/blob/41fbe34a4725a4e357a83fda66afb382828767b2/db/post_migrate/20210707210916_finalize_ci_stages_bigint_conversion.rb). diff --git a/doc/development/database/background_migrations.md b/doc/development/database/background_migrations.md index fe62bbc6b14..457694c7abd 100644 --- a/doc/development/database/background_migrations.md +++ b/doc/development/database/background_migrations.md @@ -236,7 +236,7 @@ Next we need a post-deployment migration that schedules the migration for existing data. ```ruby -class ScheduleExtractIntegrationsUrl < Gitlab::Database::Migration[2.0] +class ScheduleExtractIntegrationsUrl < Gitlab::Database::Migration[2.1] disable_ddl_transaction! MIGRATION = 'ExtractIntegrationsUrl' @@ -263,7 +263,7 @@ jobs and manually run on any un-migrated rows. Such a migration would look like this: ```ruby -class ConsumeRemainingExtractIntegrationsUrlJobs < Gitlab::Database::Migration[2.0] +class ConsumeRemainingExtractIntegrationsUrlJobs < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -412,7 +412,7 @@ When looking at the batch execution time versus the delay time, the execution ti should fit comfortably within the delay time for a few reasons: - To allow for a variance in query times. -- To allow autovacuum to catch up after periods of high churn. +- To allow `autovacuum` to catch up after periods of high churn. Never try to optimize by fully filling the delay window even if you are confident the queries themselves have no timing variance. diff --git a/doc/development/database/batched_background_migrations.md b/doc/development/database/batched_background_migrations.md index ca11e9c8dd3..71df4da59c3 100644 --- a/doc/development/database/batched_background_migrations.md +++ b/doc/development/database/batched_background_migrations.md @@ -285,7 +285,7 @@ In the second (filtered) example, we know exactly 100 will be updated with each 1. In the post-deployment migration, enqueue the batched background migration: ```ruby - class BackfillNamespaceType < Gitlab::Database::Migration[2.0] + class BackfillNamespaceType < Gitlab::Database::Migration[2.1] MIGRATION = 'BackfillNamespaceType' DELAY_INTERVAL = 2.minutes @@ -308,7 +308,7 @@ In the second (filtered) example, we know exactly 100 will be updated with each NOTE: When applying additional filters, it is important to ensure they are properly covered by an index to optimize `EachBatch` performance. -In the example above we need an index on `(type, id)` to support the filters. See [the `EachBatch` docs for more information](../iterating_tables_in_batches.md). +In the example above we need an index on `(type, id)` to support the filters. See [the `EachBatch` docs for more information](iterating_tables_in_batches.md). ## Example @@ -366,7 +366,7 @@ background migration. 1. Create a post-deployment migration that queues the migration for existing data: ```ruby - class QueueBackfillRoutesNamespaceId < Gitlab::Database::Migration[2.0] + class QueueBackfillRoutesNamespaceId < Gitlab::Database::Migration[2.1] MIGRATION = 'BackfillRouteNamespaceId' DELAY_INTERVAL = 2.minutes @@ -403,7 +403,7 @@ background migration. that checks that the batched background migration is completed. For example: ```ruby - class FinalizeBackfillRouteNamespaceId < Gitlab::Database::Migration[2.0] + class FinalizeBackfillRouteNamespaceId < Gitlab::Database::Migration[2.1] MIGRATION = 'BackfillRouteNamespaceId' disable_ddl_transaction! @@ -452,7 +452,7 @@ the batching column. Database post-migration: ```ruby -class ProjectsWithIssuesMigration < Gitlab::Database::Migration[2.0] +class ProjectsWithIssuesMigration < Gitlab::Database::Migration[2.1] MIGRATION = 'BatchProjectsWithIssues' INTERVAL = 2.minutes BATCH_SIZE = 5000 diff --git a/doc/development/database/creating_enums.md b/doc/development/database/creating_enums.md index 73c3f546728..e2ae36f7481 100644 --- a/doc/development/database/creating_enums.md +++ b/doc/development/database/creating_enums.md @@ -79,7 +79,7 @@ This works as-is, however, it has a couple of downside that: - When it happens, we have to ship a database migration to fix the data integrity, which might be impossible if you cannot recover the original value. -Also, you might observe a workaround for this concern by setting an offset in EE's values. +Also, you might observe a workaround for this concern by setting an offset in the `EE` module's values. For example, this example sets `1000` as the offset: ```ruby diff --git a/doc/development/database/database_debugging.md b/doc/development/database/database_debugging.md index 0d6e9955a19..edc35dd95e8 100644 --- a/doc/development/database/database_debugging.md +++ b/doc/development/database/database_debugging.md @@ -53,7 +53,7 @@ bundle exec rake db:reset RAILS_ENV=test - `bundle exec rake db:migrate:up:main VERSION=20170926203418 RAILS_ENV=development`: Set up a migration - `bundle exec rake db:migrate:redo:main VERSION=20170926203418 RAILS_ENV=development`: Re-run a specific migration -Replace `main` in the above commands to execute agains the `ci` database instead of `main`. +Replace `main` in the above commands to execute against the `ci` database instead of `main`. ## Manually access the database @@ -107,7 +107,7 @@ The new connection should be working now. Use these instructions for exploring the GitLab database while developing with the GDK: 1. Install or open [Visual Studio Code](https://code.visualstudio.com/download). -1. Install the [PostgreSQL VSCode Extension](https://marketplace.visualstudio.com/items?itemName=ckolkman.vscode-postgres). +1. Install the [PostgreSQL VS Code Extension](https://marketplace.visualstudio.com/items?itemName=ckolkman.vscode-postgres). 1. In Visual Studio Code select **PostgreSQL Explorer** in the left toolbar. 1. In the top bar of the new window, select `+` to **Add Database Connection**, and follow the prompts to fill in the details: 1. **Hostname**: the path to the PostgreSQL folder in your GDK directory (for example `/dev/gitlab-development-kit/postgresql`). diff --git a/doc/development/database/database_dictionary.md b/doc/development/database/database_dictionary.md index bd6dbc54316..d74d7e77edb 100644 --- a/doc/development/database/database_dictionary.md +++ b/doc/development/database/database_dictionary.md @@ -11,7 +11,8 @@ locate the feature categories responsible for specific database tables. ## Location -Database dictionary metadata files are stored in the `gitlab` project under `db/docs/`. +Database dictionary metadata files are stored in the `gitlab` project under `db/docs/` for the `main` and `ci` databases. +For the `geo` database, the dictionary files are stored under `ee/db/docs/`. ## Example dictionary file @@ -29,23 +30,43 @@ milestone: '13.0' ## Schema -| Attribute | Type | Required | Description | -|----------------------|---------------|----------|--------------------------------------------------------------------------| -| `table_name` | String | yes | Database table name | -| `classes` | Array(String) | no | List of classes that respond to `.table_name` with the `table_name` | -| `feature_categories` | Array(String) | yes | List of feature categories using this table | -| `description` | String | no | Text description of the information stored in the table and it's purpose | -| `introduced_by_url` | URL | no | URL to the merge request or commit which introduced this table | -| `milestone` | String | no | The milestone that introduced this table | +| Attribute | Type | Required | Description | +|----------------------------|---------------|----------|-----------------------------------------------------------------------------------| +| `table_name` / `view_name` | String | yes | Database table name or view name | +| `classes` | Array(String) | no | List of classes that are associated to this table or view. | +| `feature_categories` | Array(String) | yes | List of feature categories using this table or view. | +| `description` | String | no | Text description of the information stored in the table or view, and its purpose. | +| `introduced_by_url` | URL | no | URL to the merge request or commit which introduced this table or view. | +| `milestone` | String | no | The milestone that introduced this table or view. | +| `gitlab_schema` | String | yes | GitLab schema name. | ## Adding tables -When adding a new table, create a new file under `db/docs/` named -`<table_name>.yml` containing as much information as you know about the table. +When adding a new table, create a new file under `db/docs/` for the `main` and `ci` databases. +For the `geo` database use `ee/db/docs/`. +Name the file as `<table_name>.yml`, containing as much information as you know about the table. Include this file in the commit with the migration that creates the table. ## Dropping tables -When dropping a table, you must remove the metadata file from `db/docs/` -in the same commit with the migration that drops the table. +When dropping a table, you must remove the metadata file from `db/docs/` for `main` and `ci` databases. +For the `geo` database, you must remove the file from `ee/db/docs/`. +Use the same commit with the migration that drops the table. + +## Adding views + +When adding a new view, you should: + +1. Create a new file for this view in the appropriate directory: + - `main` database: `db/docs/views/` + - `ci` database: `db/docs/views/` + - `geo` database: `ee/db/docs/views/` +1. Name the file `<view_name>.yml`, and include as much information as you know about the view. +1. Include this file in the commit with the migration that creates the view. + +## Dropping views + +When dropping a view, you must remove the metadata file from `db/docs/views/`. +For the `geo` database, you must remove the file from `ee/db/docs/views/`. +Use the same commit with the migration that drops the view. diff --git a/doc/development/database/efficient_in_operator_queries.md b/doc/development/database/efficient_in_operator_queries.md index fb7ff3c1cc2..78b310ae708 100644 --- a/doc/development/database/efficient_in_operator_queries.md +++ b/doc/development/database/efficient_in_operator_queries.md @@ -126,8 +126,7 @@ For very large groups the database queries can easily time out, causing HTTP 500 ## Optimizing ordered `IN` queries -In the talk -["How to teach an elephant to dance rock'n'roll"](https://www.youtube.com/watch?v=Ha38lcjVyhQ), +In the talk ["How to teach an elephant to dance rock and roll"](https://www.youtube.com/watch?v=Ha38lcjVyhQ), Maxim Boguk demonstrated a technique to optimize a special class of ordered `IN` queries, such as our ordered group-level queries. @@ -160,7 +159,7 @@ The technique can only optimize `IN` queries that satisfy the following requirem in the following order: `column_for_the_in_query`, `order by column 1`, and `order by column 2`. - The columns in the `ORDER BY` clause are distinct - (the combination of the columns uniquely identifies one particular column in the table). + (the combination of the columns uniquely identifies one particular row in the table). WARNING: This technique does not improve the performance of the `COUNT(*)` queries. @@ -918,11 +917,11 @@ the `LIMIT` is reached or no more data can be found. Here's an outline of the steps we take in the recursive CTE query (expressing the steps in SQL is non-trivial but is explained next): -1. Sort the initial resultset according to the `ORDER BY` clause. +1. Sort the initial `resultset` according to the `ORDER BY` clause. 1. Pick the top cursor to fetch the record, this is our first record. In the example, this cursor would be (`2020-01-05`, `3`) for `project_id=9`. 1. We can use (`2020-01-05`, `3`) to fetch the next issue respecting the `ORDER BY` clause -`project_id=9` filter. This produces an updated resultset. +`project_id=9` filter. This produces an updated `resultset`. | `project_ids` | `created_at_values` | `id_values` | | ------------- | ------------------- | ----------- | @@ -931,7 +930,7 @@ this cursor would be (`2020-01-05`, `3`) for `project_id=9`. | 10 | 2020-01-15 | 7 | | **9** | **2020-01-06** | **6** | -1. Repeat 1 to 3 with the updated resultset until we have fetched `N=20` records. +1. Repeat 1 to 3 with the updated `resultset` until we have fetched `N=20` records. ### Initializing the recursive CTE query diff --git a/doc/development/database/foreign_keys.md b/doc/development/database/foreign_keys.md index d9506ae614a..25b3d815d7a 100644 --- a/doc/development/database/foreign_keys.md +++ b/doc/development/database/foreign_keys.md @@ -49,7 +49,7 @@ To replace a foreign key: foreign key before removing the old one. ```ruby - class ReplaceFkOnPackagesPackagesProjectId < Gitlab::Database::Migration[2.0] + class ReplaceFkOnPackagesPackagesProjectId < Gitlab::Database::Migration[2.1] disable_ddl_transaction! NEW_CONSTRAINT_NAME = 'fk_new' @@ -69,7 +69,7 @@ To replace a foreign key: 1. [Validate the new foreign key](add_foreign_key_to_existing_column.md#validate-the-foreign-key) ```ruby - class ValidateFkNew < Gitlab::Database::Migration[2.0] + class ValidateFkNew < Gitlab::Database::Migration[2.1] NEW_CONSTRAINT_NAME = 'fk_new' # foreign key added in <link to MR or path to migration adding new FK> @@ -86,7 +86,7 @@ To replace a foreign key: 1. Remove the old foreign key: ```ruby - class RemoveFkOld < Gitlab::Database::Migration[2.0] + class RemoveFkOld < Gitlab::Database::Migration[2.1] OLD_CONSTRAINT_NAME = 'fk_old' # new foreign key added in <link to MR or path to migration adding new FK> diff --git a/doc/development/database/index.md b/doc/development/database/index.md index 87b1b4a9d87..c244d784422 100644 --- a/doc/development/database/index.md +++ b/doc/development/database/index.md @@ -75,12 +75,24 @@ info: To determine the technical writer assigned to the Stage/Group associated w - [Pagination performance guidelines](pagination_performance_guidelines.md) - [Efficient `IN` operator queries](efficient_in_operator_queries.md) - [Data layout and access patterns](layout_and_access_patterns.md) +- [Check for background migrations before upgrading](../../update/background_migrations.md) ## Case studies - [Database case study: Filtering by label](filtering_by_label.md) - [Database case study: Namespaces storage statistics](namespaces_storage_statistics.md) +## PostgreSQL information for GitLab administrators + +- [Configure GitLab using an external PostgreSQL service](../../administration/postgresql/external.md) +- [Configuring PostgreSQL for scaling](../../administration/postgresql/index.md) +- [Database Load Balancing](../../administration/postgresql/database_load_balancing.md) +- [Moving GitLab databases to a different PostgreSQL instance](../../administration/postgresql/moving.md) +- [Replication and failover with Omnibus GitLab](../../administration/postgresql/replication_and_failover.md) +- [Standalone PostgreSQL using Omnibus GitLab](../../administration/postgresql/standalone.md) +- [Troubleshooting PostgreSQL](../../administration/troubleshooting/postgresql.md) +- [Working with the bundled PgBouncer service](../../administration/postgresql/pgbouncer.md) + ## Miscellaneous - [Maintenance operations](maintenance_operations.md) diff --git a/doc/development/database/keyset_pagination.md b/doc/development/database/keyset_pagination.md index 21bce41012e..42d7458b45a 100644 --- a/doc/development/database/keyset_pagination.md +++ b/doc/development/database/keyset_pagination.md @@ -159,7 +159,7 @@ configuration is necessary: - Function-based ordering. - Ordering with a custom tie-breaker column, like `iid`. -These order objects can be defined in the model classes as normal ActiveRecord scopes, there is no special behavior that prevents using these scopes elsewhere (kaminari, background jobs). +These order objects can be defined in the model classes as normal ActiveRecord scopes, there is no special behavior that prevents using these scopes elsewhere (Kaminari, background jobs). ### `NULLS LAST` ordering diff --git a/doc/development/database/loose_foreign_keys.md b/doc/development/database/loose_foreign_keys.md index 962cd2602bc..daa022a3de2 100644 --- a/doc/development/database/loose_foreign_keys.md +++ b/doc/development/database/loose_foreign_keys.md @@ -134,7 +134,7 @@ scripts/decomposition/generate-loose-foreign-key -c ci_job_token_project_scope_l ``` To swap all the foreign keys (all having `_id` appended), but not create a new branch (only commit -the changes) and not create rspecs, run: +the changes) and not create RSpec tests, run: ```shell scripts/decomposition/generate-loose-foreign-key -c --no-branch --no-rspec _id @@ -192,7 +192,7 @@ trigger needs to be configured only once. If the model already has at least one `loose_foreign_key` definition, then this step can be skipped: ```ruby -class TrackProjectRecordChanges < Gitlab::Database::Migration[2.0] +class TrackProjectRecordChanges < Gitlab::Database::Migration[2.1] include Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers enable_lock_retries! @@ -227,7 +227,7 @@ trigger. If the foreign key is deleted earlier, there is a good chance of introducing data inconsistency which needs manual cleanup: ```ruby -class RemoveProjectsCiPipelineFk < Gitlab::Database::Migration[2.0] +class RemoveProjectsCiPipelineFk < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -258,7 +258,7 @@ records in the database. Migration for removing the trigger: ```ruby -class UnTrackProjectRecordChanges < Gitlab::Database::Migration[2.0] +class UnTrackProjectRecordChanges < Gitlab::Database::Migration[2.1] include Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers enable_lock_retries! @@ -278,7 +278,7 @@ table however, there is still a chance for having leftover pending records in th must be removed with an inline data migration. ```ruby -class RemoveLeftoverProjectDeletions < Gitlab::Database::Migration[2.0] +class RemoveLeftoverProjectDeletions < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up diff --git a/doc/development/database/migrations_for_multiple_databases.md b/doc/development/database/migrations_for_multiple_databases.md index b4d2656121b..bc0ef654336 100644 --- a/doc/development/database/migrations_for_multiple_databases.md +++ b/doc/development/database/migrations_for_multiple_databases.md @@ -58,7 +58,7 @@ Example migration adding a concurrent index that is treated as change of the str that is executed on all configured databases. ```ruby -class AddUserIdAndStateIndexToMergeRequestReviewers < Gitlab::Database::Migration[2.0] +class AddUserIdAndStateIndexToMergeRequestReviewers < Gitlab::Database::Migration[2.1] disable_ddl_transaction! INDEX_NAME = 'index_on_merge_request_reviewers_user_id_and_state' @@ -75,16 +75,24 @@ end #### Example: Add a new table to store in a single database -1. Define the [GitLab Schema](multiple_databases.md#gitlab-schema) of the table in [`lib/gitlab/database/gitlab_schemas.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/database/gitlab_schemas.yml): +1. Add the table to the [database dictionary](database_dictionary.md) in [`db/docs/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/db/docs): ```yaml - ssh_signatures: :gitlab_main + table_name: ssh_signatures + description: Description example + introduced_by_url: Merge request link + milestone: Milestone example + feature_categories: + - Feature category example + classes: + - Class example + gitlab_schema: gitlab_main ``` 1. Create the table in a schema migration: ```ruby - class CreateSshSignatures < Gitlab::Database::Migration[2.0] + class CreateSshSignatures < Gitlab::Database::Migration[2.1] def change create_table :ssh_signatures do |t| t.timestamps_with_timezone null: false @@ -125,7 +133,7 @@ Example migration updating `archived` column of `projects` that is executed only for the database containing `gitlab_main` schema. ```ruby -class UpdateProjectsArchivedState < Gitlab::Database::Migration[2.0] +class UpdateProjectsArchivedState < Gitlab::Database::Migration[2.1] disable_ddl_transaction! restrict_gitlab_migration gitlab_schema: :gitlab_main @@ -157,7 +165,7 @@ databases. For example, running migration in context of `ci:` and reading featur from `main:`, as no established connection to another database is present. ```ruby -class UpdateProjectsArchivedState < Gitlab::Database::Migration[2.0] +class UpdateProjectsArchivedState < Gitlab::Database::Migration[2.1] disable_ddl_transaction! restrict_gitlab_migration gitlab_schema: :gitlab_main @@ -195,7 +203,7 @@ that is marked in `lib/gitlab/database/gitlab_schemas.yml` as `gitlab_shared`. This migration is executed across all configured databases. ```ruby -class DeleteAllLooseForeignKeyRecords < Gitlab::Database::Migration[2.0] +class DeleteAllLooseForeignKeyRecords < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -211,13 +219,13 @@ end #### Example: run DML `gitlab_shared` only on the database containing the given `gitlab_schema` Example migration updating `loose_foreign_keys_deleted_records` table -that is marked in `lib/gitlab/database/gitlab_schemas.yml` as `gitlab_shared`. +that is marked in `db/docs/loose_foreign_keys_deleted_records.yml` as `gitlab_shared`. This migration since it configures restriction on `gitlab_ci` is executed only in context of database containing `gitlab_ci` schema. ```ruby -class DeleteCiBuildsLooseForeignKeyRecords < Gitlab::Database::Migration[2.0] +class DeleteCiBuildsLooseForeignKeyRecords < Gitlab::Database::Migration[2.1] disable_ddl_transaction! restrict_gitlab_migration gitlab_schema: :gitlab_ci @@ -261,7 +269,7 @@ the `database_tasks: false` set. `gitlab:db:validate_config` always runs before ## Validation Validation in a nutshell uses [`pg_query`](https://github.com/pganalyze/pg_query) to analyze -each query and classify tables with information from [`gitlab_schema.yml`](multiple_databases.md#gitlab-schema). +each query and classify tables with information from [`db/docs/`](database_dictionary.md). The migration is skipped if the specified `gitlab_schema` is outside of a list of schemas managed by a given database connection (`Gitlab::Database::gitlab_schemas_for_connection`). @@ -279,7 +287,7 @@ as part of the migration run and prevent the migration from being completed. ### Exception 1: migration running in DDL mode does DML select ```ruby -class UpdateProjectsArchivedState < Gitlab::Database::Migration[2.0] +class UpdateProjectsArchivedState < Gitlab::Database::Migration[2.1] disable_ddl_transaction! # Missing: @@ -310,7 +318,7 @@ running in **DDL** mode, but the executed payload appears to be reading data fro ### Exception 2: migration running in DML mode changes the structure ```ruby -class AddUserIdAndStateIndexToMergeRequestReviewers < Gitlab::Database::Migration[2.0] +class AddUserIdAndStateIndexToMergeRequestReviewers < Gitlab::Database::Migration[2.1] disable_ddl_transaction! # restrict_gitlab_migration if defined indicates DML, it should be removed @@ -341,7 +349,7 @@ but the executed payload appears to be doing structure changes (DDL). ### Exception 3: migration running in DML mode accesses data from a table in another schema ```ruby -class UpdateProjectsArchivedState < Gitlab::Database::Migration[2.0] +class UpdateProjectsArchivedState < Gitlab::Database::Migration[2.1] disable_ddl_transaction! # Since it modifies `projects` it should use `gitlab_main` @@ -372,7 +380,7 @@ data in `gitlab_main`. ### Exception 4: mixing DDL and DML mode ```ruby -class UpdateProjectsArchivedState < Gitlab::Database::Migration[2.0] +class UpdateProjectsArchivedState < Gitlab::Database::Migration[2.1] disable_ddl_transaction! # This migration is invalid regardless of specification @@ -435,4 +443,4 @@ tables in any database, just like any ordinary Sidekiq worker can. ## How to determine `gitlab_schema` for a given table -See [GitLab Schema](multiple_databases.md#gitlab-schema). +See [database dictionary](database_dictionary.md). diff --git a/doc/development/database/multiple_databases.md b/doc/development/database/multiple_databases.md index e5b6cfb8866..d22e3209096 100644 --- a/doc/development/database/multiple_databases.md +++ b/doc/development/database/multiple_databases.md @@ -14,9 +14,9 @@ On GitLab.com we are using two separate databases. ## GitLab Schema For properly discovering allowed patterns between different databases -the GitLab application implements the `lib/gitlab/database/gitlab_schemas.yml` YAML file. +the GitLab application implements the [database dictionary](database_dictionary.md). -This file provides a virtual classification of tables into a `gitlab_schema` +The database dictionary provides a virtual classification of tables into a `gitlab_schema` which conceptually is similar to [PostgreSQL Schema](https://www.postgresql.org/docs/current/ddl-schemas.html). We decided as part of [using database schemas to better isolated CI decomposed features](https://gitlab.com/gitlab-org/gitlab/-/issues/333415) that we cannot use PostgreSQL schema due to complex migration procedures. Instead we implemented diff --git a/doc/development/database/not_null_constraints.md b/doc/development/database/not_null_constraints.md index 53ab9a83d60..77fa23bbb19 100644 --- a/doc/development/database/not_null_constraints.md +++ b/doc/development/database/not_null_constraints.md @@ -25,7 +25,7 @@ For example, consider a migration that creates a table with two `NOT NULL` colum `db/migrate/20200401000001_create_db_guides.rb`: ```ruby -class CreateDbGuides < Gitlab::Database::Migration[2.0] +class CreateDbGuides < Gitlab::Database::Migration[2.1] def change create_table :db_guides do |t| t.bigint :stars, default: 0, null: false @@ -44,7 +44,7 @@ For example, consider a migration that adds a new `NOT NULL` column `active` to `db/migrate/20200501000001_add_active_to_db_guides.rb`: ```ruby -class AddExtendedTitleToSprints < Gitlab::Database::Migration[2.0] +class AddExtendedTitleToSprints < Gitlab::Database::Migration[2.1] def change add_column :db_guides, :active, :boolean, default: true, null: false end @@ -116,7 +116,7 @@ with `validate: false` in a post-deployment migration, `db/post_migrate/20200501000001_add_not_null_constraint_to_epics_description.rb`: ```ruby -class AddNotNullConstraintToEpicsDescription < Gitlab::Database::Migration[2.0] +class AddNotNullConstraintToEpicsDescription < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -147,7 +147,7 @@ so we add a post-deployment migration for the 13.0 milestone (current), `db/post_migrate/20200501000002_cleanup_epics_with_null_description.rb`: ```ruby -class CleanupEpicsWithNullDescription < Gitlab::Database::Migration[2.0] +class CleanupEpicsWithNullDescription < Gitlab::Database::Migration[2.1] # With BATCH_SIZE=1000 and epics.count=29500 on GitLab.com # - 30 iterations will be run # - each requires on average ~150ms @@ -185,7 +185,7 @@ migration helper in a final post-deployment migration, `db/post_migrate/20200601000001_validate_not_null_constraint_on_epics_description.rb`: ```ruby -class ValidateNotNullConstraintOnEpicsDescription < Gitlab::Database::Migration[2.0] +class ValidateNotNullConstraintOnEpicsDescription < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up diff --git a/doc/development/database/pagination_performance_guidelines.md b/doc/development/database/pagination_performance_guidelines.md index 0f98b50d95c..b06839979da 100644 --- a/doc/development/database/pagination_performance_guidelines.md +++ b/doc/development/database/pagination_performance_guidelines.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Pagination performance guidelines -The following document gives a few ideas for improving the pagination (sorting) performance. These apply both on [offset](pagination_guidelines.md#offset-pagination) and [keyset](pagination_guidelines.md#keyset-pagination) paginations. +The following document gives a few ideas for improving the pagination (sorting) performance. These apply both on [offset](pagination_guidelines.md#offset-pagination) and [keyset](pagination_guidelines.md#keyset-pagination) pagination. ## Tie-breaker column diff --git a/doc/development/database/polymorphic_associations.md b/doc/development/database/polymorphic_associations.md index f3c9bf1276f..bb7eb46b448 100644 --- a/doc/development/database/polymorphic_associations.md +++ b/doc/development/database/polymorphic_associations.md @@ -98,10 +98,10 @@ AND source_id = 4 Instead such a table should be broken up into separate tables. For example, you may end up with 4 tables in this case: -- project_members -- group_members -- pending_project_members -- pending_group_members +- `project_members` +- `group_members` +- `pending_project_members` +- `pending_group_members` This makes querying data trivial. For example, to get the members of a group you'd run: diff --git a/doc/development/database/query_performance.md b/doc/development/database/query_performance.md index 61fd80338fe..73a6a40f801 100644 --- a/doc/development/database/query_performance.md +++ b/doc/development/database/query_performance.md @@ -15,13 +15,14 @@ When you are optimizing your SQL queries, there are two dimensions to pay attent ## Timing guidelines for queries -| Query Type | Maximum Query Time | Notes | -|----|----|---| -| General queries | `100ms` | This is not a hard limit, but if a query is getting above it, it is important to spend time understanding why it can or cannot be optimized. | -| Queries in a migration | `100ms` | This is different than the total [migration time](../migration_style_guide.md#how-long-a-migration-should-take). | -| Concurrent operations in a migration | `5min` | Concurrent operations do not block the database, but they block the GitLab update. This includes operations such as `add_concurrent_index` and `add_concurrent_foreign_key`. | -| Background migrations | `1s` | | -| Service Ping | `1s` | See the [Service Ping docs](../service_ping/implement.md) for more details. | +| Query Type | Maximum Query Time | Notes | +|-------------------------------------------|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| General queries | `100ms` | This is not a hard limit, but if a query is getting above it, it is important to spend time understanding why it can or cannot be optimized. | +| Queries in a migration | `100ms` | This is different than the total [migration time](../migration_style_guide.md#how-long-a-migration-should-take). | +| Concurrent operations in a migration | `5min` | Concurrent operations do not block the database, but they block the GitLab update. This includes operations such as `add_concurrent_index` and `add_concurrent_foreign_key`. | +| Concurrent operations in a post migration | `20min` | Concurrent operations do not block the database, but they block the GitLab post update process. This includes operations such as `add_concurrent_index` and `add_concurrent_foreign_key`. If index creation exceeds 20 minutes, consider [async index creation](adding_database_indexes.md#create-indexes-asynchronously). | +| Background migrations | `1s` | | +| Service Ping | `1s` | See the [Service Ping docs](../service_ping/implement.md) for more details. | - When analyzing your query's performance, pay attention to if the time you are seeing is on a [cold or warm cache](#cold-and-warm-cache). These guidelines apply for both cache types. - When working with batched queries, change the range and batch size to see how it effects the query timing and caching. diff --git a/doc/development/database/query_recorder.md b/doc/development/database/query_recorder.md index f1540e7e2ae..84bd0fc938f 100644 --- a/doc/development/database/query_recorder.md +++ b/doc/development/database/query_recorder.md @@ -47,7 +47,7 @@ the longest common prefix, grouping similar queries together. In some cases, N+1 specs have been written to include three requests: first one to warm the cache, second one to establish a control, third one to validate that -ther are no N+1 queries. Rather than make an extra request to warm the cache, prefer two requests +there are no N+1 queries. Rather than make an extra request to warm the cache, prefer two requests (control and test) and configure your test to ignore [cached queries](#cached-queries) in N+1 specs. ## Cached queries diff --git a/doc/development/database/single_table_inheritance.md b/doc/development/database/single_table_inheritance.md index 32de1fdea35..dcf696b85bc 100644 --- a/doc/development/database/single_table_inheritance.md +++ b/doc/development/database/single_table_inheritance.md @@ -31,7 +31,7 @@ could result in loading unexpected code or associations which may cause unintend side effects or failures during upgrades. ```ruby -class SomeMigration < Gitlab::Database::Migration[2.0] +class SomeMigration < Gitlab::Database::Migration[2.1] class Services < MigrationRecord self.table_name = 'services' self.inheritance_column = :_type_disabled @@ -47,7 +47,7 @@ This ensures that the migration loads the columns for the migration in isolation and the helper disables STI by default. ```ruby -class EnqueueSomeBackgroundMigration < Gitlab::Database::Migration[2.0] +class EnqueueSomeBackgroundMigration < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up diff --git a/doc/development/database/strings_and_the_text_data_type.md b/doc/development/database/strings_and_the_text_data_type.md index fb005e51902..47e89c1ce0f 100644 --- a/doc/development/database/strings_and_the_text_data_type.md +++ b/doc/development/database/strings_and_the_text_data_type.md @@ -50,7 +50,7 @@ For example, consider a migration that creates a table with two text columns, `db/migrate/20200401000001_create_db_guides.rb`: ```ruby -class CreateDbGuides < Gitlab::Database::Migration[2.0] +class CreateDbGuides < Gitlab::Database::Migration[2.1] def change create_table :db_guides do |t| t.bigint :stars, default: 0, null: false @@ -74,7 +74,7 @@ For example, consider a migration that adds a new text column `extended_title` t `db/migrate/20200501000001_add_extended_title_to_sprints.rb`: ```ruby -class AddExtendedTitleToSprints < Gitlab::Database::Migration[2.0] +class AddExtendedTitleToSprints < Gitlab::Database::Migration[2.1] # rubocop:disable Migration/AddLimitToTextColumns # limit is added in 20200501000002_add_text_limit_to_sprints_extended_title @@ -89,7 +89,7 @@ A second migration should follow the first one with a limit added to `extended_t `db/migrate/20200501000002_add_text_limit_to_sprints_extended_title.rb`: ```ruby -class AddTextLimitToSprintsExtendedTitle < Gitlab::Database::Migration[2.0] +class AddTextLimitToSprintsExtendedTitle < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -165,7 +165,7 @@ in a post-deployment migration, `db/post_migrate/20200501000001_add_text_limit_migration.rb`: ```ruby -class AddTextLimitMigration < Gitlab::Database::Migration[2.0] +class AddTextLimitMigration < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -196,7 +196,7 @@ to add a background migration for the 13.0 milestone (current), `db/post_migrate/20200501000002_schedule_cap_title_length_on_issues.rb`: ```ruby -class ScheduleCapTitleLengthOnIssues < Gitlab::Database::Migration[2.0] +class ScheduleCapTitleLengthOnIssues < Gitlab::Database::Migration[2.1] # Info on how many records will be affected on GitLab.com # time each batch needs to run on average, etc ... BATCH_SIZE = 5000 @@ -236,7 +236,7 @@ helper in a final post-deployment migration, `db/post_migrate/20200601000001_validate_text_limit_migration.rb`: ```ruby -class ValidateTextLimitMigration < Gitlab::Database::Migration[2.0] +class ValidateTextLimitMigration < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -255,7 +255,7 @@ Increasing text limits on existing database columns can be safely achieved by fi and then dropping the previous limit: ```ruby -class ChangeMaintainerNoteLimitInCiRunner < Gitlab::Database::Migration[2.0] +class ChangeMaintainerNoteLimitInCiRunner < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up diff --git a/doc/development/database/table_partitioning.md b/doc/development/database/table_partitioning.md index ac715b871da..5f1deb77b6c 100644 --- a/doc/development/database/table_partitioning.md +++ b/doc/development/database/table_partitioning.md @@ -173,7 +173,7 @@ An example migration of partitioning the `audit_events` table by its `created_at` column would look like: ```ruby -class PartitionAuditEvents < Gitlab::Database::Migration[2.0] +class PartitionAuditEvents < Gitlab::Database::Migration[2.1] include Gitlab::Database::PartitioningMigrationHelpers def up @@ -200,7 +200,7 @@ into the partitioned copy. Continuing the above example, the migration would look like: ```ruby -class BackfillPartitionAuditEvents < Gitlab::Database::Migration[2.0] +class BackfillPartitionAuditEvents < Gitlab::Database::Migration[2.1] include Gitlab::Database::PartitioningMigrationHelpers def up @@ -233,7 +233,7 @@ failed jobs. Once again, continuing the example, this migration would look like: ```ruby -class CleanupPartitionedAuditEventsBackfill < Gitlab::Database::Migration[2.0] +class CleanupPartitionedAuditEventsBackfill < Gitlab::Database::Migration[2.1] include Gitlab::Database::PartitioningMigrationHelpers def up @@ -273,7 +273,7 @@ Include the partitioning key in the following constraints: Add the partitioning key column. For example, in a rails migration: ```ruby -class AddPartitionNumberForPartitioning < Gitlab::Database::Migration[2.0] +class AddPartitionNumberForPartitioning < Gitlab::Database::Migration[2.1] enable_lock_retries! TABLE_NAME = :table_name @@ -291,7 +291,7 @@ end Add indexes including the partitioning key column. For example, in a rails migration: ```ruby -class PrepareIndexesForPartitioning < Gitlab::Database::Migration[2.0] +class PrepareIndexesForPartitioning < Gitlab::Database::Migration[2.1] disable_ddl_transaction! TABLE_NAME = :table_name @@ -312,7 +312,7 @@ end Swap the primary key including the partitioning key column. For example, in a rails migration: ```ruby -class PreparePrimaryKeyForPartitioning < Gitlab::Database::Migration[2.0] +class PreparePrimaryKeyForPartitioning < Gitlab::Database::Migration[2.1] disable_ddl_transaction! TABLE_NAME = :table_name @@ -347,7 +347,7 @@ end Enforce unique indexes including the partitioning key column. For example, in a rails migration: ```ruby -class PrepareUniqueContraintForPartitioning < Gitlab::Database::Migration[2.0] +class PrepareUniqueContraintForPartitioning < Gitlab::Database::Migration[2.1] disable_ddl_transaction! TABLE_NAME = :table_name @@ -373,7 +373,7 @@ end Enforce foreign keys including the partitioning key column. For example, in a rails migration: ```ruby -class PrepareForeignKeyForPartitioning < Gitlab::Database::Migration[2.0] +class PrepareForeignKeyForPartitioning < Gitlab::Database::Migration[2.1] disable_ddl_transaction! SOURCE_TABLE_NAME = :source_table_name @@ -410,7 +410,7 @@ partition by using the following helpers provided by the database team. For example, using list partitioning in Rails post migrations: ```ruby -class PrepareTableConstraintsForListPartitioning < Gitlab::Database::Migration[2.0] +class PrepareTableConstraintsForListPartitioning < Gitlab::Database::Migration[2.1] include Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers disable_ddl_transaction! @@ -441,7 +441,7 @@ end ``` ```ruby -class ConvertTableToListPartitioning < Gitlab::Database::Migration[2.0] +class ConvertTableToListPartitioning < Gitlab::Database::Migration[2.1] include Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers disable_ddl_transaction! diff --git a/doc/development/database/understanding_explain_plans.md b/doc/development/database/understanding_explain_plans.md index fff9d755e9a..094bd6b346f 100644 --- a/doc/development/database/understanding_explain_plans.md +++ b/doc/development/database/understanding_explain_plans.md @@ -826,4 +826,4 @@ A more extensive guide on understanding query plans can be found in the [presentation](https://public.dalibo.com/exports/conferences/_archives/_2012/201211_explain/understanding_explain.pdf) from [Dalibo.org](https://www.dalibo.com/en/). -Depesz's blog also has a good [section](https://www.depesz.com/tag/unexplainable/) dedicated to query plans. +The Depesz blog also has a good [section](https://www.depesz.com/tag/unexplainable/) dedicated to query plans. diff --git a/doc/development/database_debugging.md b/doc/development/database_debugging.md deleted file mode 100644 index f18830ee7ca..00000000000 --- a/doc/development/database_debugging.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/database_debugging.md' -remove_date: '2022-11-06' ---- - -This document was moved to [another location](database/database_debugging.md). - -<!-- This redirect file can be deleted after <2022-11-06>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/database_query_comments.md b/doc/development/database_query_comments.md deleted file mode 100644 index 7f9def7e567..00000000000 --- a/doc/development/database_query_comments.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/database_query_comments.md' -remove_date: '2022-11-05' ---- - -This document was moved to [another location](database/database_query_comments.md). - -<!-- This redirect file can be deleted after <2022-11-05>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/database_review.md b/doc/development/database_review.md index 7fbc48af91c..e66be062986 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -150,7 +150,7 @@ Include in the MR description: - Write the raw SQL in the MR description. Preferably formatted nicely with [pgFormatter](https://sqlformat.darold.net) or - [paste.depesz.com](https://paste.depesz.com) and using regular quotes + <https://paste.depesz.com> and using regular quotes <!-- vale gitlab.NonStandardQuotes = NO --> (for example, `"projects"."id"`) and avoiding smart quotes (for example, `“projects”.“id”`). <!-- vale gitlab.NonStandardQuotes = YES --> diff --git a/doc/development/db_dump.md b/doc/development/db_dump.md deleted file mode 100644 index c632302329a..00000000000 --- a/doc/development/db_dump.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/db_dump.md' -remove_date: '2022-11-06' ---- - -This document was moved to [another location](database/db_dump.md). - -<!-- This redirect file can be deleted after <2022-11-06>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/directory_structure.md b/doc/development/directory_structure.md index 27384fa559f..34ee86d9ee5 100644 --- a/doc/development/directory_structure.md +++ b/doc/development/directory_structure.md @@ -1,94 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: 'software_design.md' +remove_date: '2023-01-24' --- -# Backend directory structure +This document was moved to [another location](software_design.md) -## Use namespaces to define bounded contexts - -A healthy application is divided into macro and sub components that represent the contexts at play, -whether they are related to business domain or infrastructure code. - -As GitLab code has so many features and components it's hard to see what contexts are involved. -We should expect any class to be defined inside a module/namespace that represents the contexts where it operates. - -When we namespace classes inside their domain: - -- Similar terminology becomes unambiguous as the domain clarifies the meaning: - For example, `MergeRequests::Diff` and `Notes::Diff`. -- Top-level namespaces could be associated to one or more groups identified as domain experts. -- We can better identify the interactions and coupling between components. - For example, several classes inside `MergeRequests::` domain interact more with `Ci::` - domain and less with `ImportExport::`. - -```ruby -# bad -class MyClass -end - -# good -module MyDomain - class MyClass - end -end -``` - -### About namespace naming - -A good guideline for naming a top-level namespace (bounded context) is to use the related -[feature category](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/data/categories.yml). For example, `Continuous Integration` feature category maps to `Ci::` namespace. - -Alternatively a new class could be added to `Projects::` or `Groups::` if it's either: - -- Strictly related to one of these domains. For example `Projects::Alias`. -- A new component that does not have yet a more specific domain. In this case, when - a more explicit domain does emerge we would need to move the class to a more specific - namespace. - -Do not use the [stage or group name](https://about.gitlab.com/handbook/product/categories/#devops-stages) -since a feature category could be reassigned to a different group in the future. - -```ruby -# bad -module Create - class Commit - end -end - -# good -module Repositories - class Commit - end -end -``` - -On the other hand, a feature category may sometimes be too granular. Features tend to be -treated differently according to Product and Marketing, while they may share a lot of -domain models and behavior under the hood. In this case, having too many bounded contexts -could make them shallow and more coupled with other contexts. - -Bounded contexts (or top-level namespaces) can be seen as macro-components in the overall app. -Good bounded contexts should be [deep](https://medium.com/@nakabonne/depth-of-module-f62dac3c2fdb) -so consider having nested namespaces to further break down complex parts of the domain. -For example, `Ci::Config::`. - -For example, instead of having separate and granular bounded contexts like: `ContainerScanning::`, -`ContainerHostSecurity::`, `ContainerNetworkSecurity::`, we could have: - -```ruby -module ContainerSecurity - module HostSecurity - end - - module NetworkSecurity - end - - module Scanning - end -end -``` - -If classes that are defined into a namespace have a lot in common with classes in other namespaces, -chances are that these two namespaces are part of the same bounded context. +<!-- This redirect file can be deleted after <2023-01-24>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index d52db71b633..a9f2726ea93 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -12,7 +12,7 @@ The GitLab documentation is [intended as the single source of truth (SSOT)](http In addition to this page, the following resources can help you craft and contribute to documentation: - [Style Guide](styleguide/index.md) - What belongs in the docs, language guidelines, Markdown standards to follow, links, and more. -- [Topic type template](structure.md) - Learn about the different types of topics. +- [Topic types](topic_types/index.md) - Learn about the different types of topics. - [Documentation process](workflow.md). - [Markdown Guide](../../user/markdown.md) - A reference for all Markdown syntax supported by GitLab. - [Site architecture](site_architecture/index.md) - How <https://docs.gitlab.com> is built. @@ -159,26 +159,17 @@ You can use a Rake task to update the `CODEOWNERS` file. To update the `CODEOWNERS` file: -1. Open a merge request to update - [the Rake task](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/tasks/gitlab/tw/codeowners.rake) - with the latest [TW team assignments](https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments). -1. Assign the merge request to a backend maintainer for review and merge. -1. After the MR is merged, go to the root of the `gitlab` repository. -1. Run the Rake task and save the output in a file: - - ```shell - bundle exec rake tw:codeowners > ~/Desktop/updates.md - ``` - -1. Open the file (for example, `~/Desktop/updates.md`) and copy everything - except the errors at the bottom of the file. -1. Open the [`CODEOWNERS`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/CODEOWNERS) - file and paste the lines into the `^[Documentation Pages]` section. - - WARNING: - The documentation section is not the last section of the `CODEOWNERS` file. Don't - delete data that isn't ours! - +1. Review the [TW team assignments](https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments) + in the [`codeowners.rake`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/tasks/gitlab/tw/codeowners.rake) + file. If any assignments have changed: + 1. Update the `codeowners.rake` file with the changes. + 1. Assign the merge request to a technical writing manager for review and merge. +1. After the changes to `codeowners.rake` are merged, go to the root of the `gitlab` repository. +1. Run the Rake task with this command: `bundle exec rake tw:codeowners` +1. Review the command output for any pages that need attention to + their metadata. Handle any needed changes in a separate merge request. +1. Add the changes to the CODEOWNERS file to Git: `git add .gitlab/CODEOWNERS` +1. Commit your changes to your branch, and push your branch up to `origin`. 1. Create a merge request and assign it to a technical writing manager for review. ## Move, rename, or delete a page diff --git a/doc/development/documentation/site_architecture/deployment_process.md b/doc/development/documentation/site_architecture/deployment_process.md index 18cc27adaaa..2ba69ca0987 100644 --- a/doc/development/documentation/site_architecture/deployment_process.md +++ b/doc/development/documentation/site_architecture/deployment_process.md @@ -167,30 +167,5 @@ If you do not have the Maintainer role to perform this task, ask for help in the ## Docker files -The [`dockerfiles` directory](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/) contains all needed -Dockerfiles to build and deploy <https://docs.gitlab.com>. It is heavily inspired by Docker's -[Dockerfile](https://github.com/docker/docker.github.io/blob/06ed03db13895bfe867761b6fc2ad40acf6026dd/Dockerfile). - -| Dockerfile | Docker image | Description | -|:---------------------------------------------------------------------------------------------------------------------------|:------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [`bootstrap.Dockerfile`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/bootstrap.Dockerfile) | `gitlab-docs:bootstrap` | Contains all the dependencies that are needed to build the website. If the gems are updated and `Gemfile{,.lock}` changes, the image must be rebuilt. | -| [`builder.onbuild.Dockerfile`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/builder.onbuild.Dockerfile) | `gitlab-docs:builder-onbuild` | Base image to build the docs website. It uses `ONBUILD` to perform all steps and depends on `gitlab-docs:bootstrap`. | -| [`nginx.onbuild.Dockerfile`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/nginx.onbuild.Dockerfile) | `gitlab-docs:nginx-onbuild` | Base image to use for building documentation archives. It uses `ONBUILD` to perform all required steps to copy the archive, and relies upon its parent `Dockerfile.builder.onbuild` that is invoked when building single documentation archives (see the `Dockerfile` of each branch) | -| [`archives.Dockerfile`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/archives.Dockerfile) | `gitlab-docs:archives` | Contains all the versions of the website in one archive. It copies all generated HTML files from every version in one location. | - -### How to build the images - -Although build images are built automatically via GitLab CI/CD, you can build and tag all tooling images locally: - -1. Make sure you have [Docker installed](https://docs.docker.com/get-docker/). -1. Make sure you're in the `dockerfiles/` directory of the `gitlab-docs` repository. -1. Build the images: - - ```shell - docker build -t registry.gitlab.com/gitlab-org/gitlab-docs:bootstrap -f Dockerfile.bootstrap ../ - docker build -t registry.gitlab.com/gitlab-org/gitlab-docs:builder-onbuild -f Dockerfile.builder.onbuild ../ - docker build -t registry.gitlab.com/gitlab-org/gitlab-docs:nginx-onbuild -f Dockerfile.nginx.onbuild ../ - ``` - -For each image, there's a manual job under the `images` stage in -[`.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/.gitlab-ci.yml) which can be invoked at any time. +The [`dockerfiles` directory](https://gitlab.com/gitlab-org/gitlab-docs/-/tree/main/dockerfiles) contains Dockerfiles needed +to build, test, and deploy <https://docs.gitlab.com>. diff --git a/doc/development/documentation/site_architecture/global_nav.md b/doc/development/documentation/site_architecture/global_nav.md index d95ca720119..ef6f3c0ae18 100644 --- a/doc/development/documentation/site_architecture/global_nav.md +++ b/doc/development/documentation/site_architecture/global_nav.md @@ -18,12 +18,14 @@ Global navigation is the left-most pane in the documentation. You can use the Research shows that people use Google to search for GitLab product documentation. When they land on a result, we want them to find topics nearby that are related to the content they're reading. The global nav provides this information. -At the highest level, our global nav is workflow-based. Navigation needs to help users build a mental model of how to use GitLab. +At the highest level, our global nav is **workflow-based**. Navigation needs to help users build a mental model of how to use GitLab. The levels under each of the higher workflow-based topics are the names of features. For example: **Use GitLab** (_workflow_) **> Build your application** (_workflow_) **> CI/CD** (_feature_) **> Pipelines** (_feature_) +While some older sections of the nav are alphabetical, the nav should primarily be workflow-based. + ## Choose the right words for your navigation entry Before you add an item to the left nav, choose the parts of speech you want to use. diff --git a/doc/development/documentation/structure.md b/doc/development/documentation/structure.md deleted file mode 100644 index 35a93f08f66..00000000000 --- a/doc/development/documentation/structure.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'topic_types/index.md' -remove_date: '2022-11-16' ---- - -This document was moved to [another location](topic_types/index.md). - -<!-- This redirect file can be deleted after <2022-11-16>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
\ No newline at end of file diff --git a/doc/development/documentation/styleguide/img/tier_badge.png b/doc/development/documentation/styleguide/img/tier_badge.png Binary files differdeleted file mode 100644 index 5fc38e08172..00000000000 --- a/doc/development/documentation/styleguide/img/tier_badge.png +++ /dev/null diff --git a/doc/development/documentation/styleguide/index.md b/doc/development/documentation/styleguide/index.md index ef934186981..3e55b334992 100644 --- a/doc/development/documentation/styleguide/index.md +++ b/doc/development/documentation/styleguide/index.md @@ -97,7 +97,7 @@ move in this direction, so we can address these issues: information into a format that is geared toward helping others, rather than documenting how a feature was implemented. -GitLab uses these [topic types](../structure.md). +GitLab uses these [topic types](../topic_types/index.md). ### Link instead of repeating text @@ -147,7 +147,7 @@ help benefit translation. For example, we: - [since and because](word_list.md#since) - [once and after](word_list.md#once) - [it](word_list.md#it) -- Avoid [ing](word_list.md#-ing-words) words. +- Avoid [-ing](word_list.md#-ing-words) words. [The GitLab voice](#the-gitlab-voice) dictates that we write clearly and directly, and with translation in mind. [The word list](word_list.md) and our Vale rules @@ -158,7 +158,7 @@ also aid in consistency, which is important for localization. All GitLab documentation is written using [Markdown](https://en.wikipedia.org/wiki/Markdown). The [documentation website](https://docs.gitlab.com) uses [GitLab Kramdown](https://gitlab.com/gitlab-org/gitlab_kramdown), -a "flavored" Kramdown engine to render pages from Markdown to HTML. The use of Kramdown's +a "flavored" Kramdown engine to render pages from Markdown to HTML. The use of Kramdown features is limited by our linters, so, use regular Markdown and follow the rules in the linked style guide. You can't use Kramdown-specific markup (for example, `{:.class}`). @@ -345,10 +345,36 @@ Some contractions, however, should be avoided: <!-- vale gitlab.Possessive = YES --> +### Possessives + +Try to avoid using possessives (`'s`) for proper nouns, like organization or product names. + +For example, instead of `Docker's CLI`, use `the Docker CLI`. + +For details, see [the Google documentation style guide](https://developers.google.com/style/possessives#product,-feature,-and-company-names). + +### Prepositions + +Use prepositions at the end of the sentence when needed. +Dangling or stranded prepositions are fine. For example: + +- You can leave the group you're a member of. +- Share the credentials with users you want to give access to. + +These constructions are more casual than the alternatives: + +- You can leave the group of which you're a member. +- Share the credentials with users to which you want to give access. + ### Acronyms If you use an acronym, spell it out on first use on a page. You do not need to spell it out more than once on a page. -When possible, try to avoid acronyms in topic titles. + +- **Titles:** Try to avoid acronyms in topic titles, especially if the acronym is not widely used. +- **Plurals:** Try not to make acronyms plural. For example, use `YAML files`, not `YAMLs`. If you must make an acronym plural, do not use an apostrophe. For example, use `APIs`, not `API's`. +- **Possessives:** Use caution when making an acronym possessive. If possible, + write the sentence to avoid making the acronym possessive. If you must make the + acronym possessive, consider spelling out the words. ### Numbers @@ -383,8 +409,12 @@ when published. Example: ### Emphasis +<!-- vale gitlab.Spelling = NO --> + Use **bold** rather than italic to provide emphasis. GitLab uses a sans-serif font and italic text does not stand out as much as it would in a serif font. For details, see [Butterick's Practical Typography guide on bold or italic](https://practicaltypography.com/bold-or-italic.html). +<!-- vale gitlab.Spelling = YES --> + You can use italics when you are introducing a term for the first time. Otherwise, use bold. - Use double asterisks (`**`) to mark a word or text in bold (`**bold**`). @@ -700,7 +730,7 @@ Put the entire link on a single line so that [linters](../testing.md) can find i ### Links in separate repositories To link to a page in a different repository, use an absolute URL. -For example, to link from a page in the GitLab repo to the Charts repo, +For example, to link from a page in the GitLab repository to the Charts repository, use a URL like `https://docs.gitlab.com/charts/`. ### Anchor links @@ -737,6 +767,28 @@ in your merge request fails. ### Text for links +Follow these guidelines for link text. + +#### Standard text + +As much as possible, use text that follows one of these patterns: + +- `For more information, see [LINK TEXT](LINK)`. +- `To [DO THIS THING], see [LINK TEXT](LINK)` + +For example: + +- `For more information, see [merge requests](../../../user/project/merge_requests/index.md).` +- `To create a review app, see [review apps](../../../ci/review_apps/index.md).` + +You can expand on this text by using phrases like +`For more information about this feature, see...` + +Do not to use alternate phrases, like `Learn more about...` or +`To read more...`. + +#### Descriptive text rather than `here` + Use descriptive text for links, rather than words like `here` or `this page.` For example, instead of: @@ -748,6 +800,14 @@ Use: - `For more information, see [merge requests](LINK)`. +#### Links to issues + +When linking to an issue, include the issue number in the link. For example: + +- `For more information, see [issue 12345](LINK).` + +Do not use the pound sign (`issue #12345`). + ### Links to external documentation When possible, avoid links to external documentation. These links can easily become outdated, and are difficult to maintain. @@ -1382,10 +1442,12 @@ Here's some other content in tab two. For tab titles, be brief and consistent. Ensure they are parallel, and start each with a capital letter. For example: -- `Omnibus package`, `Helm chart`, `Source` +- `Linux package (Omnibus)`, `Helm chart (Kubernetes)` (when documenting configuration edits, follow the + [configuration edits guide](#configuration-documentation-for-different-installation-methods)) - `15.1 and earlier`, `15.2 and later` -See [Pajamas](https://design.gitlab.com/components/tabs/#guidelines) for details. +See [Pajamas](https://design.gitlab.com/components/tabs/#guidelines) for more +details on tabs. ## Terms @@ -1414,9 +1476,7 @@ When names change, it is more complicated to search or grep text that has line b ### Product tier badges Tier badges are displayed as orange text next to a topic title. These badges link to the GitLab -pricing page. For example: - - +pricing page. You must assign a tier badge: @@ -1436,17 +1496,17 @@ functionality is described. #### Available product tier badges -| Tier in which feature is available | Tier badge | -|:------------------------------------------------------------------------|:----------------------| -| GitLab Free self-managed and SaaS, and higher tiers | `**(FREE)**` | -| GitLab Premium self-managed and SaaS, and their higher tiers | `**(PREMIUM)**` | -| GitLab Ultimate self-managed and SaaS | `**(ULTIMATE)**` | -| Only GitLab Free self-managed and higher tiers (no SaaS-based tiers) | `**(FREE SELF)**` | -| Only GitLab Premium self-managed and higher tiers (no SaaS-based tiers) | `**(PREMIUM SELF)**` | -| Only GitLab Ultimate self-managed (no SaaS-based tiers) | `**(ULTIMATE SELF)**` | -| Only GitLab Free SaaS and higher tiers (no self-managed instances) | `**(FREE SAAS)**` | -| Only GitLab Premium SaaS and higher tiers (no self-managed instances) | `**(PREMIUM SAAS)**` | -| Only GitLab Ultimate SaaS (no self-managed instances) | `**(ULTIMATE SAAS)**` | +| Where feature is available | Tier badge | +|:-----------------------------------------------------------------------------------------|:----------------------| +| On GitLab self-managed and GitLab SaaS, available in all tiers. | `**(FREE)**` | +| On GitLab self-managed and GitLab SaaS, available in Premium and Ultimate. | `**(PREMIUM)**` | +| On GitLab self-managed and GitLab SaaS, available in Ultimate. | `**(ULTIMATE)**` | +| On GitLab self-managed, available in all tiers. Not available on GitLab SaaS. | `**(FREE SELF)**` | +| On GitLab self-managed, available in Premium and Ultimate. Not available on GitLab SaaS. | `**(PREMIUM SELF)**` | +| On GitLab self-managed, available in Ultimate. Not available on GitLab SaaS. | `**(ULTIMATE SELF)**` | +| On GitLab SaaS, available in all tiers. Not available on self-managed. | `**(FREE SAAS)**` | +| On GitLab SaaS, available in Premium and Ultimate. Not available on self-managed. | `**(PREMIUM SAAS)**` | +| On GitLab SaaS, available in Ultimate. Not available on self-managed. | `**(ULTIMATE SAAS)**` | Topics that are only for instance administrators should be badged `<TIER> SELF`. Instance administrator documentation often includes sections that mention: @@ -1487,21 +1547,35 @@ we install Ruby from source. To update the guide for a new Ruby version: - Replace the sha256sum. It's available on the [downloads page](https://www.ruby-lang.org/en/downloads/) of the Ruby website. -### Configuration documentation for source and Omnibus installations +### Configuration documentation for different installation methods -GitLab supports two installation methods: installations from source, and Omnibus -packages. Possible configuration settings include: +GitLab supports four installation methods: -- Settings that touch configuration files in `config/`. -- NGINX settings. -- Other settings in `lib/support/`. +- Linux package (Omnibus) +- Helm chart (Kubernetes) +- Docker +- Self-compiled (source) Configuration procedures can require users to edit configuration files, reconfigure -GitLab, or restart GitLab. Use these styles to document these steps, replacing -`PATH/TO` with the appropriate path: +GitLab, or restart GitLab. In this case: + +- Use [tabs](#tabs) to differentiate among the various installation methods. +- Use the installation methods names exactly as described in the previous list. +- Use them in the order described below. +- Indent the code blocks to line up with the list item they belong to. +- Use the appropriate syntax highlighting for each code block (`ruby`, `shell`, or `yaml`). +- For the YAML files, always include the parent settings. +- The final step to reconfigure or restart GitLab can be used verbatim since it's + the same every time. + +You can copy and paste the following snippet when describing a configuration +edit: +<!-- markdownlint-disable tabs-blank-lines --> ````markdown -**For Omnibus installations** +::Tabs + +:::TabTitle Linux package (Omnibus) 1. Edit `/etc/gitlab/gitlab.rb`: @@ -1509,32 +1583,159 @@ GitLab, or restart GitLab. Use these styles to document these steps, replacing external_url "https://gitlab.example.com" ``` -1. Save the file and [reconfigure](PATH/TO/administration/restart_gitlab.md#omnibus-gitlab-reconfigure) - GitLab for the changes to take effect. +1. Save the file and reconfigure GitLab: ---- + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: + + ```yaml + global: + hosts: + gitlab: + name: gitlab.example.com + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + environment: + GITLAB_OMNIBUS_CONFIG: | + external_url "https://gitlab.example.com" + ``` -**For installations from source** +1. Save the file and restart GitLab: -1. Edit `config/gitlab.yml`: + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) + +1. Edit `/home/git/gitlab/config/gitlab.yml`: ```yaml - gitlab: - host: "gitlab.example.com" + production: &base + gitlab: + host: "gitlab.example.com" + ``` + +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart ``` -1. Save the file and [restart](PATH/TO/administration/restart_gitlab.md#installations-from-source) - GitLab for the changes to take effect. +::EndTabs ```` +<!-- markdownlint-enable tabs-blank-lines --> + +It renders as: + +::Tabs + +:::TabTitle Linux package (Omnibus) + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + external_url "https://gitlab.example.com" + ``` + +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: + + ```yaml + global: + hosts: + gitlab: + name: gitlab.example.com + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + environment: + GITLAB_OMNIBUS_CONFIG: | + external_url "https://gitlab.example.com" + ``` -In this case: +1. Save the file and restart GitLab: -- Bold the installation method's name. -- Separate the methods with three dashes (`---`) to create a horizontal line. -- Indent the code blocks to line up with the list item they belong to.. -- Use the appropriate syntax highlighting for each code block. -- Use the [GitLab Restart](#gitlab-restart) section to explain any required - restart or reconfigure of GitLab. + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: &base + gitlab: + host: "gitlab.example.com" + ``` + +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +::EndTabs ## Feature flags diff --git a/doc/development/documentation/styleguide/word_list.md b/doc/development/documentation/styleguide/word_list.md index d28972a644b..333a5521536 100644 --- a/doc/development/documentation/styleguide/word_list.md +++ b/doc/development/documentation/styleguide/word_list.md @@ -745,6 +745,11 @@ For **MB** and **GB**, follow the [Microsoft guidance](https://learn.microsoft.c Do not use first-person singular. Use **you**, **we**, or **us** instead. ([Vale](../testing.md#vale) rule: [`FirstPerson.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/FirstPerson.yml)) +## member + +When you add a [user account](#user-account) to a group or project, +the user account becomes a **member**. + ## merge requests Use lowercase for **merge requests**. If you use **MR** as the acronym, spell it out on first use. @@ -767,20 +772,41 @@ Do not use **navigate**. Use **go** instead. For example: ([Vale](../testing.md#vale) rule: [`SubstitutionSuggestions.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/SubstitutionSuggestions.yml)) -## need to, should +## need to + +Try to avoid **need to**, because it's wordy. + +For example, when a variable is **required**, +instead of **You need to set the variable**, use: + +- Set the variable. +- You must set the variable. + +When the variable is **recommended**: + +- You should set the variable. -Try to avoid **needs to**, because it's wordy. If something is recommended, use **should** instead. If something is required, use **must**. +When the variable is **optional**: + +- You can set the variable. + +## normal, normally + +Don't use **normal** to mean the usual, typical, or standard way of doing something. +Use those terms instead. Use: -- You should set the variable. (recommended) -- You must set the variable. (required) -- Set the variable. (required) +- Typically, you specify a certificate. +- Usually, you specify a certificate. +- Follow the standard Git workflow. Instead of: -- You need to set the variable. -- We recommend that you set the variable. +- Normally, you specify a certificate. +- Follow the normal Git workflow. + +([Vale](../testing.md#vale) rule: [`Normal.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/Normal.yml)) ## note that @@ -898,6 +924,15 @@ For example, you might write something like: Use lowercase for **push rules**. +## recommend, we recommend + +Instead of **we recommend**, use **you should**. We want to talk to the user the way +we would talk to a colleague, and to avoid differentiation between `we` and `them`. + +- You should set the variable. (It's recommended.) +- Set the variable. (It's required.) +- You can set the variable. (It's optional.) + ## register Use **register** instead of **sign up** when talking about creating an account. @@ -1068,6 +1103,21 @@ Do not use **slave**. Another option is **secondary**. ([Vale](../testing.md#val Use **subgroup** (no hyphen) instead of **sub-group**. ([Vale](../testing.md#vale) rule: [`SubstitutionSuggestions.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/SubstitutionSuggestions.yml)) +## subscription tier + +Do not confuse **subscription** or **subscription tier** with **[license](#license)**. +A user purchases a **subscription**. That subscription has a **tier**. + +To describe tiers: + +| Instead of | Use | +|---------------------------------|----------------------------------------| +| In the Free tier or greater | In all tiers | +| In the Free tier or higher | In all tiers | +| In the Premium tier or greater | In the Premium and Ultimate tier | +| In the Premium tier or higher | In the Premium and Ultimate tier | +| In the Premium tier or lower | In the Free and Premium tier | + ## that Do not use **that** when describing a noun. For example: @@ -1127,6 +1177,11 @@ Always follow these words with a noun. For example: - Use: **Those settings** need to be configured. (Or even better, **Configure those settings.**) - Instead of: **Those** need to be configured. +## to which, of which + +Try to avoid **to which** and **of which**, and let the preposition dangle at the end of the sentence instead. +For examples, see [Prepositions](index.md#prepositions). + ## to-do item Use lowercase and hyphenate **to-do** item. ([Vale](../testing.md#vale) rule: [`ToDo.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/ToDo.yml)) @@ -1198,18 +1253,10 @@ See also [downgrade](#downgrade) and [roll back](#roll-back). Do not use **useful**. If the user doesn't find the process to be useful, we lose their trust. ([Vale](../testing.md#vale) rule: [`Simplicity.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/Simplicity.yml)) -## user, users - -When possible, address the reader directly, instead of calling them **users**. -Use the [second person](#you-your-yours), **you**, instead. +## user account -Use: - -- You can configure a pipeline. - -Instead of: - -- Users can configure a pipeline. +You create a **user account**. The user account has an [access level](#access-level). +When you add a **user account** to a group or project, the user account becomes a **member**. ## utilize @@ -1270,7 +1317,7 @@ in present tense, active voice. ## you, your, yours -Use **you**, **your**, and **yours** instead of [**the user** and **the user's**](#user-users). +Use **you**, **your**, and **yours** instead of **the user** and **the user's**. Documentation should be from the [point of view](https://design.gitlab.com/content/voice-tone/#point-of-view) of the reader. Use: diff --git a/doc/development/documentation/topic_types/concept.md b/doc/development/documentation/topic_types/concept.md index 7be6bef4fad..e01b06c2c07 100644 --- a/doc/development/documentation/topic_types/concept.md +++ b/doc/development/documentation/topic_types/concept.md @@ -11,7 +11,7 @@ A concept introduces a single feature or concept. A concept should answer the questions: - What is this? -- Why would I use it? +- Why would you use it? Think of everything someone might want to know if they've never heard of this concept before. diff --git a/doc/development/documentation/topic_types/index.md b/doc/development/documentation/topic_types/index.md index 8e8c474ce3c..964b41303cb 100644 --- a/doc/development/documentation/topic_types/index.md +++ b/doc/development/documentation/topic_types/index.md @@ -6,20 +6,19 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Documentation topic types (CTRT) -At GitLab, we have not traditionally used types for our content. However, we are starting to -move in this direction, and we now use four primary topic types: +Each topic on a page should be one of the following topic types: - [Concept](concept.md) - [Task](task.md) - [Reference](reference.md) - [Troubleshooting](troubleshooting.md) +Even if a page is short, the page usually starts with a concept and then +includes a task or reference topic. + The tech writing team sometimes uses the acronym `CTRT` to refer to our topic types. The acronym refers to the first letter of each topic type. -In general, each page in the GitLab documentation contains multiple topics. -Each topic on a page should be recognizable as a specific topic type. - In addition to the four primary topic types, we also have a page type for [Tutorials](tutorial.md) and [Get started](#get-started). @@ -66,9 +65,9 @@ Some pages are solely a list of links to other documentation. We do not encourage this page type. Lists of links can get out-of-date quickly and offer little value to users, who prefer to search to find information. -## Topic text guidelines +## Topic title guidelines -In general, for topic text: +In general, for topic titles: - Be clear and direct. Make every word count. - Use articles and prepositions. diff --git a/doc/development/documentation/topic_types/task.md b/doc/development/documentation/topic_types/task.md index 78d670a16d6..0dba3e079b6 100644 --- a/doc/development/documentation/topic_types/task.md +++ b/doc/development/documentation/topic_types/task.md @@ -69,6 +69,12 @@ For example, `Create an issue when you want to track bugs or future work`. To start the task steps, use a succinct action followed by a colon. For example, `To create an issue:` +## Task prerequisites + +As a best practice, if the task requires the user to have a role other than Guest, +put the minimum role in the prerequisites. See [the Word list](../styleguide/word_list.md) for +how to write the phrase for each role. + ## Related topics - [View the format for writing task steps](../styleguide/index.md#navigation). diff --git a/doc/development/documentation/versions.md b/doc/development/documentation/versions.md index 030bdec0361..334dcd73ea5 100644 --- a/doc/development/documentation/versions.md +++ b/doc/development/documentation/versions.md @@ -46,7 +46,7 @@ The item text must include these words in order. Capitalization doesn't matter. - `introduced`, `enabled`, `deprecated`, `changed`, `moved`, `recommended`, `removed`, or `renamed` - `in` or `to` -- `GitLab` +- `GitLab` (or, for external projects, the name of the project) If possible, include a link to the related issue, merge request, or epic. Do not link to the pricing page. Do not include the subscription tier. @@ -203,8 +203,8 @@ We cannot guarantee future feature work, and promises like these can raise legal issues. Instead, say that an issue exists. For example: -- Support for improvements is proposed in issue `[issue-number](LINK-TO-ISSUE)`. -- You cannot do this thing, but issue `[issue-number](LINK-TO-ISSUE)` proposes to change this behavior. +- Support for improvements is proposed in `[issue <issue_number>](LINK-TO-ISSUE)`. +- You cannot do this thing, but `[issue 12345](LINK-TO-ISSUE)` proposes to change this behavior. You can say that we plan to remove a feature. diff --git a/doc/development/documentation/workflow.md b/doc/development/documentation/workflow.md index 9d8d25607c8..2effa21b266 100644 --- a/doc/development/documentation/workflow.md +++ b/doc/development/documentation/workflow.md @@ -13,29 +13,32 @@ Anyone can contribute to the GitLab documentation! You can create a merge reques accomplish their work with GitLab. If you are working on a feature or enhancement, use the -[feature workflow process described in the GitLab Handbook](https://about.gitlab.com/handbook/product/ux/technical-writing/workflow/#for-a-product-change). +[feature workflow process described in the GitLab Handbook](https://about.gitlab.com/handbook/product/ux/technical-writing/workflow/#documentation-for-a-product-change). ## How to update the docs If you are not a GitLab team member, or do not have the Developer role for the GitLab repository, to update GitLab documentation: -1. Select an issue you'd like to work on. +1. Select an [issue](https://about.gitlab.com/handbook/product/ux/technical-writing/#community-contribution-opportunities) you'd like to work on. - You don't need an issue to open a merge request. - For a Hackathon, in the issue, in a comment, mention the person who opened the issue and ask for the issue to be assigned to you. To be fair to other contributors, if you see someone has already asked to work on the issue, choose another issue. If you are looking for issues to work on and don't see any that suit you, you can always fix [Vale](testing.md#vale) issues. 1. Go to the [GitLab repository](https://gitlab.com/gitlab-org/gitlab). -1. In the top-right, select **Fork**. Forking makes a copy of the repository on GitLab.com. -1. In your fork, find the documentation page by going to the `\doc` directory. +1. In the top right, select **Fork**. Forking makes a copy of the repository on GitLab.com. +1. In your fork, find the documentation page in the `\doc` directory. 1. If you know Git, make your changes and open a merge request. If not, follow these steps: - 1. In the top right, select **Edit**, make the changes, and **Save**. - 1. From the left menu, select **Merge requests**. + 1. On the top right, select **Edit** if it is visible. If it is not, select the down arrow (**{chevron-lg-down}**) next to **Open in Web IDE** or **Gitpod**, and select **Edit**. + 1. In the **Commit message** text box, enter a commit message. Use 3-5 words, start with a capital letter, and do not end with a period. + 1. Select **Commit changes**. + 1. On the left sidebar, select **Merge requests**. + 1. Select **New merge request**. 1. For the source branch, select your fork and branch. If you did not create a branch, select `master`. For the target branch, select the [GitLab repository](https://gitlab.com/gitlab-org/gitlab) `master` branch. - 1. For the commit message, use 3-5 words, start with a capital letter, and do not end with a period. - 1. Select **Commit changes**. A merge request opens. + 1. Select **Compare branches and continue**. A new merge request opens. 1. Select the **Documentation** template. In the description, write a brief summary of the changes and link to the related issue, if there is one. + 1. Select **Create merge request**. If you need help while working on the page, view: @@ -65,7 +68,7 @@ If you are a member of the GitLab Slack workspace, you can request help in `#doc When you author an issue or merge request, you must add these labels: -- A [type label](../contributing/issue_workflow.md#type-labels). +- A [type label](../contributing/issue_workflow.md#type-labels), either `~"type::feature"` or `~"type::maintenance"`. - A [stage label](../contributing/issue_workflow.md#stage-labels) and [group label](../contributing/issue_workflow.md#group-labels). For example, `~devops::create` and `~group::source code`. - A `~documentation` [specialization label](../contributing/issue_workflow.md#specialization-labels). @@ -75,9 +78,8 @@ A member of the Technical Writing team adds these labels: - A [documentation scoped label](../../user/project/labels.md#scoped-labels) with the `docs::` prefix. For example, `~docs::improvement`. - The [`~Technical Writing` team label](../contributing/issue_workflow.md#team-labels). -- A type label: either `~"type::feature"` or `~"type::maintenance"`. -### Reviewing and merging +## Reviewing and merging Anyone with the Maintainer role to the relevant GitLab project can merge documentation changes. Maintainers must make a good-faith effort to ensure that the content: @@ -111,13 +113,24 @@ The process involves the following: The process is reflected in the **Documentation** [merge request template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/merge_request_templates/Documentation.md). -## Other ways to help +### Before merging -If you have ideas for further documentation resources please -[create an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Documentation) -using the Documentation template. +Ensure the following if skipping an initial Technical Writer review: + +- [Product badges](styleguide/index.md#product-tier-badges) are applied. +- The GitLab [version](versions.md) that + introduced the feature is included. +- Changes to topic titles don't affect in-app hyperlinks. +- Specific [user permissions](../../user/permissions.md) are documented. +- New documents are linked from higher-level indexes, for discoverability. +- The style guide is followed: + - For [directories and files](site_architecture/folder_structure.md). + - For [images](styleguide/index.md#images). + +Merge requests that change the location of documentation must always be reviewed by a Technical +Writer before merging. -## Post-merge reviews +### Post-merge reviews If not assigned to a Technical Writer for review prior to merging, a review must be scheduled immediately after merge by the developer or maintainer. For this, @@ -146,19 +159,8 @@ Remember: - The Technical Writer can also help decide that documentation can be merged without Technical writer review, with the review to occur soon after merge. -### Before merging - -Ensure the following if skipping an initial Technical Writer review: - -- [Product badges](styleguide/index.md#product-tier-badges) are applied. -- The GitLab [version](versions.md) that - introduced the feature is included. -- Changes to topic titles don't affect in-app hyperlinks. -- Specific [user permissions](../../user/permissions.md) are documented. -- New documents are linked from higher-level indexes, for discoverability. -- The style guide is followed: - - For [directories and files](site_architecture/folder_structure.md). - - For [images](styleguide/index.md#images). +## Other ways to help -Merge requests that change the location of documentation must always be reviewed by a Technical -Writer before merging. +If you have ideas for further documentation resources please +[create an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Documentation) +using the Documentation template. diff --git a/doc/development/ee_features.md b/doc/development/ee_features.md index 14df73b8779..5e236c3e322 100644 --- a/doc/development/ee_features.md +++ b/doc/development/ee_features.md @@ -94,6 +94,25 @@ setting the [`FOSS_ONLY` environment variable](https://gitlab.com/gitlab-org/git to something that evaluates as `true`. The same works for running tests (for example `FOSS_ONLY=1 yarn jest`). +### Simulate a CE instance with a licensed GDK + +To simulate a CE instance without deleting the license in a GDK: + +1. Create an `env.runit` file in the root of your GDK with the line: + + ```shell + export FOSS_ONLY=1 + ``` + +1. Then restart the GDK: + + ```shell + gdk restart rails && gdk restart webpack + ``` + +Remove the line in `env.runit` if you want to revert back to an EE +installation, and repeat step 2. + #### Run feature specs as CE When running [feature specs](testing_guide/best_practices.md#system--feature-tests) diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md index ab2d241a781..88a417b4745 100644 --- a/doc/development/elasticsearch.md +++ b/doc/development/elasticsearch.md @@ -69,7 +69,7 @@ The `whitespace` tokenizer was selected to have more control over how tokens are Please see the `code` filter for an explanation on how tokens are split. NOTE: -The [Elasticsearch code_analyzer doesn't account for all code cases](../integration/advanced_search/elasticsearch_troubleshooting.md#elasticsearch-code_analyzer-doesnt-account-for-all-code-cases). +The [Elasticsearch `code_analyzer` doesn't account for all code cases](../integration/advanced_search/elasticsearch_troubleshooting.md#elasticsearch-code_analyzer-doesnt-account-for-all-code-cases). #### `code_search_analyzer` @@ -113,9 +113,9 @@ Uses a [Pattern Capture token filter](https://www.elastic.co/guide/en/elasticsea Patterns: -- `"(\\p{Ll}+|\\p{Lu}\\p{Ll}+|\\p{Lu}+)"`: captures CamelCased and lowedCameCased strings as separate tokens +- `"(\\p{Ll}+|\\p{Lu}\\p{Ll}+|\\p{Lu}+)"`: captures CamelCase and lowerCamelCase strings as separate tokens - `"(\\d+)"`: extracts digits -- `"(?=([\\p{Lu}]+[\\p{L}]+))"`: captures CamelCased strings recursively. Ex: `ThisIsATest` => `[ThisIsATest, IsATest, ATest, Test]` +- `"(?=([\\p{Lu}]+[\\p{L}]+))"`: captures CamelCase strings recursively. For example: `ThisIsATest` => `[ThisIsATest, IsATest, ATest, Test]` - `'"((?:\\"|[^"]|\\")*)"'`: captures terms inside quotes, removing the quotes - `"'((?:\\'|[^']|\\')*)'"`: same as above, for single-quotes - `'\.([^.]+)(?=\.|\s|\Z)'`: separate terms with periods in-between diff --git a/doc/development/fe_guide/accessibility.md b/doc/development/fe_guide/accessibility.md index 67166a93cb4..af45603782f 100644 --- a/doc/development/fe_guide/accessibility.md +++ b/doc/development/fe_guide/accessibility.md @@ -109,15 +109,15 @@ Text input examples: </gl-form-group> ``` -Textarea examples: +`textarea` examples: ```html -<!-- Textarea with label --> +<!-- textarea with label --> <gl-form-group :label="__('Issue description')" label-for="issue-description"> <gl-form-textarea id="issue-description" v-model="description" /> </gl-form-group> -<!-- Textarea with hidden label --> +<!-- textarea with hidden label --> <gl-form-group :label="__('Issue description')" label-for="issue-description" label-sr-only> <gl-form-textarea id="issue-description" v-model="description" /> </gl-form-group> @@ -347,7 +347,7 @@ Keep in mind that: See the [Pajamas Keyboard-only page](https://design.gitlab.com/accessibility-audits/keyboard-only/) for more detail. -## Tabindex +## `tabindex` Prefer **no** `tabindex` to using `tabindex`, since: diff --git a/doc/development/fe_guide/content_editor.md b/doc/development/fe_guide/content_editor.md index 8cc274c732e..982033cf2ad 100644 --- a/doc/development/fe_guide/content_editor.md +++ b/doc/development/fe_guide/content_editor.md @@ -11,7 +11,7 @@ experience for [GitLab Flavored Markdown](../../user/markdown.md) in the GitLab It also serves as the foundation for implementing Markdown-focused editors that target other engines, like static site generators. -We use [tiptap 2.0](https://tiptap.dev/) and [ProseMirror](https://prosemirror.net/) +We use [Tiptap 2.0](https://tiptap.dev/) and [ProseMirror](https://prosemirror.net/) to build the Content Editor. These frameworks provide a level of abstraction on top of the native [`contenteditable`](https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/Editable_content) web technology. @@ -209,7 +209,7 @@ the following events: - `blur` - `error`. -Learn more about these events in [Tiptap's event guide](https://tiptap.dev/api/events/). +Learn more about these events in [the Tiptap event guide](https://tiptap.dev/api/events/). ```html <script> @@ -255,13 +255,13 @@ provides all the necessary extensions to support #### Implement new extensions Extensions are the building blocks of the Content Editor. You can learn how to implement -new ones by reading [Tiptap's guide](https://tiptap.dev/guide/custom-extensions). +new ones by reading [the Tiptap guide](https://tiptap.dev/guide/custom-extensions). We recommend checking the list of built-in [nodes](https://tiptap.dev/api/nodes) and [marks](https://tiptap.dev/api/marks) before implementing a new extension from scratch. Store the Content Editor extensions in the `~/content_editor/extensions` directory. -When using a Tiptap's built-in extension, wrap it in a ES6 module inside this directory: +When using a Tiptap built-in extension, wrap it in a ES6 module inside this directory: ```javascript export { Bold as default } from '@tiptap/extension-bold'; @@ -326,10 +326,10 @@ sequenceDiagram A->>E: setContent(document) ``` -Deserializers live in the extension modules. Read Tiptap's -[parseHTML](https://tiptap.dev/guide/custom-extensions#parse-html) and -[addAttributes](https://tiptap.dev/guide/custom-extensions#attributes) documentation to -learn how to implement them. Titap's API is a wrapper around ProseMirror's +Deserializers live in the extension modules. Read Tiptap documentation about +[`parseHTML`](https://tiptap.dev/guide/custom-extensions#parse-html) and +[`addAttributes`](https://tiptap.dev/guide/custom-extensions#attributes) to +learn how to implement them. The Tiptap API is a wrapper around ProseMirror's [schema spec API](https://prosemirror.net/docs/ref/#model.SchemaSpec). #### Serialization diff --git a/doc/development/fe_guide/customizable_dashboards.md b/doc/development/fe_guide/customizable_dashboards.md index 38ee750d421..807f83f5bec 100644 --- a/doc/development/fe_guide/customizable_dashboards.md +++ b/doc/development/fe_guide/customizable_dashboards.md @@ -41,16 +41,12 @@ export default { // All values are grid row/column numbers up to 12. // We use the default 12 column grid https://github.com/gridstack/gridstack.js#change-grid-columns. gridAttributes: { - size: { - height: 4, - width: 6, - minHeight: 4, - minWidth: 6, - }, - position: { - xPos: 0, - yPos: 0, - }, + height: 4, + width: 6, + minHeight: 4, + minWidth: 6, + xPos: 0, + yPos: 0, }, // Options that are used to set bespoke values for each widget. // Available customizations are determined by the widget itself. diff --git a/doc/development/fe_guide/dark_mode.md b/doc/development/fe_guide/dark_mode.md index d687d9740c9..55181edd64c 100644 --- a/doc/development/fe_guide/dark_mode.md +++ b/doc/development/fe_guide/dark_mode.md @@ -19,9 +19,9 @@ Note the following: - We define two types of variables in `_dark.scss`: - SCSS variables are used in framework, components, and utility classes. - CSS variables are used for any colors within the `app/assets/stylesheets/page_bundles` directory. -- `app/views/layouts/_head.html.haml` then loads application or application_dark based on the user's theme preference. +- `app/views/layouts/_head.html.haml` then loads `application` or `application_dark` based on the user's theme preference. -As we do not want to generate separate `_dark.css` variants of every page_bundle file, +As we do not want to generate separate `_dark.css` variants of every `page_bundle` file, we use CSS variables with SCSS variables as fallbacks. This is because when we generate the `page_bundles` CSS, we get the variable values from `_variables.scss`, so any SCSS variables have light mode values. diff --git a/doc/development/fe_guide/development_process.md b/doc/development/fe_guide/development_process.md index fc91ff55b24..232689080ea 100644 --- a/doc/development/fe_guide/development_process.md +++ b/doc/development/fe_guide/development_process.md @@ -45,7 +45,7 @@ Use your best judgment when to use it and contribute new points through merge re - [ ] Check off tasks on your created task list to keep everyone updated on the progress - [ ] [Share your work early with reviewers/maintainers](#share-your-work-early) -- [ ] Share your work with UXer and Product Manager with Screenshots and/or [GIF's](https://about.gitlab.com/handbook/product/making-gifs/). They are easy to create for you and keep them up to date. +- [ ] Share your work with UXer and Product Manager with Screenshots and/or [GIF images](https://about.gitlab.com/handbook/product/making-gifs/). They are easy to create for you and keep them up to date. - [ ] If you are blocked on something let everyone on the issue know through a comment. - [ ] Are you unable to work on this issue for a longer period of time, also let everyone know. - [ ] **Documentation** Update/add docs for the new feature, see `docs/`. Ping one of the documentation experts/reviewers @@ -58,7 +58,7 @@ Use your best judgment when to use it and contribute new points through merge re - [ ] Did you check the mobile view? - [ ] Check the built webpack bundle (For the report run `WEBPACK_REPORT=true gdk start`, then open `webpack-report/index.html`) if we have unnecessary bloat due to wrong references, including libraries multiple times, etc.. If you need help contact the webpack [domain expert](https://about.gitlab.com/handbook/engineering/frontend/#frontend-domain-experts) - [ ] **Tests** Not only greenfield tests - Test also all bad cases that come to your mind. -- [ ] If you have multiple MR's then also smoke test against the final merge. +- [ ] If you have multiple MRs then also smoke test against the final merge. - [ ] Are there any big changes on how and especially how frequently we use the API then let production know about it - [ ] Smoke test of the RC on dev., staging., canary deployments and .com - [ ] Follow up on issues that came out of the review. Create issues for discovered edge cases that should be covered in future iterations. diff --git a/doc/development/fe_guide/graphql.md b/doc/development/fe_guide/graphql.md index 28ea84301a6..0fec38b1200 100644 --- a/doc/development/fe_guide/graphql.md +++ b/doc/development/fe_guide/graphql.md @@ -109,7 +109,7 @@ Default client accepts two parameters: `resolvers` and `config`. If you are making multiple queries to the same Apollo client object you might encounter the following error: `Cache data may be lost when replacing the someProperty field of a Query object. To address this problem, either ensure all objects of SomeEntityhave an id or a custom merge function`. We are already checking `ID` presence for every GraphQL type that has an `ID`, so this shouldn't be the case. Most likely, the `SomeEntity` type doesn't have an `ID` property, and to fix this warning we need to define a custom merge function. -We have some client-wide types with `merge: true` defined in the default client as [typePolicies](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/lib/graphql.js) (this means that Apollo will merge existing and incoming responses in the case of subsequent queries). Consider adding `SomeEntity` there or defining a custom merge function for it. +We have some client-wide types with `merge: true` defined in the default client as [`typePolicies`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/lib/graphql.js) (this means that Apollo will merge existing and incoming responses in the case of subsequent queries). Consider adding `SomeEntity` there or defining a custom merge function for it. ## GraphQL Queries @@ -1138,7 +1138,7 @@ query getPipelineEtag { ```javascript /* pipeline_editor/components/header/pipeline_status.vue */ -import getPipelineEtag from '~/pipeline_editor/graphql/queries/client/pipeline_etag.query.graphql'; +import getPipelineEtag from '~/ci/pipeline_editor/graphql/queries/client/pipeline_etag.query.graphql'; apollo: { pipelineEtag: { diff --git a/doc/development/fe_guide/haml.md b/doc/development/fe_guide/haml.md index d76d718e8e5..9dc5b265783 100644 --- a/doc/development/fe_guide/haml.md +++ b/doc/development/fe_guide/haml.md @@ -81,11 +81,7 @@ When using the GitLab UI form builder, the following components are available fo NOTE: Currently only the listed components are available but more components are planned. -<!-- vale gitlab.Spelling = NO --> - -#### gitlab_ui_checkbox_component - -<!-- vale gitlab.Spelling = YES --> +#### `gitlab_ui_checkbox_component` [GitLab UI Docs](https://gitlab-org.gitlab.io/gitlab-ui/?path=/story/base-form-form-checkbox--default) @@ -110,11 +106,7 @@ This component supports [ViewComponent slots](https://viewcomponent.org/guide/sl | `label` | Checkbox label content. This slot can be used instead of the `label` argument. | | `help_text` | Help text content displayed below the checkbox. This slot can be used instead of the `help_text` argument. | -<!-- vale gitlab.Spelling = NO --> - -#### gitlab_ui_radio_component - -<!-- vale gitlab.Spelling = YES --> +#### `gitlab_ui_radio_component` [GitLab UI Docs](https://gitlab-org.gitlab.io/gitlab-ui/?path=/story/base-form-form-radio--default) diff --git a/doc/development/fe_guide/merge_request_widget_extensions.md b/doc/development/fe_guide/merge_request_widget_extensions.md index 61d3e79a080..49c6664c6d6 100644 --- a/doc/development/fe_guide/merge_request_widget_extensions.md +++ b/doc/development/fe_guide/merge_request_widget_extensions.md @@ -355,7 +355,12 @@ To generate these known events for a single widget: 1. `redis_slot` = `code_review` 1. `category` = `code_review` 1. `aggregation` = `weekly` -1. Add each event to the appropriate aggregates in `config/metrics/aggregates/code_review.yml` +1. Add each event (those listed in the command in step 7, replacing `test_reports` + with the appropriate name slug) to the aggregate files: + 1. `config/metrics/counts_7d/{timestamp}_code_review_category_monthly_active_users.yml` + 1. `config/metrics/counts_7d/{timestamp}_code_review_group_monthly_active_users.yml` + 1. `config/metrics/counts_28d/{timestamp}_code_review_category_monthly_active_users.yml` + 1. `config/metrics/counts_28d/{timestamp}_code_review_group_monthly_active_users.yml` ### Add new events diff --git a/doc/development/fe_guide/security.md b/doc/development/fe_guide/security.md index 4b7ce6d11e3..d578449e578 100644 --- a/doc/development/fe_guide/security.md +++ b/doc/development/fe_guide/security.md @@ -88,7 +88,7 @@ readability. If you need to output raw HTML, you should sanitize it. -If you are using Vue, you can use the[`v-safe-html` directive](https://gitlab-org.gitlab.io/gitlab-ui/?path=/story/directives-safe-html-directive--default) from GitLab UI. +If you are using Vue, you can use the[`v-safe-html` directive](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/vue_shared/directives/safe_html.js). For other use cases, wrap a preconfigured version of [`dompurify`](https://www.npmjs.com/package/dompurify) that also allows the icons to be rendered: diff --git a/doc/development/fe_guide/source_editor.md b/doc/development/fe_guide/source_editor.md index b7cd903485e..4cfc68553e0 100644 --- a/doc/development/fe_guide/source_editor.md +++ b/doc/development/fe_guide/source_editor.md @@ -66,9 +66,9 @@ with additional functions on the instance level: | Function | Arguments | Description | --------------------- | ----- | ----- | -| `updateModelLanguage` | `path`: String | Updates the instance's syntax highlighting to follow the extension of the passed `path`. Available only on the instance level.| -| `use` | Array of objects | Array of extensions to apply to the instance. Accepts only the array of _objects_. You must fetch the extensions' ES6 modules must be fetched and resolved in your views or components before they are passed to `use`. This property is available on _instance_ (applies extension to this particular instance) and _global editor_ (applies the same extension to all instances) levels. | -| Monaco Editor options | See [documentation](https://microsoft.github.io/monaco-editor/api/interfaces/monaco.editor.IStandaloneCodeEditor.html) | Default Monaco editor options | +| `updateModelLanguage` | `path`: String | Updates the instance's syntax highlighting to follow the extension of the passed `path`. Available only on the instance level. | +| `use` | Array of objects | Array of extensions to apply to the instance. Accepts only an array of **objects**. The extensions' ES6 modules must be fetched and resolved in your views or components before they're passed to `use`. Available on the instance and global editor (all instances) levels. | +| Monaco Editor options | See [documentation](https://microsoft.github.io/monaco-editor/api/interfaces/monaco.editor.IStandaloneCodeEditor.html) | Default Monaco editor options. | ## Tips diff --git a/doc/development/fe_guide/style/javascript.md b/doc/development/fe_guide/style/javascript.md index 38fb926197b..0536d1c5c77 100644 --- a/doc/development/fe_guide/style/javascript.md +++ b/doc/development/fe_guide/style/javascript.md @@ -7,7 +7,7 @@ disqus_identifier: 'https://docs.gitlab.com/ee/development/fe_guide/style_guide_ # JavaScript style guide -We use [Airbnb's JavaScript Style Guide](https://github.com/airbnb/javascript) and its accompanying +We use [the Airbnb JavaScript Style Guide](https://github.com/airbnb/javascript) and its accompanying linter to manage most of our JavaScript style guidelines. In addition to the style guidelines set by Airbnb, we also have a few specific rules @@ -16,11 +16,11 @@ listed below. NOTE: You can run ESLint locally by running `yarn run lint:eslint:all` or `yarn run lint:eslint $PATH_TO_FILE`. -## Avoid forEach +## Avoid `forEach` -Avoid forEach when mutating data. Use `map`, `reduce` or `filter` instead of `forEach` +Avoid `forEach` when mutating data. Use `map`, `reduce` or `filter` instead of `forEach` when mutating data. This minimizes mutations in functions, -which aligns with [Airbnb's style guide](https://github.com/airbnb/javascript#testing--for-real). +which aligns with [the Airbnb style guide](https://github.com/airbnb/javascript#testing--for-real). ```javascript // bad diff --git a/doc/development/fe_guide/style/scss.md b/doc/development/fe_guide/style/scss.md index 98f74813231..7a5c955db93 100644 --- a/doc/development/fe_guide/style/scss.md +++ b/doc/development/fe_guide/style/scss.md @@ -41,13 +41,13 @@ GitLab differs from the scale used in the Bootstrap library. For a Bootstrap pad utility, you may need to double the size of the applied utility to achieve the same visual result (such as `ml-1` becoming `gl-ml-2`). -#### Where should I put new utility classes? +#### Where should you put new utility classes? -If a class you need has not been added to GitLab UI, you get to add it! Follow the naming patterns documented in the [utility files](https://gitlab.com/gitlab-org/gitlab-ui/-/tree/main/src/scss/utility-mixins) and refer to [GitLab UI's CSS documentation](https://gitlab.com/gitlab-org/gitlab-ui/-/blob/main/doc/contributing/adding_css.md#adding-utility-mixins) for more details, especially about adding responsive and stateful rules. +If a class you need has not been added to GitLab UI, you get to add it! Follow the naming patterns documented in the [utility files](https://gitlab.com/gitlab-org/gitlab-ui/-/tree/main/src/scss/utility-mixins) and refer to the [GitLab UI CSS documentation](https://gitlab.com/gitlab-org/gitlab-ui/-/blob/main/doc/contributing/adding_css.md#adding-utility-mixins) for more details, especially about adding responsive and stateful rules. If it is not possible to wait for a GitLab UI update (generally one day), add the class to [`utilities.scss`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/stylesheets/utilities.scss) following the same naming conventions documented in GitLab UI. A follow-up issue to backport the class to GitLab UI and delete it from GitLab should be opened. -#### When should I create component classes? +#### When should you create component classes? We recommend a "utility-first" approach. diff --git a/doc/development/fe_guide/tooling.md b/doc/development/fe_guide/tooling.md index c9efb8e939d..762ef852d74 100644 --- a/doc/development/fe_guide/tooling.md +++ b/doc/development/fe_guide/tooling.md @@ -212,7 +212,7 @@ yarn run lint:prettier:fix Formats all files in the repository with Prettier. -### VSCode Settings +### VS Code Settings #### Select Prettier as default formatter diff --git a/doc/development/fe_guide/troubleshooting.md b/doc/development/fe_guide/troubleshooting.md index 873a14b8f14..3e4c5007b64 100644 --- a/doc/development/fe_guide/troubleshooting.md +++ b/doc/development/fe_guide/troubleshooting.md @@ -10,13 +10,13 @@ Running into a problem? Maybe this will help ¯\＿(ツ)＿/¯. ## Troubleshooting issues -### This guide doesn't contain the issue I ran into +### This guide doesn't contain the issue you ran into If you run into a Frontend development issue that is not in this guide, consider updating this guide with your issue and possible remedies. This way future adventurers can face these dragons with more success, being armed with your experience and knowledge. ## Testing issues -### ``Property or method `nodeType` is not defined`` but I'm not using `nodeType` anywhere +### ``Property or method `nodeType` is not defined`` but you're not using `nodeType` anywhere This issue can happen in Vue component tests, when an expectation fails, but there is an error thrown when Jest tries to pretty print the diff in the console. It's been noted that using `toEqual` with an array as a diff --git a/doc/development/fe_guide/view_component.md b/doc/development/fe_guide/view_component.md index b61c23cadef..90bb75514d8 100644 --- a/doc/development/fe_guide/view_component.md +++ b/doc/development/fe_guide/view_component.md @@ -180,7 +180,7 @@ The `Pajamas::CheckboxComponent` follows the [Pajamas Checkbox](https://design.g NOTE: `Pajamas::CheckboxComponent` is used internally by the [GitLab UI form builder](haml.md#use-the-gitlab-ui-form-builder) and requires an instance of [ActionView::Helpers::FormBuilder](https://api.rubyonrails.org/v6.1.0/classes/ActionView/Helpers/FormBuilder.html) to be passed as the `form` argument. -It is preferred to use the [gitlab_ui_checkbox_component](haml.md#gitlab_ui_checkbox_component) method to render this ViewComponent. +It is preferred to use the [`gitlab_ui_checkbox_component`](haml.md#gitlab_ui_checkbox_component) method to render this ViewComponent. To use a checkbox without an instance of [ActionView::Helpers::FormBuilder](https://api.rubyonrails.org/v6.1.0/classes/ActionView/Helpers/FormBuilder.html) use [CheckboxTagComponent](#checkbox-tag). For the full list of options, see its diff --git a/doc/development/fe_guide/vue3_migration.md b/doc/development/fe_guide/vue3_migration.md index aae7674d190..b9e819a95bd 100644 --- a/doc/development/fe_guide/vue3_migration.md +++ b/doc/development/fe_guide/vue3_migration.md @@ -182,13 +182,13 @@ In the step-by-step guide, we will be migrating [VueApollo Demo](https://gitlab. #### Initial state -Right after cloning, you could run [VueApollo Demo](https://gitlab.com/gitlab-org/frontend/vue3-migration-vue-apollo/-/tree/main/src/vue3compat) with Vue.js 2 using `yarn serve` or with Vue.js 3 (compat build) using `yarn serve:vue3`. However latter immediately crashes: +Right after cloning, you could run [VueApollo Demo](https://gitlab.com/gitlab-org/frontend/vue3-migration-vue-apollo/-/tree/main/src/vue3compat) with Vue.js 2 using `yarn serve` or with Vue.js 3 (`compat` build) using `yarn serve:vue3`. However latter immediately crashes: ```javascript Uncaught TypeError: Cannot read properties of undefined (reading 'loading') ``` -VueApollo v3 (used for Vue.js 2) fails to initialize in Vue.js compat +VueApollo v3 (used for Vue.js 2) fails to initialize in Vue.js `compat` NOTE: While stubbing `Vue.version` will solve VueApollo-related issues in the demo project, it will still lose reactivity on specific scenarios, so an upgrade is still needed diff --git a/doc/development/fe_guide/widgets.md b/doc/development/fe_guide/widgets.md index 3364c778d76..edb8559da48 100644 --- a/doc/development/fe_guide/widgets.md +++ b/doc/development/fe_guide/widgets.md @@ -44,7 +44,7 @@ All editable sidebar widgets should use [`SidebarEditableItem`](https://gitlab.c We aim to make widgets as reusable as possible. That's why we should avoid adding any external state bindings to widgets or to their child components. This includes Vuex mappings and mediator stores. -## Widget's responsibility +## Widget responsibility A widget is responsible for fetching and updating an entity it's designed for (assignees, iterations, and so on). This means a widget should **always** fetch data (if it's not in Apollo cache already). @@ -78,7 +78,7 @@ To handle the same logic for query updates, we **alias** query fields. For examp - `group` or `project` become `workspace` - `issue`, `epic`, or `mergeRequest` become `issuable` -Unfortunately, Apollo assigns aliased fields a typename of `undefined`, so we need to fetch `__typename` explicitly: +Unfortunately, Apollo assigns aliased fields a `typename` of `undefined`, so we need to fetch `__typename` explicitly: ```plaintext query issueConfidential($fullPath: ID!, $iid: String) { diff --git a/doc/development/feature_categorization/index.md b/doc/development/feature_categorization/index.md index 0bf506b53ba..47663915ea7 100644 --- a/doc/development/feature_categorization/index.md +++ b/doc/development/feature_categorization/index.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/269) in GitLab 13.2. -Each Sidekiq worker, controller action, or API endpoint +Each Sidekiq worker, controller action, [test example](../testing_guide/best_practices.md#feature-category-metadata) or API endpoint must declare a `feature_category` attribute. This attribute maps each of these to a [feature category](https://about.gitlab.com/handbook/product/categories/). This is done for error budgeting, alert routing, and team attribution. @@ -166,3 +166,40 @@ end As with Rails controllers, an API class must specify the category for every single action unless the same category is used for every action within that class. + +## RSpec Examples + +You must set feature category metadata for each RSpec example. This information is used for flaky test +issues to identify the group that owns the feature. + +The `feature_category` should be a value from [`categories.json`](https://about.gitlab.com/categories.json). + +The `feature_category` metadata can be set: + +- [In the top-level `RSpec.describe` blocks](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104274/diffs#6bd01173381e873f3e1b6c55d33cdaa3d897156b_5_5). +- [In `describe` blocks](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104274/diffs#a520db2677a30e7f1f5593584f69c49031b894b9_12_12). + +Consider splitting the file in the case there are multiple feature categories identified in the same file. + +Example: + + ```ruby + RSpec.describe Admin::Geo::SettingsController, :geo, feature_category: :geo_replication do + ``` + +For examples that don't have a `feature_category` set we add a warning when running them in local environment. + +In order to disable the warning use `RSPEC_WARN_MISSING_FEATURE_CATEGORY=false` when running RSpec tests: + +```shell +RSPEC_WARN_MISSING_FEATURE_CATEGORY=false bin/rspec spec/<test_file> +``` + +### Excluding specs from feature categorization + +In the rare case an action cannot be tied to a feature category this +can be done using the `not_owned` feature category. + +```ruby +RSpec.describe Utils, feature_category: :not_owned do +``` diff --git a/doc/development/feature_development.md b/doc/development/feature_development.md index 76447124177..b9e093b9479 100644 --- a/doc/development/feature_development.md +++ b/doc/development/feature_development.md @@ -19,7 +19,7 @@ Consult these topics for information on contributing to specific GitLab features ### General -- [Directory structure](directory_structure.md) +- [Software design guides](software_design.md) - [GitLab EventStore](event_store.md) to publish/subscribe to domain events - [GitLab utilities](utilities.md) - [Newlines style guide](newlines_styleguide.md) @@ -54,7 +54,7 @@ Consult these topics for information on contributing to specific GitLab features ### Debugging - [Pry debugging](pry_debugging.md) -- [Sidekiq debugging](../administration/troubleshooting/sidekiq.md) +- [Sidekiq debugging](../administration/sidekiq/sidekiq_troubleshooting.md) ### Git specifics @@ -127,7 +127,7 @@ See [database guidelines](database/index.md). - [Security Scanners](integrations/secure.md) - [Secure Partner Integration](integrations/secure_partner_integration.md) - [How to run Jenkins in development environment](integrations/jenkins.md) -- [How to run local `Codesandbox` integration for Web IDE Live Preview](integrations/codesandbox.md) +- [How to run local CodeSandbox integration for Web IDE Live Preview](integrations/codesandbox.md) The following integration guides are internal. Some integrations require access to administrative accounts of third-party services and are available only for GitLab team members to contribute to: @@ -204,3 +204,4 @@ The following integration guides are internal. Some integrations require access - [Run full Auto DevOps cycle in a GDK instance](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/auto_devops.md) - [Using GitLab Runner with the GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/runner.md) - [Using the Web IDE terminal with the GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/web_ide_terminal_gdk_setup.md) +- [Gitpod configuration internals page](gitpod_internals.md) diff --git a/doc/development/feature_flags/controls.md b/doc/development/feature_flags/controls.md index e804a888e98..3e6491a92b5 100644 --- a/doc/development/feature_flags/controls.md +++ b/doc/development/feature_flags/controls.md @@ -67,10 +67,8 @@ a (very) rough estimate of how your feature will look and behave on GitLab.com. Both of these instances are connected to Sentry so make sure you check the projects there for any exceptions while testing your feature after enabling the feature flag. -For these pre-production environments, the commands should be run in a -Slack channel for the stage the feature is relevant to. For example, use the -`#s_monitor` channel for features developed by the Monitor stage, Health -group. +For these pre-production environments, it's strongly encouraged to run the command in +`#staging`, `#production`, or `#chatops-ops-test`, for improved visibility. To enable a feature for 25% of the time, run the following in Slack: @@ -289,10 +287,47 @@ To disable a feature flag that has been enabled for a specific project you can r /chatops run feature set --project=gitlab-org/gitlab some_feature false ``` -You cannot selectively disable feature flags for a specific project/group/user without applying a [specific method of implementing](index.md#selectively-disable-by-actor) the feature flags. +You cannot selectively disable feature flags for a specific project/group/user without applying a [specific method of implementing](controls.md#selectively-disable-by-actor) the feature flags. If a feature flag is disabled via ChatOps, that will take precedence over the `default_enabled` value in the YAML. In other words, you could have a feature enabled for on-premise installations but not for GitLab.com. +#### Selectively disable by actor + +By default you cannot selectively disable a feature flag by actor. + +```shell +# This will not work how you would expect. +/chatops run feature set some_feature true +/chatops run feature set --project=gitlab-org/gitlab some_feature false +``` + +However, if you add two feature flags, you can write your conditional statement in such a way that the equivalent selective disable is possible. + +```ruby +Feature.enabled?(:a_feature, project) && Feature.disabled?(:a_feature_override, project) +``` + +```shell +# This will enable a feature flag globally, except for gitlab-org/gitlab +/chatops run feature set a_feature true +/chatops run feature set --project=gitlab-org/gitlab a_feature_override true +``` + +#### Percentage-based actor selection + +When using the percentage rollout of actors on multiple feature flags, the actors for each feature flag are selected separately. + +For example, the following feature flags are enabled for a certain percentage of actors: + +```plaintext +/chatops run feature set feature-set-1 25 --actors +/chatops run feature set feature-set-2 25 --actors +``` + +If a project A has `:feature-set-1` enabled, there is no guarantee that project A also has `:feature-set-2` enabled. + +For more detail, see [This is how percentages work in Flipper](https://www.hackwithpassion.com/this-is-how-percentages-work-in-flipper/). + ### Feature flag change logging #### ChatOps level diff --git a/doc/development/feature_flags/index.md b/doc/development/feature_flags/index.md index 500afa8ba1d..5ff4292dfb6 100644 --- a/doc/development/feature_flags/index.md +++ b/doc/development/feature_flags/index.md @@ -187,7 +187,7 @@ Only feature flags that have a YAML definition file can be used when running the ```shell $ bin/feature-flag my_feature_flag >> Specify the group introducing the feature flag, like `group::apm`: -?> group::memory +?> group::application performance >> URL of the MR introducing the feature flag (enter to skip): ?> https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38602 @@ -202,7 +202,7 @@ create config/feature_flags/development/my_feature_flag.yml name: my_feature_flag introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38602 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/232533 -group: group::memory +group: group::application performance type: development default_enabled: false ``` @@ -398,44 +398,8 @@ Feature.enabled?(:feature_flag, group) Feature.enabled?(:feature_flag, user) ``` -Please see [Feature flag controls](controls.md#process) for more details on working with feature flags. - -#### Selectively disable by actor - -By default you cannot selectively disable a feature flag by actor. - -```shell -# This will not work how you would expect. -/chatops run feature set some_feature true -/chatops run feature set --project=gitlab-org/gitlab some_feature false -``` - -However, if you add two feature flags, you can write your conditional statement in such a way that the equivalent selective disable is possible. - -```ruby -Feature.enabled?(:a_feature, project) && Feature.disabled?(:a_feature_override, project) -``` - -```shell -# This will enable a feature flag globally, except for gitlab-org/gitlab -/chatops run feature set a_feature true -/chatops run feature set --project=gitlab-org/gitlab a_feature_override true -``` - -#### Percentage-based actor selection - -When using the percentage rollout of actors on multiple feature flags, the actors for each feature flag are selected separately. - -For example, the following feature flags are enabled for a certain percentage of actors: - -```plaintext -/chatops run feature set feature-set-1 25 --actors -/chatops run feature set feature-set-2 25 --actors -``` - -If a project A has `:feature-set-1` enabled, there is no guarantee that project A also has `:feature-set-2` enabled. - -For more detail, see [This is how percentages work in Flipper](https://www.hackwithpassion.com/this-is-how-percentages-work-in-flipper/). +See [Feature flags in the development of GitLab](controls.md#process) for details on how to use ChatOps +to selectively enable or disable feature flags in GitLab-provided environments, like staging and production. #### Use actors for verifying in production diff --git a/doc/development/file_storage.md b/doc/development/file_storage.md index 11acb8a2161..c346d55f639 100644 --- a/doc/development/file_storage.md +++ b/doc/development/file_storage.md @@ -36,7 +36,7 @@ There are many places where file uploading is used, according to contexts: GitLab started saving everything on local disk. While directory location changed from previous versions, they are still not 100% standardized. You can see them below: -| Description | In DB? | Relative path (from CarrierWave.root) | Uploader class | model_type | +| Description | In DB? | Relative path (from CarrierWave.root) | Uploader class | Model type | | ------------------------------------- | ------ | ----------------------------------------------------------- | ---------------------- | ---------- | | Instance logo | yes | `uploads/-/system/appearance/logo/:id/:filename` | `AttachmentUploader` | Appearance | | Header logo | yes | `uploads/-/system/appearance/header_logo/:id/:filename` | `AttachmentUploader` | Appearance | diff --git a/doc/development/filtering_by_label.md b/doc/development/filtering_by_label.md deleted file mode 100644 index 675fe004c22..00000000000 --- a/doc/development/filtering_by_label.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/filtering_by_label.md' -remove_date: '2022-11-05' ---- - -This document was moved to [another location](database/filtering_by_label.md). - -<!-- This redirect file can be deleted after <2022-11-05>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/fips_compliance.md b/doc/development/fips_compliance.md index c6208d45c77..187a9b0cc93 100644 --- a/doc/development/fips_compliance.md +++ b/doc/development/fips_compliance.md @@ -69,6 +69,7 @@ listed here that also do not work properly in FIPS mode: when operating in FIPS-compliant mode. - Advanced Search is currently not included in FIPS mode. It must not be enabled to be FIPS-compliant. - [Gravatar or Libravatar-based profile images](../administration/libravatar.md) are not FIPS-compliant. +- [Personal Access Tokens](../user/profile/personal_access_tokens.md) are not available for use or creation. Additionally, these package repositories are disabled in FIPS mode: @@ -441,13 +442,27 @@ def default_min_key_size(name) end ``` -## Nightly Omnibus FIPS builds +## Omnibus FIPS packages -The Distribution team has created [nightly FIPS Omnibus builds](https://packages.gitlab.com/gitlab/nightly-fips-builds). These -GitLab builds are compiled to use the system OpenSSL instead of the Omnibus-embedded version of OpenSSL. +GitLab has a dedicated repository +([`gitlab/gitlab-fips`](https://packages.gitlab.com/gitlab/gitlab-fips)) +for builds of the Omnibus GitLab which are built with FIPS compliance. +These GitLab builds are compiled to use the system OpenSSL, instead of +the Omnibus-embedded version of OpenSSL. These packages are built for: + +- RHEL 8 (and compatible) +- AmazonLinux 2 +- Ubuntu + +These are [consumed by the GitLab Environment Toolkit](#install-gitlab-with-fips-compliance) (GET). See [the section on how FIPS builds are created](#how-fips-builds-are-created). +### Nightly Omnibus FIPS builds + +The Distribution team has created [nightly FIPS Omnibus builds](https://packages.gitlab.com/gitlab/nightly-fips-builds), +which can be used for *testing* purposes. These should never be used for production environments. + ## Runner See the [documentation on installing a FIPS-compliant GitLab Runner](https://docs.gitlab.com/runner/install/#fips-compliant-gitlab-runner). diff --git a/doc/development/foreign_keys.md b/doc/development/foreign_keys.md deleted file mode 100644 index cdf655bf0bf..00000000000 --- a/doc/development/foreign_keys.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/foreign_keys.md' -remove_date: '2022-11-05' ---- - -This document was moved to [another location](database/foreign_keys.md). - -<!-- This redirect file can be deleted after <2022-11-05>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/gemfile.md b/doc/development/gemfile.md index 3c7dc19da8e..36ef1bcd834 100644 --- a/doc/development/gemfile.md +++ b/doc/development/gemfile.md @@ -47,7 +47,8 @@ This needs to be done for any new, or updated gems. We do not allow gems that are fetched from Git repositories. All gems have to be available in the RubyGems index. We want to minimize external build -dependencies and build times. +dependencies and build times. It's enforced by the RuboCop rule +[`Cop/GemFetcher`](https://gitlab.com/gitlab-org/ruby/gems/gitlab-styles/-/blob/master/lib/rubocop/cop/gem_fetcher.rb). ## Review the new dependency for quality @@ -56,7 +57,7 @@ This means that new dependencies should, at a minimum, meet the following criter - They have an active developer community. At the minimum a maintainer should still be active to merge change requests in case of emergencies. -- There are no issues open that we know may impact the availablity or performance of GitLab. +- There are no issues open that we know may impact the availability or performance of GitLab. - The project is tested using some form of test automation. The test suite must be passing using the Ruby version currently used by GitLab. - If the project uses a C extension, consider requesting an additional review from a C or MRI diff --git a/doc/development/geo.md b/doc/development/geo.md index 884c09cc174..76c75cb1c6a 100644 --- a/doc/development/geo.md +++ b/doc/development/geo.md @@ -200,7 +200,7 @@ sequenceDiagram S->>TDB: Insert to `job_artifact_registry` ``` -- [Sidekiq-cron](https://github.com/ondrejbartas/sidekiq-cron) enqueues a `Geo::Secondary::RegistryConsistencyWorker` job every minute. As long as it is actively doing work (creating and deleting rows), this job immediately reenqueues itself. This job uses an exclusive lease to prevent multiple instances of itself from running simultaneously. +- [Sidekiq-cron](https://github.com/ondrejbartas/sidekiq-cron) enqueues a `Geo::Secondary::RegistryConsistencyWorker` job every minute. As long as it is actively doing work (creating and deleting rows), this job immediately re-enqueues itself. This job uses an exclusive lease to prevent multiple instances of itself from running simultaneously. - [Sidekiq](architecture.md#sidekiq) picks up `Geo::Secondary::RegistryConsistencyWorker` job - Sidekiq queries `ci_job_artifacts` table for up to 10000 rows - Sidekiq queries `job_artifact_registry` table for up to 10000 rows @@ -308,7 +308,7 @@ sequenceDiagram - Sidekiq queries `job_artifact_registry` in the [PostgreSQL Geo Tracking Database](#tracking-database) for the number of rows marked "pending verification" or "failed verification and ready to retry" - Sidekiq enqueues one or more `Geo::VerificationBatchWorker` jobs, limited by the "maximum verification concurrency" setting - Sidekiq picks up `Geo::VerificationBatchWorker` job - - Sidekiq queries `job_artifact_registry` in the PostgreSQL Geo Tracking Databasef for rows marked "pending verification" + - Sidekiq queries `job_artifact_registry` in the PostgreSQL Geo Tracking Database for rows marked "pending verification" - If the previous step yielded less than 10 rows, then Sidekiq queries `job_artifact_registry` for rows marked "failed verification and ready to retry" - For each row - Sidekiq marks it "started verification" @@ -588,23 +588,45 @@ When some write actions are not allowed because the site is a The database itself will already be read-only in a replicated setup, so we don't need to take any extra step for that. -## Steps needed to replicate a new data type - -As GitLab evolves, we constantly need to add new resources to the Geo replication system. -The implementation depends on resource specifics, but there are several things -that need to be taken care of: - -- Event generation on the primary site. Whenever a new resource is changed/updated, we need to - create a task for the Log Cursor. -- Event handling. The Log Cursor needs to have a handler for every event type generated by the primary site. -- Dispatch worker (cron job). Make sure the backfill condition works well. -- Sync worker. -- Registry with all possible states. -- Verification. -- Cleaner. When sync settings are changed for the secondary site, some resources need to be cleaned up. -- Geo Node Status. We need to provide API endpoints as well as some presentation in the GitLab Admin Area. -- Health Check. If we can perform some pre-cheсks and make site unhealthy if something is wrong, we should do that. - The `rake gitlab:geo:check` command has to be updated too. +## Ensuring a new feature has Geo support + +Geo depends on PostgreSQL replication of the main and CI databases, so if you add a new table or field, it should already work on secondary Geo sites. + +However, if you introduce a new kind of data which is stored outside of the main and CI PostgreSQL databases, then you need to ensure that this data is replicated and verified by Geo. This is necessary for customers to be able to rely on their secondary sites for [disaster recovery](../administration/geo/disaster_recovery/index.md). + +The following subsections describe how to determine whether work is needed, and if so, how to proceed. If you have any questions, [contact the Geo team](https://about.gitlab.com/handbook/product/categories/#geo-group). + +For comparison with your own features, see [Supported Geo data types](../administration/geo/replication/datatypes.md). It has a detailed, up-to-date list of the types of data that Geo replicates and verifies. + +### Git repositories + +If you add a feature that is backed by Git repositories, then you must add Geo support. See [the repository replicator strategy of the Geo self-service framework](geo/framework.md#repository-replicator-strategy). + +### Blobs + +If you add a subclass of `CarrierWave::Uploader::Base`, then you are adding what Geo calls a blob. If you specifically subclass [`AttachmentUploader` as generally recommended](uploads/working_with_uploads.md#recommendations), then the data has Geo support with no work needed. This is because `AttachmentUploader` tracks blobs with the `Upload` model using the `uploads` table, and Geo support is already implemented for that model. + +If your blobs are tracked in a new table, perhaps because you expect millions of rows at GitLab.com scale, then you must add Geo support. See [the blob replicator strategy of the Geo self-service framework](geo/framework.md#blob-replicator-strategy). + +[Geo detects new blobs with a spec](https://gitlab.com/gitlab-org/gitlab/-/blob/eeba0e4d231ae39012a5bbaeac43a72c2bd8affb/ee/spec/uploaders/every_gitlab_uploader_spec.rb) that fails when an `Uploader` does not have a corresponding `Replicator`. + +### Features with more than one kind of data + +If a new complex feature is backed by multiple kinds of data, for example, a Git repository and a blob, then you can likely consider each kind of data separately. + +Taking [Designs](../user/project/issues/design_management.md) as an example, each issue has a Git repository which can have many LFS objects, and each LFS object may have an automatically generated thumbnail. + +- LFS objects were already supported by Geo, so no Geo-specific work was needed. +- The implementation of thumbnails reused the `Upload` model, so no Geo-specific work was needed. +- Design Git repositories were not inherently supported by Geo, so work was needed. + +As another example, [Dependency Proxy](../administration/packages/dependency_proxy.md) is backed by two kinds of blobs, `DependencyProxy::Blob` and `DependencyProxy::Manifest`. We can use [the blob replicator strategy of the Geo self-service framework](geo/framework.md#blob-replicator-strategy) on each type, independent of each other. + +### Other kinds of data + +If a new feature introduces a new kind of data which is not a Git repository, or a blob, or a combination of the two, then contact the Geo team to discuss how to handle it. + +As an example, Container Registry data does not easily fit into the above categories. It is backed by a registry service which owns the data, and GitLab interacts with the registry service's API. So a one off approach is required for Geo support of Container Registry. Still, we are able to reuse much of the glue code of [the Geo self-service framework](geo/framework.md#repository-replicator-strategy). ## History of communication channel diff --git a/doc/development/geo/framework.md b/doc/development/geo/framework.md index 3624d280f86..60529db5ce6 100644 --- a/doc/development/geo/framework.md +++ b/doc/development/geo/framework.md @@ -18,6 +18,14 @@ across Geo sites. This API is presented as a Ruby Domain-Specific Language (DSL) and aims to make it possible to replicate data with minimal effort of the engineer who created a data type. +## Geo is a requirement in the definition of done + +Geo is the GitLab solution for [disaster recovery](https://about.gitlab.com/direction/geo/disaster_recovery/). A robust disaster recovery solution must replicate **all GitLab data** such that all GitLab services can be successfully restored in their entirety with minimal data loss in the event of a disaster. + +For this reason, Geo replication and verification support for GitLab generated data is part of the [definition of done](../contributing/merge_request_workflow.md#definition-of-done). This ensures that new features ship with Geo support and our customers are not exposed to data loss. + +Adding Geo support with the Self Service Framework (SSF) is easy and outlined in detail on this page for various types of data. However, for a more general guide that can help you decide if and how you need to add Geo support for a new GitLab feature, [you may start here](../geo.md#ensuring-a-new-feature-has-geo-support). + ## Nomenclature Before digging into the API, developers need to know some Geo-specific diff --git a/doc/development/geo/proxying.md b/doc/development/geo/proxying.md index 67d4129a9d0..f3136890788 100644 --- a/doc/development/geo/proxying.md +++ b/doc/development/geo/proxying.md @@ -260,12 +260,7 @@ S-->>C: return Git response from primary ## Git push -### Unified URLs - -With unified URLs, a push will redirect to a local path formatted as `/-/push_from_secondary/$SECONDARY_ID/*`. Further -requests through this path will be proxied to the primary, which will handle the push. - -#### Git push over SSH +### Git push over SSH As SSH operations go through GitLab Shell instead of Workhorse, they are not proxied through the mechanism used for Workhorse requests. With SSH operations, they are proxied as Git HTTP requests to the primary site by the secondary @@ -293,7 +288,12 @@ I-->>S: <response> S-->>C: return Git response from primary ``` -#### Git push over HTTP(s) +### Git push over HTTP(S) + +#### Git push over HTTP(S) unified URLs + +With unified URLs, a push redirects to a local path formatted as `/-/push_from_secondary/$SECONDARY_ID/*`. Further +requests through this path are proxied to the primary, which will handle the push. ```mermaid sequenceDiagram @@ -326,7 +326,7 @@ W-->>Wsec: Pipe messages to the Git client Wsec-->>C: Return piped messages from Git ``` -### Git push over HTTP with Separate URLs +#### Git push over HTTP(S) with separate URLs With separate URLs, the secondary will redirect to a URL formatted like `$PRIMARY/-/push_from_secondary/$SECONDARY_ID/*`. diff --git a/doc/development/git_object_deduplication.md b/doc/development/git_object_deduplication.md index dcfcd2e864a..6014ccbfb39 100644 --- a/doc/development/git_object_deduplication.md +++ b/doc/development/git_object_deduplication.md @@ -171,7 +171,7 @@ There are three different things that can go wrong here. #### 1. SQL says repository A belongs to pool P but Gitaly says A has no alternate objects -In this case, we miss out on disk space savings but all RPC's on A +In this case, we miss out on disk space savings but all RPCs on A itself function fine. The next time garbage collection runs on A, the alternates connection gets established in Gitaly. This is done by `Projects::GitDeduplicationService` in GitLab Rails. diff --git a/doc/development/gitaly.md b/doc/development/gitaly.md index a570b5dd7eb..a23f1dd2d80 100644 --- a/doc/development/gitaly.md +++ b/doc/development/gitaly.md @@ -56,7 +56,7 @@ they are merged. ### `gitaly-ruby` -It is possible to implement and test RPC's in Gitaly using Ruby code, +It is possible to implement and test RPCs in Gitaly using Ruby code, in [`gitaly-ruby`](https://gitlab.com/gitlab-org/gitaly/tree/master/ruby). This should make it easier to contribute for developers who are less @@ -324,7 +324,7 @@ the integration by using GDK: 1. The state of the flag must be observable. To check it, you must enable it by fetching the Prometheus metrics: - 1. Navigate to GDK's root directory. + 1. Navigate to the GDK root directory. 1. Make sure you have the proper branch checked out for Gitaly. 1. Recompile it with `make gitaly-setup` and restart the service with `gdk restart gitaly`. 1. Make sure your setup is running: `gdk status | grep praefect`. @@ -342,7 +342,7 @@ the integration by using GDK: 1. After you observe the metrics for the new feature flag and it increments, you can enable the new feature: - 1. Navigate to GDK's root directory. + 1. Navigate to the GDK root directory. 1. Start a Rails console: ```shell diff --git a/doc/development/github_importer.md b/doc/development/github_importer.md index edf78d5f83d..17b4a28f57d 100644 --- a/doc/development/github_importer.md +++ b/doc/development/github_importer.md @@ -192,7 +192,7 @@ briefly waits for jobs to complete before deciding what the next action should be. For small projects, this may slow down the import process a bit, but it also reduces pressure on the system as a whole. -## Refreshing import JIDs +## Refreshing import job IDs GitLab includes a worker called `Gitlab::Import::StuckProjectImportJobsWorker` that periodically runs and marks project imports as failed if they have been @@ -203,7 +203,7 @@ often we hit the GitHub rate limit (more on this below), but we don't want To prevent this from happening we periodically refresh the expiration time of the import process. This works by storing the JID of the import job in the -database, then refreshing this JID's TTL at various stages throughout the import +database, then refreshing this JID TTL at various stages throughout the import process. This is done by calling `ProjectImportState#refresh_jid_expiration`. By refreshing this TTL we can ensure our import does not get marked as failed so long we're still performing work. diff --git a/doc/development/gitlab_flavored_markdown/specification_guide/index.md b/doc/development/gitlab_flavored_markdown/specification_guide/index.md index 17afebcf6ee..85c6653180b 100644 --- a/doc/development/gitlab_flavored_markdown/specification_guide/index.md +++ b/doc/development/gitlab_flavored_markdown/specification_guide/index.md @@ -157,7 +157,7 @@ official specifications, but are part of the GitHub and GitLab internal Markdown implementations. These internal extensions are often dependent upon the GitHub or GitLab implementations or environments, and may depend upon metadata which is only available via interacting with those environments. For example, -[GitHub supports GitHub-specific autolinked references](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/autolinked-references-and-urls), +[GitHub supports GitHub-specific automatically linked references](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/autolinked-references-and-urls), and [GitLab also supports GitLab-specific references](../../../user/markdown.md#gitlab-specific-references). These may also be implemented by third-party Markdown rendering engines which integrate with @@ -458,7 +458,7 @@ You can see the RSpec shared context containing these fixtures in In some cases, fixtures may not be usable, because they do not provide control over the varying values. In these cases, we can introduce support for a environment variable into the production -code, which allows us to override the randommness in our test environment when we are +code, which allows us to override the randomness in our test environment when we are generating the HTML for footnote examples. Even though it is in the production code path, it has no effect unless it is explicitly set, therefore it is innocuous. It allows us to avoid the more-complex regex-based normalization described below. @@ -1056,7 +1056,7 @@ allows control over other aspects of the snapshot example generation process. the example will only be run by `ee/spec/requests/api/markdown_snapshot_spec.rb`, not by `spec/requests/api/markdown_snapshot_spec.rb`. - The `api_request_override_path` field overrides the API endpoint path which is used to - generate the `static` HTML for the specifed example. Different endpoints can generate different + generate the `static` HTML for the specified example. Different endpoints can generate different HTML in some cases, so we want to be able to exercise different API endpoints for the same Markdown. By default, the `/markdown` endpoint is used. @@ -1149,7 +1149,7 @@ These files are Markdown specification files containing examples generated based similar to the `output_spec/spec.txt` and `output_spec/spec.html`, with the following differences: 1. They contain a superset of _all_ examples from - the Commonmark, GitHub Flavored Markdown, and GitLab Flavored Markdown specifications, whereas + the CommonMark, GitHub Flavored Markdown, and GitLab Flavored Markdown specifications, whereas `spec.*` only contains the GLFM specification. This is to provide a single place to refer to all examples when working with [snapshot testing](#markdown-snapshot-testing). 1. They contain _only_ header sections which contain examples. They do not contain any prose-only diff --git a/doc/development/gitpod_internals.md b/doc/development/gitpod_internals.md new file mode 100644 index 00000000000..a4674df758d --- /dev/null +++ b/doc/development/gitpod_internals.md @@ -0,0 +1,30 @@ +--- +stage: none +group: Engineering Productivity +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Gitpod internal configuration + +## Settings + +The settings for `gitlab-org/gitlab` are defined under a [project's settings in a Gitpod dashboard](https://gitpod.io/t/gitlab-org/gitlab/settings). To view the settings, you must first join the `gitlab-org` team on [Gitpod.io](https://gitpod.io/). You can join the team using the bookmark link at the top of `#gitpod-gdk` internal Slack channel. + +The current settings are: + +- `Enable Incremental Prebuilds`: Uses an earlier successful prebuild to create new prebuilds faster. +- `Use Last Successful Prebuild`: Skips waiting for prebuilds currently in progress and uses the last successful prebuild from previous commits on the same branch. + +## Webhooks + +A webhook that starts with `https://gitpod.io/` is created to enable prebuilds (see [Enabling Prebuilds](https://www.gitpod.io/docs/configure/authentication/gitlab#enabling-prebuilds) for more details). The webhook is maintained by an [Engineering Productivity team](https://about.gitlab.com/handbook/engineering/quality/engineering-productivity/). + +You can find this webhook in [Webhook Settings in `gitlab-org/gitlab`](https://gitlab.com/gitlab-org/gitlab/-/hooks). If you cannot access this setting, please chat to the [Engineering Productivity team](https://about.gitlab.com/handbook/engineering/quality/engineering-productivity/). + +### Troubleshooting a failed webhook + +If a webhook failed to connect for a long time, then it may have been disabled in the project. + +To re-enable a failing or failed webhook, send a test request in [Webhook Settings](https://gitlab.com/gitlab-org/gitlab/-/hooks). See [Re-enable disabled webhooks page](https://docs.gitlab.com/15.4/ee/user/project/integrations/webhooks.html#re-enable-disabled-webhooks) for more details. + +After re-enabling, check the prebuilds' health in a [project's prebuilds](https://gitpod.io/t/gitlab-org/gitlab/prebuilds) and confirm that prebuilds start without any errors. diff --git a/doc/development/go_guide/index.md b/doc/development/go_guide/index.md index b561ebc4285..e7e628f5293 100644 --- a/doc/development/go_guide/index.md +++ b/doc/development/go_guide/index.md @@ -391,7 +391,7 @@ functionality: This gives us a thin abstraction over underlying implementations that is consistent across Workhorse, Gitaly, and, in future, other Go servers. For example, in the case of `gitlab.com/gitlab-org/labkit/tracing` we can switch -from using `Opentracing` directly to using `Zipkin` or Gokit's own tracing wrapper +from using `Opentracing` directly to using `Zipkin` or the Go kit's own tracing wrapper without changes to the application code, while still keeping the same consistent configuration mechanism (that is, the `GITLAB_TRACING` environment variable). diff --git a/doc/development/graphql_guide/batchloader.md b/doc/development/graphql_guide/batchloader.md index ef0b97f4f62..6a716de61b8 100644 --- a/doc/development/graphql_guide/batchloader.md +++ b/doc/development/graphql_guide/batchloader.md @@ -180,7 +180,7 @@ def resolve(args = {}, context = { current_user: current_user }) end ``` -We can also use [QueryRecorder](../query_recorder.md) to make sure we are performing only **one SQL query** per call. +We can also use [QueryRecorder](../database/query_recorder.md) to make sure we are performing only **one SQL query** per call. ```ruby it 'executes only 1 SQL query' do diff --git a/doc/development/hash_indexes.md b/doc/development/hash_indexes.md deleted file mode 100644 index 2a9f7e5a25d..00000000000 --- a/doc/development/hash_indexes.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/hash_indexes.md' -remove_date: '2022-11-06' ---- - -This document was moved to [another location](database/hash_indexes.md). - -<!-- This redirect file can be deleted after <2022-11-06>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index 91e2efcb2a3..2269a28e496 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -284,9 +284,7 @@ expect(wrapper.text()).toEqual(MSG_ALERT_SETTINGS_FORM_ERROR); ### Dynamic translations -Sometimes there are dynamic translations that the parser can't find when running -`bin/rake gettext:find`. For these scenarios you can use the [`N_` method](https://github.com/grosser/gettext_i18n_rails/blob/c09e38d481e0899ca7d3fc01786834fa8e7aab97/Readme.md#unfound-translations-with-rake-gettextfind). -There's also an alternative method to [translate messages from validation errors](https://github.com/grosser/gettext_i18n_rails/blob/c09e38d481e0899ca7d3fc01786834fa8e7aab97/Readme.md#option-a). +For more details you can see how we [keep translations dynamic](#keep-translations-dynamic). ## Working with special content @@ -764,6 +762,10 @@ class MyPresenter end ``` +Sometimes there are dynamic translations that the parser can't find when running +`bin/rake gettext:find`. For these scenarios you can use the [`N_` method](https://github.com/grosser/gettext_i18n_rails/blob/c09e38d481e0899ca7d3fc01786834fa8e7aab97/Readme.md#unfound-translations-with-rake-gettextfind). +There's also an alternative method to [translate messages from validation errors](https://github.com/grosser/gettext_i18n_rails/blob/c09e38d481e0899ca7d3fc01786834fa8e7aab97/Readme.md#option-a). + ### Splitting sentences Never split a sentence, as it assumes the sentence's grammar and structure is the same in all diff --git a/doc/development/image_scaling.md b/doc/development/image_scaling.md index 4b19c21a457..502a18fecd7 100644 --- a/doc/development/image_scaling.md +++ b/doc/development/image_scaling.md @@ -1,6 +1,6 @@ --- -stage: Data Stores -group: Application Performance +stage: Manage +group: Workspace info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -26,7 +26,7 @@ Whether we allow an image to be rescaled or not is decided by combination of har The hard-coded rules only permit: - [Project, group and user avatars](https://gitlab.com/gitlab-org/gitlab/-/blob/fd08748862a5fe5c25b919079858146ea85843ae/app/controllers/concerns/send_file_upload.rb#L65-67) -- [PNGs or JPEGs](https://gitlab.com/gitlab-org/gitlab/-/blob/5dff8fa3814f2a683d8884f468cba1ec06a60972/lib/gitlab/file_type_detection.rb#L23) +- [PNG or JPEG images](https://gitlab.com/gitlab-org/gitlab/-/blob/5dff8fa3814f2a683d8884f468cba1ec06a60972/lib/gitlab/file_type_detection.rb#L23) - [Specific dimensions](https://gitlab.com/gitlab-org/gitlab/-/blob/5dff8fa3814f2a683d8884f468cba1ec06a60972/app/models/concerns/avatarable.rb#L6) Furthermore, configuration in Workhorse can lead to the image scaler rejecting a request if: diff --git a/doc/development/import_project.md b/doc/development/import_project.md index b879d635350..7f5a0faf8fb 100644 --- a/doc/development/import_project.md +++ b/doc/development/import_project.md @@ -57,7 +57,7 @@ This method takes longer to import than the other methods and depends on several This script was introduced in GitLab 12.6 for importing large GitLab project exports. -As part of this script we also disable direct and background upload to avoid situations where a huge archive is being uploaded to GCS (while being inside a transaction, which can cause idle transaction timeouts). +As part of this script we also disable direct upload to avoid situations where a huge archive is being uploaded to GCS (while being inside a transaction, which can cause idle transaction timeouts). We can run this script from the terminal: diff --git a/doc/development/insert_into_tables_in_batches.md b/doc/development/insert_into_tables_in_batches.md deleted file mode 100644 index ced5332e880..00000000000 --- a/doc/development/insert_into_tables_in_batches.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/insert_into_tables_in_batches.md' -remove_date: '2022-11-05' ---- - -This document was moved to [another location](database/insert_into_tables_in_batches.md). - -<!-- This redirect file can be deleted after <2022-11-05>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/integrations/codesandbox.md b/doc/development/integrations/codesandbox.md index d7d0ea48db0..b7fe3fbd1c4 100644 --- a/doc/development/integrations/codesandbox.md +++ b/doc/development/integrations/codesandbox.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Set up local CodeSandbox development environment This guide walks through setting up a local [CodeSandbox repository](https://github.com/codesandbox/codesandbox-client) and integrating it with a local GitLab instance. CodeSandbox -is used to power the Web IDE's [Live Preview feature](../../user/project/web_ide/index.md#live-preview). Having a local CodeSandbox setup is useful for debugging upstream issues or +is used to power the Web IDE [Live Preview feature](../../user/project/web_ide/index.md#live-preview). Having a local CodeSandbox setup is useful for debugging upstream issues or creating upstream contributions like [this one](https://github.com/codesandbox/codesandbox-client/pull/5137). ## Initial setup @@ -43,7 +43,7 @@ Before using CodeSandbox with your local GitLab instance, you must: GitLab integrates with two parts of CodeSandbox: - An npm package called `smooshpack` (called `sandpack` in the `codesandbox-client` project). - This exposes an entrypoint for us to kick off Codesandbox's bundler. + This exposes an entrypoint for us to kick off CodeSandbox's bundler. - A server that houses CodeSandbox assets for bundling and previewing. This is hosted on a separate server for security. diff --git a/doc/development/integrations/index.md b/doc/development/integrations/index.md index 2639da818c6..1c9144a1163 100644 --- a/doc/development/integrations/index.md +++ b/doc/development/integrations/index.md @@ -58,7 +58,7 @@ end `Integration.prop_accessor` installs accessor methods on the class. Here we would have `#url`, `#url=` and `#url_changed?`, to manage the `url` field. Fields stored in `Integration#properties` should be accessed by these accessors directly on the model, just like other ActiveRecord attributes. -You should always access the properties through their getters, and not interact with the `properties` hash directly. +You should always access the properties through their `getters`, and not interact with the `properties` hash directly. You **must not** write to the `properties` hash, you **must** use the generated setter method instead. Direct writes to this hash are not persisted. @@ -124,6 +124,39 @@ module Integrations end ``` +## Define configuration test + +Optionally, you can define a configuration test of an integration's settings. The test is executed from the integration form's **Test** button, and results are returned to the user. + +A good configuration test: + +- Does not change data on the service. For example, it should not trigger a CI build. Sending a message is okay. +- Is meaningful and as thorough as possible. + +If it's not possible to follow the above guidelines, consider not adding a configuration test. + +To add a configuration test, define a `#test` method for the integration model. + +The method receives `data`, which is a test push event payload. +It should return a hash, containing the keys: + +- `success` (required): a boolean to indicate if the configuration test has passed. +- `result` (optional): a message returned to the user if the configuration test has failed. + +For example: + +```ruby +module Integrations + class FooBar < Integration + def test(data) + success = test_api_key(data) + + { success: success, result: 'API key is invalid' } + end + end +end +``` + ### Customize the frontend form The frontend form is generated dynamically based on metadata defined in the model. @@ -282,6 +315,8 @@ You can also refer to our general [documentation guidelines](../documentation/in ## Testing +Testing should not be confused with [defining configuration tests](#define-configuration-test). + It is often sufficient to add tests for the integration model in `spec/models/integrations`, and a factory with example settings in `spec/factories/integrations.rb`. diff --git a/doc/development/integrations/jira_connect.md b/doc/development/integrations/jira_connect.md index d4215662db4..c7bb77a6a5d 100644 --- a/doc/development/integrations/jira_connect.md +++ b/doc/development/integrations/jira_connect.md @@ -59,6 +59,19 @@ To install the app in Jira: _Note that any changes to the app descriptor requires you to uninstall then reinstall the app._ +## Simple setup + +To avoid external dependencies like Gitpod and a Jira Cloud instance, use the [Jira connect test tool](https://gitlab.com/gitlab-org/manage/integrations/jira-connect-test-tool) and your local GDK: + +1. Clone the [**Jira-connect-test-tool**](https://gitlab.com/gitlab-org/manage/integrations/jira-connect-test-tool) `git clone git@gitlab.com:gitlab-org/manage/integrations/jira-connect-test-tool.git`. +1. Start the app `bundle exec rackup`. (The app requires your GDK GitLab to be available on `http://127.0.0.1:3000`.). +1. Open `config/gitlab.yml` and uncomment the `jira_connect` config. +1. Restart GDK. +1. Go to `http://127.0.0.1:3000/-/profile/personal_access_tokens`. +1. Create a new token with the `api` scope and copy the token. +1. Go to `http://localhost:9292`. +1. Paste the token and select **Install GitLab.com Jira Cloud app**. + ### Troubleshooting If the app install failed, you might need to delete `jira_connect_installations` from your database. @@ -68,7 +81,7 @@ If the app install failed, you might need to delete `jira_connect_installations` #### Not authorized to access the file -If you use Gitpod and you get an error about Jira not being able to access the descriptor file, you might need to make the GDK's port public by following these steps: +If you use Gitpod and you get an error about Jira not being able to access the descriptor file, you might need to make the GDK port public by following these steps: 1. Open your GitLab workspace in Gitpod. 1. When the GDK is running, select **Ports** in the bottom-right corner. diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index 787b46133ad..190a6f6eda2 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -102,7 +102,7 @@ it's declared under the `reports:sast` key in the job definition, not because of ### Policies -Certain GitLab workflows, such as [AutoDevOps](../../topics/autodevops/customize.md#disable-jobs), +Certain GitLab workflows, such as [AutoDevOps](../../topics/autodevops/cicd_variables.md#job-disabling-variables), define CI/CD variables to indicate that given scans should be disabled. You can check for this by looking for variables such as: @@ -328,21 +328,6 @@ You can find the schemas for these scanners here: - [SAST](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/sast-report-format.json) - [Secret Detection](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/secret-detection-report-format.json) -### Retention period for vulnerabilities - -GitLab has the following retention policies for vulnerabilities on non-default branches. Vulnerabilities are no longer available: - -- When the related CI job artifact expires. -- 90 days after the pipeline is created, even if the related CI job artifacts are locked. - -To view vulnerabilities, either: - -- Run a new pipeline. -- Download the related CI job artifacts if they are available. - -NOTE: -This does not apply for the vulnerabilities existing on the default branch. - ### Report validation > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/351000) in GitLab 15.0. diff --git a/doc/development/integrations/secure_partner_integration.md b/doc/development/integrations/secure_partner_integration.md index bcbc02d4827..853541144fb 100644 --- a/doc/development/integrations/secure_partner_integration.md +++ b/doc/development/integrations/secure_partner_integration.md @@ -90,7 +90,7 @@ and complete an integration with the Secure stage. - Documentation for [SAST reports](../../user/application_security/sast/index.md#reports-json-format). - Documentation for [Dependency Scanning reports](../../user/application_security/dependency_scanning/index.md#reports-json-format). - Documentation for [Container Scanning reports](../../user/application_security/container_scanning/index.md#reports-json-format). - - See this [example secure job definition that also defines the artifact created](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml). + - See this [example secure job definition that also defines the artifact created](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml). - If you need a new kind of scan or report, [create an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new#) and add the label `devops::secure`. - Once the job is completed, the data can be seen: diff --git a/doc/development/iterating_tables_in_batches.md b/doc/development/iterating_tables_in_batches.md deleted file mode 100644 index 589e38a5cb0..00000000000 --- a/doc/development/iterating_tables_in_batches.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/iterating_tables_in_batches.md' -remove_date: '2022-11-06' ---- - -This document was moved to [another location](database/iterating_tables_in_batches.md). - -<!-- This redirect file can be deleted after <2022-11-06>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/lfs.md b/doc/development/lfs.md index 4d3371af1d4..dd7687cd28b 100644 --- a/doc/development/lfs.md +++ b/doc/development/lfs.md @@ -4,7 +4,10 @@ group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Git LFS **(FREE)** +# Git LFS developer information **(FREE)** + +This page contains developer-centric information for GitLab team members. For the +user documentation, see [Git Large File Storage](../topics/git/lfs/index.md). ## Deep Dive @@ -86,3 +89,9 @@ request is not preserved for the internal API requests made by Gitaly correlation IDs for those API requests are random values until [this Workhorse issue](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/309) is resolved. + +## Related topics + +- Blog post: [Getting started with Git LFS](https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/) +- User documentation: [Git Large File Storage (LFS)](../topics/git/lfs/index.md) +- [GitLab Git Large File Storage (LFS) Administration](../administration/lfs/index.md) for self-managed instances diff --git a/doc/development/licensing.md b/doc/development/licensing.md index d7dfdb7357d..cb703e98264 100644 --- a/doc/development/licensing.md +++ b/doc/development/licensing.md @@ -10,7 +10,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Automated Testing -In order to comply with the terms the libraries we use are licensed under, we have to make sure to check new gems for compatible licenses whenever they're added. To automate this process, we use the [license_finder](https://github.com/pivotal/LicenseFinder) gem by Pivotal. It runs every time a new commit is pushed and verifies that all gems and node modules in the bundle use a license that doesn't conflict with the licensing of either GitLab Community Edition or GitLab Enterprise Edition. +To comply with the terms the libraries we use are licensed under, we have to make sure to check new gems for compatible licenses whenever they're added. To automate this process, we use the [License Finder](https://github.com/pivotal/LicenseFinder) gem by Pivotal. It runs every time a new commit is pushed and verifies that all gems and node modules in the bundle use a license that doesn't conflict with the licensing of either GitLab Community Edition or GitLab Enterprise Edition. There are some limitations with the automated testing, however. CSS, JavaScript, or Ruby libraries which are not included by way of Bundler, npm, or Yarn (for instance those manually copied into our source tree in the `vendor` directory), must be verified manually and independently. Take care whenever one such library is used, as automated tests don't catch problematic licenses from them. diff --git a/doc/development/merge_request_performance_guidelines.md b/doc/development/merge_request_performance_guidelines.md index 9d1489836fb..fd78d02202f 100644 --- a/doc/development/merge_request_performance_guidelines.md +++ b/doc/development/merge_request_performance_guidelines.md @@ -145,7 +145,7 @@ end This means running one query for every object to update. This code can easily overload a database given enough rows to update or many instances of this code running in parallel. This particular problem is known as the -["N+1 query problem"](https://guides.rubyonrails.org/active_record_querying.html#eager-loading-associations). You can write a test with [QueryRecorder](query_recorder.md) to detect this and prevent regressions. +["N+1 query problem"](https://guides.rubyonrails.org/active_record_querying.html#eager-loading-associations). You can write a test with [QueryRecorder](database/query_recorder.md) to detect this and prevent regressions. In this particular case the workaround is fairly easy: @@ -224,7 +224,7 @@ The total number of the queries (including cached ones) executed by the code mod should not increase unless absolutely necessary. The number of executed queries (including cached queries) should not depend on collection size. -You can write a test by passing the `skip_cached` variable to [QueryRecorder](query_recorder.md) to detect this and prevent regressions. +You can write a test by passing the `skip_cached` variable to [QueryRecorder](database/query_recorder.md) to detect this and prevent regressions. As an example, say you have a CI pipeline. All pipeline builds belong to the same pipeline, thus they also belong to the same project (`pipeline.project`): @@ -351,7 +351,7 @@ Post.all.includes(:author).each do |post| end ``` -Also consider using [QueryRecoder tests](query_recorder.md) to prevent a regression when eager loading. +Also consider using [QueryRecoder tests](database/query_recorder.md) to prevent a regression when eager loading. ## Memory Usage @@ -463,7 +463,7 @@ Read more about when and how feature flags should be used in We can consider the following types of storages: -- **Local temporary storage** (very-very short-term storage) This type of storage is system-provided storage, ex. `/tmp` folder. +- **Local temporary storage** (very-very short-term storage) This type of storage is system-provided storage, like a `/tmp` folder. This is the type of storage that you should ideally use for all your temporary tasks. The fact that each node has its own temporary storage makes scaling significantly easier. This storage is also very often SSD-based, thus is significantly faster. @@ -480,7 +480,7 @@ We can consider the following types of storages: Be respectful of that. - **Shared persistent storage** (long-term storage) This type of storage uses - shared network-based storage (ex. NFS). This solution is mostly used by customers running small + shared network-based storage (for example, NFS). This solution is mostly used by customers running small installations consisting of a few nodes. The files on shared storage are easily accessible, but any job that is uploading or downloading data can create a serious contention to all other jobs. This is also an approach by default used by Omnibus. @@ -525,13 +525,13 @@ end The usage of shared temporary storage is required if your intent is to persistent file for a disk-based storage, and not Object Storage. -[Workhorse direct_upload](uploads/index.md#direct-upload) when accepting file +[Workhorse direct upload](uploads/index.md#direct-upload) when accepting file can write it to shared storage, and later GitLab Rails can perform a move operation. The move operation on the same destination is instantaneous. The system instead of performing `copy` operation just re-attaches file into a new place. Since this introduces extra complexity into application, you should only try -to re-use well established patterns (ex.: `ObjectStorage` concern) instead of re-implementing it. +to re-use well established patterns (for example, `ObjectStorage` concern) instead of re-implementing it. The usage of shared temporary storage is otherwise deprecated for all other usages. @@ -549,7 +549,7 @@ that implements a seamless support for Shared and Object Storage-based persisten #### Data access Each feature that accepts data uploads or allows to download them needs to use -[Workhorse direct_upload](uploads/index.md#direct-upload). It means that uploads needs to be +[Workhorse direct upload](uploads/index.md#direct-upload). It means that uploads needs to be saved directly to Object Storage by Workhorse, and all downloads needs to be served by Workhorse. @@ -561,5 +561,5 @@ can time out, which is especially problematic for slow clients. If clients take to upload/download the processing slot might be killed due to request processing timeout (usually between 30s-60s). -For the above reasons it is required that [Workhorse direct_upload](uploads/index.md#direct-upload) is implemented +For the above reasons it is required that [Workhorse direct upload](uploads/index.md#direct-upload) is implemented for all file uploads and downloads. diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index 5764c876e4d..8f035d4aa13 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -184,13 +184,21 @@ git checkout origin/master db/structure.sql VERSION=<migration ID> bundle exec rails db:migrate:main ``` -### Adding new tables to GitLab Schema - -GitLab connects to two different Postgres databases: `main` and `ci`. New tables should be defined in [`lib/gitlab/database/gitlab_schemas.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/database/gitlab_schemas.yml) with the databases they need to be added to. - - ```yaml - <TABLE_NAME>: :gitlab_main - ``` +### Adding new tables to the database dictionary + +GitLab connects to two different Postgres databases: `main` and `ci`. New tables should be defined in [`db/docs/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/db/docs): + +```yaml +table_name: table name exmaple +description: Description example +introduced_by_url: Merge request link +milestone: Milestone example +feature_categories: +- Feature category example +classes: +- Class example +gitlab_schema: gitlab_main +``` ## Avoiding downtime @@ -305,7 +313,7 @@ of migration helpers. In this example, we use version 2.0 of the migration class: ```ruby -class TestMigration < Gitlab::Database::Migration[2.0] +class TestMigration < Gitlab::Database::Migration[2.1] def change end end @@ -338,7 +346,7 @@ is concurrently accessed and modified by other processes, acquiring the lock may a while. The lock request is waiting in a queue and it may also block other queries on the `users` table once it has been enqueued. -More information about PostgresSQL locks: [Explicit Locking](https://www.postgresql.org/docs/current/explicit-locking.html) +More information about PostgreSQL locks: [Explicit Locking](https://www.postgresql.org/docs/current/explicit-locking.html) For stability reasons, GitLab.com has a specific [`statement_timeout`](../user/gitlab_com/index.md#postgresql) set. When the migration is invoked, any database query has @@ -376,7 +384,7 @@ Occasionally a migration may need to acquire multiple locks on different objects To prevent catalog bloat, ask for all those locks explicitly before performing any DDL. A better strategy is to split the migration, so that we only need to acquire one lock at the time. -**Removing a column:** +#### Removing a column ```ruby enable_lock_retries! @@ -386,7 +394,7 @@ def change end ``` -**Multiple changes on the same table:** +#### Multiple changes on the same table With the lock-retry methodology enabled, all operations wrap into a single transaction. When you have the lock, you should do as much as possible inside the transaction rather than trying to get another lock later. @@ -406,7 +414,7 @@ def down end ``` -**Removing a foreign key:** +#### Removing a foreign key ```ruby enable_lock_retries! @@ -420,7 +428,7 @@ def down end ``` -**Changing default value for a column:** +#### Changing default value for a column ```ruby enable_lock_retries! @@ -434,7 +442,7 @@ def down end ``` -**Creating a new table with a foreign key:** +#### Creating a new table with a foreign key We can wrap the `create_table` method with `with_lock_retries`: @@ -453,7 +461,7 @@ def down end ``` -**Creating a new table when we have two foreign keys:** +#### Creating a new table when we have two foreign keys Only one foreign key should be created per transaction. This is because [the addition of a foreign key constraint requires a `SHARE ROW EXCLUSIVE` lock on the referenced table](https://www.postgresql.org/docs/12/sql-createtable.html#:~:text=The%20addition%20of%20a%20foreign%20key%20constraint%20requires%20a%20SHARE%20ROW%20EXCLUSIVE%20lock%20on%20the%20referenced%20table), and locking multiple tables in the same transaction should be avoided. @@ -600,7 +608,7 @@ by calling the method `disable_ddl_transaction!` in the body of your migration class like so: ```ruby -class MyMigration < Gitlab::Database::Migration[2.0] +class MyMigration < Gitlab::Database::Migration[2.1] disable_ddl_transaction! INDEX_NAME = 'index_name' @@ -611,10 +619,10 @@ class MyMigration < Gitlab::Database::Migration[2.0] end ``` -Verify the index is not being used anymore with this Thanos query: +You can verify that the index is not being used with [Thanos](https://thanos-query.ops.gitlab.net/graph?g0.expr=sum%20by%20(type)(rate(pg_stat_user_indexes_idx_scan%7Benv%3D%22gprd%22%2C%20indexrelname%3D%22INSERT%20INDEX%20NAME%20HERE%22%7D%5B30d%5D))&g0.tab=1&g0.stacked=0&g0.range_input=1h&g0.max_source_resolution=0s&g0.deduplicate=1&g0.partial_response=0&g0.store_matches=%5B%5D): ```sql -sum by (type)(rate(pg_stat_user_indexes_idx_scan{env="gprd", indexrelname="index_groups_on_parent_id_id"}[5m])) +sum by (type)(rate(pg_stat_user_indexes_idx_scan{env="gprd", indexrelname="INSERT INDEX NAME HERE"}[30d])) ``` Note that it is not necessary to check if the index exists prior to @@ -657,7 +665,7 @@ by calling the method `disable_ddl_transaction!` in the body of your migration class like so: ```ruby -class MyMigration < Gitlab::Database::Migration[2.0] +class MyMigration < Gitlab::Database::Migration[2.1] disable_ddl_transaction! INDEX_NAME = 'index_name' @@ -700,7 +708,7 @@ The easiest way to test for existence of an index by name is to use the be used with a name option. For example: ```ruby -class MyMigration < Gitlab::Database::Migration[2.0] +class MyMigration < Gitlab::Database::Migration[2.1] INDEX_NAME = 'index_name' def up @@ -735,7 +743,7 @@ Here's an example where we add a new column with a foreign key constraint. Note it includes `index: true` to create an index for it. ```ruby -class Migration < Gitlab::Database::Migration[2.0] +class Migration < Gitlab::Database::Migration[2.1] def change add_reference :model, :other_model, index: true, foreign_key: { on_delete: :cascade } @@ -766,12 +774,7 @@ With PostgreSQL 11 being the minimum version in GitLab 13.0 and later, adding co the standard `add_column` helper should be used in all cases. Before PostgreSQL 11, adding a column with a default was problematic as it would -have caused a full table rewrite. The corresponding helper `add_column_with_default` -has been deprecated and is scheduled to be removed in a later release. - -If a backport adding a column with a default value is needed for %12.9 or earlier versions, -it should use `add_column_with_default` helper. If a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3) -is involved, backporting to %12.9 is contraindicated. +have caused a full table rewrite. ## Removing the column default for non-nullable columns @@ -796,7 +799,7 @@ expensive and disruptive operation for larger tables, but in reality it's not. Take the following migration as an example: ```ruby -class DefaultRequestAccessGroups < Gitlab::Database::Migration[2.0] +class DefaultRequestAccessGroups < Gitlab::Database::Migration[2.1] def change change_column_default(:namespaces, :request_access_enabled, from: false, to: true) end @@ -818,7 +821,7 @@ in the `namespaces` table. Only when creating a new column with a default, all t NOTE: A faster [ALTER TABLE ADD COLUMN with a non-null default](https://www.depesz.com/2018/04/04/waiting-for-postgresql-11-fast-alter-table-add-column-with-a-non-null-default/) -was introduced on PostgresSQL 11.0, removing the need of rewriting the table when a new column with a default value is added. +was introduced on PostgreSQL 11.0, removing the need of rewriting the table when a new column with a default value is added. For the reasons mentioned above, it's safe to use `change_column_default` in a single-transaction migration without requiring `disable_ddl_transaction!`. @@ -852,8 +855,7 @@ update_column_in_batches(:projects, :foo, update_value) do |table, query| end ``` -Like `add_column_with_default`, there is a RuboCop cop to detect usage of this -on large tables. In the case of `update_column_in_batches`, it may be acceptable +In the case of `update_column_in_batches`, it may be acceptable to run on a large table, as long as it is only updating a small subset of the rows in the table, but do not ignore that without validating on the GitLab.com staging environment - or asking someone else to do so for you - beforehand. @@ -991,7 +993,7 @@ Re-add a sequence: A Rails migration example: ```ruby -class DropSequenceTest < Gitlab::Database::Migration[2.0] +class DropSequenceTest < Gitlab::Database::Migration[2.1] def up drop_sequence(:ci_pipelines_config, :pipeline_id, :ci_pipelines_config_pipeline_id_seq) end @@ -1022,7 +1024,7 @@ Under the hood, it works like this: - Add the primary key using the index defined beforehand. ```ruby -class SwapPrimaryKey < Gitlab::Database::Migration[2.0] +class SwapPrimaryKey < Gitlab::Database::Migration[2.1] TABLE_NAME = :table_name PRIMARY_KEY = :table_name_pkey OLD_INDEX_NAME = :old_index_name @@ -1113,18 +1115,18 @@ The Rails 5 natively supports `JSONB` (binary JSON) column type. Example migration adding this column: ```ruby -class AddOptionsToBuildMetadata < Gitlab::Database::Migration[2.0] +class AddOptionsToBuildMetadata < Gitlab::Database::Migration[2.1] def change add_column :ci_builds_metadata, :config_options, :jsonb end end ``` -You have to use a serializer to provide a translation layer: +By default hash keys will be strings. Optionally you can add a custom data type to provide different access to keys. ```ruby class BuildMetadata - serialize :config_options, Serializers::Json # rubocop:disable Cop/ActiveRecordSerialize + attribute :config_options, :ind_jsonb # for indifferent accesss or :sym_jsonb if you need symbols only as keys. end ``` @@ -1145,7 +1147,7 @@ Do not store `attr_encrypted` attributes as `:text` in the database; use efficient: ```ruby -class AddSecretToSomething < Gitlab::Database::Migration[2.0] +class AddSecretToSomething < Gitlab::Database::Migration[2.1] def change add_column :something, :encrypted_secret, :binary add_column :something, :encrypted_secret_iv, :binary @@ -1203,7 +1205,7 @@ If you need more complex logic, you can define and use models local to a migration. For example: ```ruby -class MyMigration < Gitlab::Database::Migration[2.0] +class MyMigration < Gitlab::Database::Migration[2.1] class Project < MigrationRecord self.table_name = 'projects' end @@ -1227,7 +1229,7 @@ Be aware of the limitations [when using models in migrations](#using-models-in-m In most circumstances, prefer migrating data in **batches** when modifying data in the database. -We introduced a new helper [each_batch_range](https://gitlab.com/gitlab-org/gitlab/-/blob/cd3e0a5cddcb464cb9b8c6e3275839cf57dfa6e2/lib/gitlab/database/dynamic_model_helpers.rb#L28-32) which facilitates the process of iterating over a collection in a performant way. The default size of the batch is defined in the `BATCH_SIZE` constant. +We introduced a new helper [`each_batch_range`](https://gitlab.com/gitlab-org/gitlab/-/blob/cd3e0a5cddcb464cb9b8c6e3275839cf57dfa6e2/lib/gitlab/database/dynamic_model_helpers.rb#L28-32) which facilitates the process of iterating over a collection in a performant way. The default size of the batch is defined in the `BATCH_SIZE` constant. See the following example to get an idea. @@ -1302,7 +1304,7 @@ in a previous migration. It is important not to leave out the `User.reset_column_information` command, to ensure that the old schema is dropped from the cache and ActiveRecord loads the updated schema information. ```ruby -class AddAndSeedMyColumn < Gitlab::Database::Migration[2.0] +class AddAndSeedMyColumn < Gitlab::Database::Migration[2.1] class User < MigrationRecord self.table_name = 'users' end diff --git a/doc/development/multi_version_compatibility.md b/doc/development/multi_version_compatibility.md index 8e2103b2f3e..36b392a0037 100644 --- a/doc/development/multi_version_compatibility.md +++ b/doc/development/multi_version_compatibility.md @@ -112,7 +112,7 @@ During an update, there will be [two different versions of GitLab running in dif Yes! We have specific instructions for [zero-downtime updates](../update/index.md#upgrading-without-downtime) because it allows us to ignore some permutations of compatibility. This is why we don't worry about Rails code making DB calls to an old PostgreSQL database schema. -## I've identified a potential backwards compatibility problem, what can I do about it? +## You've identified a potential backwards compatibility problem, what can you do about it? ### Coordinate diff --git a/doc/development/namespaces_storage_statistics.md b/doc/development/namespaces_storage_statistics.md deleted file mode 100644 index 75e79d1f693..00000000000 --- a/doc/development/namespaces_storage_statistics.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/namespaces_storage_statistics.md' -remove_date: '2022-11-05' ---- - -This document was moved to [another location](database/namespaces_storage_statistics.md). - -<!-- This redirect file can be deleted after <2022-11-05>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/development/accessibility.md b/doc/development/new_fe_guide/development/accessibility.md deleted file mode 100644 index 9575acd20c7..00000000000 --- a/doc/development/new_fe_guide/development/accessibility.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../../fe_guide/accessibility.md' -remove_date: '2022-11-15' ---- - -This document was moved to [another location](../../fe_guide/accessibility.md). - -<!-- This redirect file can be deleted after <2022-11-15>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/development/components.md b/doc/development/new_fe_guide/development/components.md deleted file mode 100644 index 9ad742272d1..00000000000 --- a/doc/development/new_fe_guide/development/components.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../../fe_guide/index.md' -remove_date: '2022-11-15' ---- - -This document was moved to [another location](../../fe_guide/index.md). - -<!-- This redirect file can be deleted after <2022-11-15>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/development/index.md b/doc/development/new_fe_guide/development/index.md deleted file mode 100644 index 9ad742272d1..00000000000 --- a/doc/development/new_fe_guide/development/index.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../../fe_guide/index.md' -remove_date: '2022-11-15' ---- - -This document was moved to [another location](../../fe_guide/index.md). - -<!-- This redirect file can be deleted after <2022-11-15>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/development/performance.md b/doc/development/new_fe_guide/development/performance.md deleted file mode 100644 index c72f3ded896..00000000000 --- a/doc/development/new_fe_guide/development/performance.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../../fe_guide/performance.md' -remove_date: '2022-11-15' ---- - -This document was moved to [another location](../../fe_guide/performance.md). - -<!-- This redirect file can be deleted after <2022-11-15>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/index.md b/doc/development/new_fe_guide/index.md deleted file mode 100644 index 83c1db696b4..00000000000 --- a/doc/development/new_fe_guide/index.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../fe_guide/index.md' -remove_date: '2022-11-15' ---- - -This document was moved to [another location](../fe_guide/index.md). - -<!-- This redirect file can be deleted after <2022-11-15>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/modules/dirty_submit.md b/doc/development/new_fe_guide/modules/dirty_submit.md deleted file mode 100644 index 9ad742272d1..00000000000 --- a/doc/development/new_fe_guide/modules/dirty_submit.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../../fe_guide/index.md' -remove_date: '2022-11-15' ---- - -This document was moved to [another location](../../fe_guide/index.md). - -<!-- This redirect file can be deleted after <2022-11-15>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/modules/index.md b/doc/development/new_fe_guide/modules/index.md deleted file mode 100644 index 9ad742272d1..00000000000 --- a/doc/development/new_fe_guide/modules/index.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../../fe_guide/index.md' -remove_date: '2022-11-15' ---- - -This document was moved to [another location](../../fe_guide/index.md). - -<!-- This redirect file can be deleted after <2022-11-15>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/modules/widget_extensions.md b/doc/development/new_fe_guide/modules/widget_extensions.md deleted file mode 100644 index 3741ee8c38a..00000000000 --- a/doc/development/new_fe_guide/modules/widget_extensions.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../../fe_guide/merge_request_widget_extensions.md' -remove_date: '2022-11-15' ---- - -This document was moved to [another location](../../fe_guide/merge_request_widget_extensions.md). - -<!-- This redirect file can be deleted after <2022-11-15>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/tips.md b/doc/development/new_fe_guide/tips.md deleted file mode 100644 index 83c1db696b4..00000000000 --- a/doc/development/new_fe_guide/tips.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../fe_guide/index.md' -remove_date: '2022-11-15' ---- - -This document was moved to [another location](../fe_guide/index.md). - -<!-- This redirect file can be deleted after <2022-11-15>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/ordering_table_columns.md b/doc/development/ordering_table_columns.md deleted file mode 100644 index b665cb0d4c7..00000000000 --- a/doc/development/ordering_table_columns.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/ordering_table_columns.md' -remove_date: '2022-11-04' ---- - -This document was moved to [another location](database/ordering_table_columns.md). - -<!-- This redirect file can be deleted after <2022-11-04>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/packages/dependency_proxy.md b/doc/development/packages/dependency_proxy.md index d5cc219cba0..cc8b202e556 100644 --- a/doc/development/packages/dependency_proxy.md +++ b/doc/development/packages/dependency_proxy.md @@ -117,7 +117,7 @@ Manifests are more complicated, partially due to [rate limiting on DockerHub](ht A manifest is essentially a recipe for creating an image. It has a list of blobs to create a certain image. So `alpine:latest` has a manifest associated with it that specifies the blobs needed to create the `alpine:latest` image. The interesting part is that `alpine:latest` can change over time, so we can't just cache the manifest and -assume it is OK to use forever. Instead, we must check the digest of the manifest, which is an Etag. This gets +assume it is OK to use forever. Instead, we must check the digest of the manifest, which is an ETag. This gets interesting because the requests for manifests often don't include the digest. So how do we know if the manifest we have cached is still the most up-to-date `alpine:latest`? DockerHub allows free HEAD requests that don't count toward their rate limit. The HEAD request returns the manifest digest so we can tell whether or not the one we @@ -143,7 +143,7 @@ Management of file uploads and caching happens in [Workhorse](../workhorse/index [`POST` routes](https://gitlab.com/gitlab-org/gitlab/-/blob/3f76455ac9cf90a927767e55c837d6b07af818df/config/routes/group.rb#L170-173) that we have for the Dependency Proxy. -The [send_dependency](https://gitlab.com/gitlab-org/gitlab/-/blob/7359d23f4e078479969c872924150219c6f1665f/app/helpers/workhorse_helper.rb#L46-53) +The [`send_dependency`](https://gitlab.com/gitlab-org/gitlab/-/blob/7359d23f4e078479969c872924150219c6f1665f/app/helpers/workhorse_helper.rb#L46-53) method makes a request to Workhorse including the previously fetched JWT from the external registry. Workhorse then can use that token to request the manifest or blob the user originally requested. The Workhorse code lives in [`workhorse/internal/dependencyproxy/dependencyproxy.go`](https://gitlab.com/gitlab-org/gitlab/-/blob/b8f44a8f3c26efe9932c2ada2df75ef7acb8417b/workhorse/internal/dependencyproxy/dependencyproxy.go#L4). diff --git a/doc/development/pages/index.md b/doc/development/pages/index.md index 6ee8a9ac433..5e56264330a 100644 --- a/doc/development/pages/index.md +++ b/doc/development/pages/index.md @@ -178,7 +178,7 @@ The `redirect-uri` must not contain any GitLab Pages site domain. auth-redirect-uri=http://pages.gdk.test:3010/auth # the authentication callback url for GitLab Pages ``` -1. If running Pages inside the GDK, you can use GDK's `protected_config_files` section under `gdk` in +1. If running Pages inside the GDK, you can use GDK `protected_config_files` section under `gdk` in your `gdk.yml` to avoid getting `gitlab-pages.conf` configuration rewritten: ```yaml diff --git a/doc/development/performance.md b/doc/development/performance.md index 3881fad0528..127cade9fee 100644 --- a/doc/development/performance.md +++ b/doc/development/performance.md @@ -37,7 +37,7 @@ consistent performance of GitLab. Refer to the [Index](#performance-documentatio - [Service measurement](../development/service_measurement.md) - Self-managed administration and customer-focused: - [File system performance benchmarking](../administration/operations/filesystem_benchmarking.md) - - [Sidekiq performance troubleshooting](../administration/troubleshooting/sidekiq.md) + - [Sidekiq performance troubleshooting](../administration/sidekiq/sidekiq_troubleshooting.md) ## Workflow @@ -74,7 +74,7 @@ GitLab provides built-in tools to help improve performance and availability: - [Profiling](profiling.md). - [Distributed Tracing](distributed_tracing.md) - [GitLab Performance Monitoring](../administration/monitoring/performance/index.md). -- [QueryRecoder](query_recorder.md) for preventing `N+1` regressions. +- [QueryRecoder](database/query_recorder.md) for preventing `N+1` regressions. - [Chaos endpoints](chaos_endpoints.md) for testing failure scenarios. Intended mainly for testing availability. - [Service measurement](service_measurement.md) for measuring and logging service execution. @@ -312,7 +312,7 @@ To export the flame graph to an SVG file, use [Brendan Gregg's FlameGraph tool]( stackprof --stackcollapse /tmp/group_member_policy_spec.rb.dump | flamegraph.pl > flamegraph.svg ``` -It's also possible to view flame graphs through [speedscope](https://github.com/jlfwong/speedscope). +It's also possible to view flame graphs through [Speedscope](https://github.com/jlfwong/speedscope). You can do this when using the [performance bar](profiling.md#speedscope-flamegraphs) and when [profiling code blocks](https://github.com/jlfwong/speedscope/wiki/Importing-from-stackprof-(ruby)). This option isn't supported by `bin/rspec-stackprof`. diff --git a/doc/development/pipelines/index.md b/doc/development/pipelines/index.md index 01bb813e794..a7b8c99bd13 100644 --- a/doc/development/pipelines/index.md +++ b/doc/development/pipelines/index.md @@ -68,8 +68,8 @@ Later on in [the `rspec fail-fast` job](#fail-fast-job-in-merge-request-pipeline In addition, there are a few circumstances where we would always run the full RSpec tests: -- when the `pipeline:run-all-rspec` label is set on the merge request -- when the `pipeline:run-full-rspec` label is set on the merge request, this label is assigned by triage automation when the merge request is approved by any reviewer +- when the `pipeline:run-all-rspec` label is set on the merge request. This label will trigger all RSpec tests including those run in the `as-if-foss` jobs. +- when the `pipeline:mr-approved` label is set on the merge request, and if the code changes satisfy the `backend-patterns` rule. Note that this label is assigned by triage automation when the merge request is approved by any reviewer. It is not recommended to apply this label manually. - when the merge request is created by an automation (for example, Gitaly update or MR targeting a stable branch) - when the merge request is created in a security mirror - when any CI configuration file is changed (for example, `.gitlab-ci.yml` or `.gitlab/ci/**/*`) @@ -88,10 +88,12 @@ In addition, there are a few circumstances where we would always run the full Je - when the `pipeline:run-all-jest` label is set on the merge request - when the merge request is created by an automation (for example, Gitaly update or MR targeting a stable branch) - when the merge request is created in a security mirror -- when any CI configuration file is changed (for example, `.gitlab-ci.yml` or `.gitlab/ci/**/*`) -- when any frontend "core" file is changed (for example, `package.json`, `yarn.lock`, `babel.config.js`, `jest.config.*.js`, `config/helpers/**/*.js`) +- when relevant CI configuration file is changed (`.gitlab/ci/rules.gitlab-ci.yml`, `.gitlab/ci/frontend.gitlab-ci.yml`) +- when any frontend dependency file is changed (for example, `package.json`, `yarn.lock`, `config/webpack.config.js`, `config/helpers/**/*.js`) - when any vendored JavaScript file is changed (for example, `vendor/assets/javascripts/**/*`) -- when any backend file is changed ([see the patterns list for details](https://gitlab.com/gitlab-org/gitlab/-/blob/3616946936c1adbd9e754c1bd06f86ba670796d8/.gitlab/ci/rules.gitlab-ci.yml#L205-216)) + +The `rules` definitions for full Jest tests are defined at `.frontend:rules:jest` in +[`rules.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/42321b18b946c64d2f6f788c38844499a5ae9141/.gitlab/ci/rules.gitlab-ci.yml#L938-955). ### Fork pipelines @@ -146,9 +148,37 @@ merge request. This prevents `rspec fail-fast` duration from exceeding the avera This number can be overridden by setting a CI/CD variable named `RSPEC_FAIL_FAST_TEST_FILE_COUNT_THRESHOLD`. +## Re-run previously failed tests in merge request pipelines + +In order to reduce the feedback time after resolving failed tests for a merge request, the `rspec rspec-pg12-rerun-previous-failed-tests` +and `rspec rspec-ee-pg12-rerun-previous-failed-tests` jobs run the failed tests from the previous MR pipeline. + +This was introduced on August 25th 2021, with <https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69053>. + +### How it works? + +1. The `detect-previous-failed-tests` job (`prepare` stage) detects the test files associated with failed RSpec + jobs from the previous MR pipeline. +1. The `rspec rspec-pg12-rerun-previous-failed-tests` and `rspec rspec-ee-pg12-rerun-previous-failed-tests` jobs + will run the test files gathered by the `detect-previous-failed-tests` job. + +```mermaid +graph LR + subgraph "prepare stage"; + A["detect-previous-failed-tests"] + end + + subgraph "test stage"; + B["rspec rspec-pg12-rerun-previous-failed-tests"]; + C["rspec rspec-ee-pg12-rerun-previous-failed-tests"]; + end + + A --"artifact: list of test files"--> B & C +``` + ## Faster feedback for merge requests that fix a broken `master` -When you need to [fix a broken `master`](https://about.gitlab.com/handbook/engineering/workflow/#resolution-of-broken-master), you can add the `pipeline:expedite-master-fixing` label to expedite the pipelines that run on the merge request. +When you need to [fix a broken `master`](https://about.gitlab.com/handbook/engineering/workflow/#resolution-of-broken-master), you can add the `pipeline:expedite` label to expedite the pipelines that run on the merge request. When this label is assigned, the following steps of the CI/CD pipeline are skipped: diff --git a/doc/development/pipelines/internals.md b/doc/development/pipelines/internals.md index 2861e2f266b..71492ed9f5d 100644 --- a/doc/development/pipelines/internals.md +++ b/doc/development/pipelines/internals.md @@ -107,7 +107,7 @@ automatically updates the Gitaly version used in the main project), [the Dependency proxy isn't accessible](https://gitlab.com/gitlab-org/gitlab/-/issues/332411#note_1130388163) and the job fails at the `Preparing the "docker+machine" executor` step. To work around that, we have a special workflow rule, that overrides the -`${GITLAB_DEPENDENCY_PROXY_ADDRESS}` variable so that Depdendency proxy isn't used in that case: +`${GITLAB_DEPENDENCY_PROXY_ADDRESS}` variable so that Dependency proxy isn't used in that case: ```yaml - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $GITLAB_USER_LOGIN =~ /project_\d+_bot\d*/' @@ -165,9 +165,9 @@ and included in `rules` definitions via [YAML anchors](../../ci/yaml/yaml_optimi | `if:` conditions | Description | Notes | |------------------|-------------|-------| -| `if-not-canonical-namespace` | Matches if the project isn't in the canonical (`gitlab-org/`) or security (`gitlab-org/security`) namespace. | Use to create a job for forks (by using `when: on_success|manual`), or **not** create a job for forks (by using `when: never`). | -| `if-not-ee` | Matches if the project isn't EE (that is, project name isn't `gitlab` or `gitlab-ee`). | Use to create a job only in the FOSS project (by using `when: on_success|manual`), or **not** create a job if the project is EE (by using `when: never`). | -| `if-not-foss` | Matches if the project isn't FOSS (that is, project name isn't `gitlab-foss`, `gitlab-ce`, or `gitlabhq`). | Use to create a job only in the EE project (by using `when: on_success|manual`), or **not** create a job if the project is FOSS (by using `when: never`). | +| `if-not-canonical-namespace` | Matches if the project isn't in the canonical (`gitlab-org/`) or security (`gitlab-org/security`) namespace. | Use to create a job for forks (by using `when: on_success` or `when: manual`), or **not** create a job for forks (by using `when: never`). | +| `if-not-ee` | Matches if the project isn't EE (that is, project name isn't `gitlab` or `gitlab-ee`). | Use to create a job only in the FOSS project (by using `when: on_success` or `when: manual`), or **not** create a job if the project is EE (by using `when: never`). | +| `if-not-foss` | Matches if the project isn't FOSS (that is, project name isn't `gitlab-foss`, `gitlab-ce`, or `gitlabhq`). | Use to create a job only in the EE project (by using `when: on_success` or `when: manual`), or **not** create a job if the project is FOSS (by using `when: never`). | | `if-default-refs` | Matches if the pipeline is for `master`, `main`, `/^[\d-]+-stable(-ee)?$/` (stable branches), `/^\d+-\d+-auto-deploy-\d+$/` (auto-deploy branches), `/^security\//` (security branches), merge requests, and tags. | Note that jobs aren't created for branches with this default configuration. | | `if-master-refs` | Matches if the current branch is `master` or `main`. | | | `if-master-push` | Matches if the current branch is `master` or `main` and pipeline source is `push`. | | @@ -203,7 +203,7 @@ and included in `rules` definitions via [YAML anchors](../../ci/yaml/yaml_optimi | `ci-qa-patterns` | Only create job for CI configuration-related changes related to the `qa` stage. | | `yaml-lint-patterns` | Only create job for YAML-related changes. | | `docs-patterns` | Only create job for docs-related changes. | -| `frontend-dependency-patterns` | Only create job when frontend dependencies are updated (that is, `package.json`, and `yarn.lock`). changes. | +| `frontend-dependency-patterns` | Only create job when frontend dependencies are updated (for example, `package.json`, and `yarn.lock`) changes. | | `frontend-patterns-for-as-if-foss` | Only create job for frontend-related changes that have impact on FOSS. | | `backend-patterns` | Only create job for backend-related changes. | | `db-patterns` | Only create job for DB-related changes. | diff --git a/doc/development/pipelines/performance.md b/doc/development/pipelines/performance.md index 1c6f9d78879..5f2df91edf3 100644 --- a/doc/development/pipelines/performance.md +++ b/doc/development/pipelines/performance.md @@ -54,7 +54,6 @@ This works well for the following reasons: - `update-qa-cache`, defined in [`.gitlab/ci/qa.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/qa.gitlab-ci.yml). - `update-assets-compile-production-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/frontend.gitlab-ci.yml). - `update-assets-compile-test-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/frontend.gitlab-ci.yml). - - `update-yarn-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/frontend.gitlab-ci.yml). - `update-storybook-yarn-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/frontend.gitlab-ci.yml). 1. These jobs can also be forced to run in merge requests with the `pipeline:update-cache` label (this can be useful to warm the caches in a MR that updates the cache keys). diff --git a/doc/development/polymorphic_associations.md b/doc/development/polymorphic_associations.md deleted file mode 100644 index 6b9158b8408..00000000000 --- a/doc/development/polymorphic_associations.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/polymorphic_associations.md' -remove_date: '2022-11-04' ---- - -This document was moved to [another location](database/polymorphic_associations.md). - -<!-- This redirect file can be deleted after <2022-11-04>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/product_qualified_lead_guide/index.md b/doc/development/product_qualified_lead_guide/index.md index 89a05d59fc9..1ad4f622829 100644 --- a/doc/development/product_qualified_lead_guide/index.md +++ b/doc/development/product_qualified_lead_guide/index.md @@ -17,7 +17,7 @@ A hand-raise PQL is a user who requests to speak to sales from within the produc 1. Set up CustomersDot using the [normal install instructions](https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/doc/setup/installation_steps.md). 1. Set the `CUSTOMER_PORTAL_URL` environment variable to your local (or ngrok) URL of your CustomersDot instance. -1. Place `export CUSTOMER_PORTAL_URL='https://XXX.ngrok.io/'` in your shell rc script (`~/.zshrc` or `~/.bash_profile` or `~/.bashrc`) and restart GDK. +1. Place `export CUSTOMER_PORTAL_URL='https://XXX.ngrok.io/'` in your shell `rc` script (`~/.zshrc` or `~/.bash_profile` or `~/.bashrc`) and restart GDK. 1. Enter the credentials on CustomersDot development to Platypus in your `/config/secrets.yml` and restart. Credentials for the Platypus Staging are in the 1Password Growth vault. The URL for staging is `https://staging.ci.nexus.gitlabenvironment.cloud`. ```yaml @@ -95,16 +95,16 @@ The `ctaTracking` parameters follow [the `data-track` attributes](../snowplow/im When embedding a new hand raise form, use a unique `glmContent` or `glm_content` field that is different to any existing values. -We currently use the following `glm content` values: +We currently use the following `glm_content` values: -| glm_content value | Notes | -| ------ | ------ | -| discover-group-security | This value is used in the group security feature discovery page. | -| discover-group-security-pqltest | This value is used in the group security feature discovery page [experiment with 3 CTAs](https://gitlab.com/gitlab-org/gitlab/-/issues/349799). | -| discover-project-security | This value is used in the project security feature discovery page. | -| discover-project-security-pqltest | This value is used in the project security feature discovery page [experiment with 3 CTAs](https://gitlab.com/gitlab-org/gitlab/-/issues/349799). | -| group-billing | This value is used in the group billing page. | -| trial-status-show-group | This value is used in the top left nav when a namespace has an active trial. | +| `glm_content` value | Notes | +|-------------------------------------|-------| +| `discover-group-security` | This value is used in the group security feature discovery page. | +| `discover-group-security-pqltest` | This value is used in the group security feature discovery page [experiment with 3 CTAs](https://gitlab.com/gitlab-org/gitlab/-/issues/349799). | +| `discover-project-security` | This value is used in the project security feature discovery page. | +| `discover-project-security-pqltest` | This value is used in the project security feature discovery page [experiment with 3 CTAs](https://gitlab.com/gitlab-org/gitlab/-/issues/349799). | +| `group-billing` | This value is used in the group billing page. | +| `trial-status-show-group` | This value is used in the top left nav when a namespace has an active trial. | ### Test the component @@ -144,7 +144,7 @@ sequenceDiagram ``` -#### Trial lead flow on CustomersDot (sync_to_gl) +#### Trial lead flow on CustomersDot (`sync_to_gl`) ```mermaid sequenceDiagram diff --git a/doc/development/profiling.md b/doc/development/profiling.md index 3eb2c7c9144..efee6ff3cd5 100644 --- a/doc/development/profiling.md +++ b/doc/development/profiling.md @@ -21,7 +21,7 @@ The first argument to the profiler is either a full URL leading slash. By default the report dump will be stored in a temporary file, which can be -interacted with using the [stackprof API](#reading-a-gitlabprofiler-report). +interacted with using the [Stackprof API](#reading-a-gitlabprofiler-report). When using the script, command-line documentation is available by passing no arguments. @@ -62,7 +62,7 @@ Pass in a `profiler_options` hash to configure the output file (`out`) of the sa Gitlab::Profiler.profile('/gitlab-org/gitlab-test', user: User.first, profiler_options: { out: 'tmp/profile.dump' }) ``` -## Reading a GitLab::Profiler report +## Reading a `GitLab::Profiler` report You can get a summary of where time was spent by running Stackprof against the sampling data. For example: @@ -131,25 +131,39 @@ Find more information about different sampling modes in the [Stackprof docs](htt This is enabled for all users that can access the performance bar. +<!-- vale gitlab.SubstitutionWarning = NO --> +<!-- Here, "bullet" is a false positive --> + ## Bullet -Bullet is a Gem that can be used to track down N+1 query problems. Bullet section is -displayed on the [performance-bar](../administration/monitoring/performance/performance_bar.md). +Bullet is a Gem that can be used to track down N+1 query problems. It logs +query problems to the Rails log and the browser console. The **Bullet** section is +displayed on the [performance bar](../administration/monitoring/performance/performance_bar.md).  -Because Bullet adds quite a bit of logging noise the logging is disabled by default. -To enable the logging, set the environment variable `ENABLE_BULLET` to a non-empty value before -starting GitLab. For example: +Bullet is enabled only in development mode by default. However, logging is disabled, +because Bullet logging is noisy. To configure Bullet and its logging: -```shell -ENABLE_BULLET=true bundle exec rails s -``` +- To manually enable or disable Bullet on an environment, add these lines to + `config/gitlab.yml`, changing the `enabled` value as needed: + + ```yaml + bullet: + enabled: false + ``` + +- To enable Bullet logging, set the `ENABLE_BULLET` environment variable to a + non-empty value before starting GitLab: + + ```shell + ENABLE_BULLET=true bundle exec rails s + ``` -Bullet logs query problems to both the Rails log as well as the browser -console. +As a follow-up to finding `N+1` queries with Bullet, consider writing a +[QueryRecoder test](database/query_recorder.md) to prevent a regression. -As a follow up to finding `N+1` queries with Bullet, consider writing a [QueryRecoder test](query_recorder.md) to prevent a regression. +<!-- vale gitlab.SubstitutionWarning = YES --> ## System stats diff --git a/doc/development/project_templates.md b/doc/development/project_templates.md index 2f1ded23e38..269724c0a7f 100644 --- a/doc/development/project_templates.md +++ b/doc/development/project_templates.md @@ -4,7 +4,9 @@ group: Workspace info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" --- -# Contribute a built-in project template +# Contribute to built-in project templates + +## Adding a new built-in project template This page provides instructions about how to contribute a [built-in project template](../user/project/working_with_projects.md#create-a-project-from-a-built-in-template). @@ -20,7 +22,7 @@ You can contribute the following types of project templates: - Enterprise: For users with GitLab Premium and above. - Non-enterprise: For users with GitLab Free and above. -## Prerequisites +### Prerequisites To add or update an existing template, you must have the following tools installed: @@ -28,22 +30,22 @@ installed: - `wget` - `tar` -## Create a project template for review +### Create a project template for review 1. In your selected namespace, create a public project. 1. Add the project content you want to use in the template. Do not include unnecessary assets or dependencies. For an example, [see this project](https://gitlab.com/gitlab-org/project-templates/dotnetcore). 1. When the project is ready for review, [create an issue](https://gitlab.com/gitlab-org/gitlab/issues) with a link to your project. - In your issue, mention the relevant [Backend Engineering Manager and Product Manager](https://about.gitlab.com/handbook/product/categories/#source-code-group) + In your issue, mention the Create:Source Code [Backend Engineering Manager and Product Manager](https://about.gitlab.com/handbook/product/categories/#source-code-group) for the Templates feature. -## Add the template SVG icon to GitLab SVGs +### Add the template SVG icon to GitLab SVGs If the project template has an SVG icon, you must add it to the [GitLab SVGs project](https://gitlab.com/gitlab-org/gitlab-svgs/-/blob/main/README.md#adding-icons-or-illustrations) before you can create a merge request with vendor details. -## Create a merge request with vendor details +### Create a merge request with vendor details Before GitLab can implement the project template, you must [create a merge request](../user/project/merge_requests/creating_merge_requests.md) in [`gitlab-org/gitlab`](https://gitlab.com/gitlab-org/gitlab) that includes vendor details about the project. @@ -111,7 +113,7 @@ Before GitLab can implement the project template, you must [create a merge reque 1. After you run the scripts, there is one new file in `vendor/project_templates/` and four changed files. Commit all changes and push your branch to update the merge request. For an example, see this [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25318). -## Test your built-in project with the GitLab Development Kit +### Test your built-in project with the GitLab Development Kit Complete the following steps to test the project template in your own GitLab Development Kit instance: @@ -124,9 +126,24 @@ Complete the following steps to test the project template in your own GitLab Dev ## Contribute an improvement to an existing template -To update an existing built-in project template: +To update an existing built-in project template, changes are usually made to the existing template, found in the [project-templates](https://gitlab.com/gitlab-org/project-templates) group. A merge request is made directly against the template and the Create:Source Code [Backend Engineering Manager and Product Manager](https://about.gitlab.com/handbook/product/categories/#source-code-group) pinged for review. + +Sometimes it is necessary to completely replace the template files. In this case the process would be: 1. Create a merge request in the relevant project of the `project-templates` and `pages` group and mention `@gitlab-org/manage/import/backend` when you are ready for a review. 1. If your merge request is accepted, either: - [Create an issue](https://gitlab.com/gitlab-org/gitlab/-/issues) to ask for the template to get updated. - [Create a merge request with vendor details](#create-a-merge-request-with-vendor-details) to update the template. + +## For GitLab team members + +Please ensure the merge request has been reviewed by the Security Counterpart before merging. + +To review a merge request which changes a vendored project template, run the `check-template-changes` script: + +```shell +scripts/check-template-changes vendor/project_templates/<template_name>.tar.gz +``` + +This script outputs a diff of the file changes against the default branch and also verifies that +the template repository matches the source template project. diff --git a/doc/development/projections.md b/doc/development/projections.md index e45a16c267d..7c727fc0901 100644 --- a/doc/development/projections.md +++ b/doc/development/projections.md @@ -20,11 +20,11 @@ Projections are a way to define relations between files. Every file can have a You can find a basic list of projection options in [projectionist.txt](https://github.com/tpope/vim-projectionist/blob/master/doc/projectionist.txt) -## Which plugins can I use +## Which plugins can you use - vim - [vim-projectionist](https://github.com/tpope/vim-projectionist) -- VSCode +- VS Code - [Alternate File](https://marketplace.visualstudio.com/items?itemName=will-wow.vscode-alternate-file) - [projectionist](https://github.com/jarsen/projectionist) - [`jumpto`](https://github.com/gmdayley/jumpto) diff --git a/doc/development/prometheus_metrics.md b/doc/development/prometheus_metrics.md index d3d809c5386..456f2eb50aa 100644 --- a/doc/development/prometheus_metrics.md +++ b/doc/development/prometheus_metrics.md @@ -36,7 +36,7 @@ After you add or change an existing common metric, you must [re-run the import s Or, you can create a database migration: ```ruby -class ImportCommonMetrics < Gitlab::Database::Migration[2.0] +class ImportCommonMetrics < Gitlab::Database::Migration[2.1] def up ::Gitlab::DatabaseImporters::CommonMetrics::Importer.new.execute end diff --git a/doc/development/query_count_limits.md b/doc/development/query_count_limits.md deleted file mode 100644 index f16c8cfc6cd..00000000000 --- a/doc/development/query_count_limits.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/query_count_limits.md' -remove_date: '2022-11-06' ---- - -This document was moved to [another location](database/query_count_limits.md). - -<!-- This redirect file can be deleted after <2022-11-06>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/query_performance.md b/doc/development/query_performance.md deleted file mode 100644 index 618d007f766..00000000000 --- a/doc/development/query_performance.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/query_performance.md' -remove_date: '2022-11-06' ---- - -This document was moved to [another location](database/query_performance.md). - -<!-- This redirect file can be deleted after <2022-11-06>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/query_recorder.md b/doc/development/query_recorder.md deleted file mode 100644 index cb05bc604af..00000000000 --- a/doc/development/query_recorder.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/query_recorder.md' -remove_date: '2022-11-06' ---- - -This document was moved to [another location](database/query_recorder.md). - -<!-- This redirect file can be deleted after <2022-11-06>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/reactive_caching.md b/doc/development/reactive_caching.md index 3239748193f..9e3b1f58abe 100644 --- a/doc/development/reactive_caching.md +++ b/doc/development/reactive_caching.md @@ -208,7 +208,7 @@ There are some `class_attribute` options which can be tweaked. self.reactive_cache_key = ->(integration) { [integration.class.model_name.singular, integration.project_id] } ``` - If your reactive_cache_key is exactly like the above, you can use the existing + If your `reactive_cache_key` is exactly like the above, you can use the existing `Integrations::ReactivelyCached` concern instead. #### `self.reactive_cache_lease_timeout` diff --git a/doc/development/redis/new_redis_instance.md b/doc/development/redis/new_redis_instance.md index d9d4bb6a9f8..0a030a0877f 100644 --- a/doc/development/redis/new_redis_instance.md +++ b/doc/development/redis/new_redis_instance.md @@ -181,7 +181,7 @@ and the [old (fallback-instance)](#fallback-instance). If we fail to fetch data from the new instance, we will fallback and read from the old Redis instance. We can monitor logs for `Gitlab::Redis::MultiStore::ReadFromPrimaryError`, and also the Prometheus counter `gitlab_redis_multi_store_read_fallback_total`. -For pipelined commands (`pipelined` and `multi`), we execute the entire operation in both stores and then compare the results. If they differ, we emit a +For `pipelined` commands (`pipelined` and `multi`), we execute the entire operation in both stores and then compare the results. If they differ, we emit a `Gitlab::Redis::MultiStore:PipelinedDiffError` error, and track it in the `gitlab_redis_multi_store_pipelined_diff_error_total` Prometheus counter. Once we stop seeing those errors, this means that we are no longer relying on the data stored on the old Redis store. @@ -218,7 +218,7 @@ MultiStore implements read and write Redis commands separately. - `flushdb` - `rpush` -##### Pipelined commands +##### `pipelined` commands **NOTE:** The Ruby block passed to these commands will be executed twice, once per each store. Thus, excluding the Redis operations performed, the block should be idempotent. @@ -238,16 +238,16 @@ a developer will need to add an implementation for missing Redis commands before | error | message | |---------------------------------------------------|---------------------------------------------------------------------------------------------| | `Gitlab::Redis::MultiStore::ReadFromPrimaryError` | Value not found on the Redis primary store. Read from the Redis secondary store successful. | -| `Gitlab::Redis::MultiStore::PipelinedDiffError` | Pipelined command executed on both stores successfully but results differ between them. | +| `Gitlab::Redis::MultiStore::PipelinedDiffError` | `pipelined` command executed on both stores successfully but results differ between them. | | `Gitlab::Redis::MultiStore::MethodMissingError` | Method missing. Falling back to execute method on the Redis secondary store. | ##### Metrics -| metrics name | type | labels | description | -|-------------------------------------------------------|--------------------|------------------------|--------------------------------------------------------| -| `gitlab_redis_multi_store_read_fallback_total` | Prometheus Counter | command, instance_name | Client side Redis MultiStore reading fallback total | -| `gitlab_redis_multi_store_pipelined_diff_error_total` | Prometheus Counter | command, instance_name | Redis MultiStore pipelined command diff between stores | -| `gitlab_redis_multi_store_method_missing_total` | Prometheus Counter | command, instance_name | Client side Redis MultiStore method missing total | +| Metrics name | Type | Labels | Description | +|-------------------------------------------------------|--------------------|----------------------------|----------------------------------------------------------| +| `gitlab_redis_multi_store_read_fallback_total` | Prometheus Counter | `command`, `instance_name` | Client side Redis MultiStore reading fallback total | +| `gitlab_redis_multi_store_pipelined_diff_error_total` | Prometheus Counter | `command`, `instance_name` | Redis MultiStore `pipelined` command diff between stores | +| `gitlab_redis_multi_store_method_missing_total` | Prometheus Counter | `command`, `instance_name` | Client side Redis MultiStore method missing total | ## Step 4: clean up after the migration diff --git a/doc/development/reusing_abstractions.md b/doc/development/reusing_abstractions.md index 58d1e20394c..e3f523fc6a7 100644 --- a/doc/development/reusing_abstractions.md +++ b/doc/development/reusing_abstractions.md @@ -198,7 +198,7 @@ Several base classes implement the service classes convention. You may consider - `BaseGroupService` for services scoped to groups. Classes that are not service objects should be -[created elsewhere](directory_structure.md#use-namespaces-to-define-bounded-contexts), +[created elsewhere](software_design.md#use-namespaces-to-define-bounded-contexts), such as in `lib`. #### ServiceResponse diff --git a/doc/development/scalability.md b/doc/development/scalability.md index 671086f33b2..de9c57c2f2a 100644 --- a/doc/development/scalability.md +++ b/doc/development/scalability.md @@ -267,7 +267,7 @@ However, there are a number of strategies to ensure queues get drained in a timely manner: - Add more processing capacity. This can be done by spinning up more - instances of Sidekiq or [Sidekiq Cluster](../administration/operations/extra_sidekiq_processes.md). + instances of Sidekiq or [Sidekiq Cluster](../administration/sidekiq/extra_sidekiq_processes.md). - Split jobs into smaller units of work. For example, `PostReceive` used to process each commit message in the push, but now it farms out this to `ProcessCommitWorker`. diff --git a/doc/development/sec/analyzer_development_guide.md b/doc/development/sec/analyzer_development_guide.md index af3a6f2b7d7..4fb32785b9f 100644 --- a/doc/development/sec/analyzer_development_guide.md +++ b/doc/development/sec/analyzer_development_guide.md @@ -21,7 +21,7 @@ There are a number of shared Go modules shared across analyzers for common behav ## How to use the analyzers Analyzers are shipped as Docker images. For example, to run the -[semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) Docker image to scan the working directory: +[Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) Docker image to scan the working directory: 1. `cd` into the directory of the source code you want to scan. 1. Run `docker login registry.gitlab.com` and provide username plus diff --git a/doc/development/sec/index.md b/doc/development/sec/index.md index fc13c960451..3f52020701f 100644 --- a/doc/development/sec/index.md +++ b/doc/development/sec/index.md @@ -102,15 +102,15 @@ After being [merged](../integrations/secure.md#tracking-and-merging-vulnerabilit ### Analyzer vulnerability translation -In the case of SAST's semgrep analyzer, there is a secondary identifier of particular importance: the identifier linking the report’s vulnerability -to the legacy analyzer (that is, bandit or eslint). +In the case of the SAST Semgrep analyzer, there is a secondary identifier of particular importance: the identifier linking the report’s vulnerability +to the legacy analyzer (that is, bandit or ESLint). To [enable vulnerability translation](../../user/application_security/sast/analyzers.md#vulnerability-translation) -the semgrep analyzer relies on a secondary identifier exactly matching the primary identifier of the legacy analyzer. +the Semgrep analyzer relies on a secondary identifier exactly matching the primary identifier of the legacy analyzer. For example, when [`eslint`](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) was previously used to generate vulnerability records, the [`semgrep`](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) analyzer must produce an identifier collection containing the -original eslint primary identifier. +original ESLint primary identifier. Given the original `eslint` report: @@ -131,7 +131,7 @@ Given the original `eslint` report: } ``` -The corresponding semgrep report must contain the `eslint_rule_id`: +The corresponding Semgrep report must contain the `eslint_rule_id`: ```json { diff --git a/doc/development/secure_coding_guidelines.md b/doc/development/secure_coding_guidelines.md index c102e99720f..bccdda9ca04 100644 --- a/doc/development/secure_coding_guidelines.md +++ b/doc/development/secure_coding_guidelines.md @@ -410,7 +410,7 @@ References: #### XSS mitigation and prevention in JavaScript and Vue - When updating the content of an HTML element using JavaScript, mark user-controlled values as `textContent` or `nodeValue` instead of `innerHTML`. -- Avoid using `v-html` with user-controlled data, use [`v-safe-html`](https://gitlab-org.gitlab.io/gitlab-ui/?path=/story/directives-safe-html-directive--default) instead. +- Avoid using `v-html` with user-controlled data, use [`v-safe-html`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/vue_shared/directives/safe_html.js) instead. - Render unsafe or unsanitized content using [`dompurify`](fe_guide/security.md#sanitize-html-output). - Consider using [`gl-sprintf`](../../ee/development/i18n/externalization.md#interpolation) to interpolate translated strings securely. - Avoid `__()` with translations that contain user-controlled values. @@ -422,7 +422,7 @@ References: ##### Vue - [isSafeURL](https://gitlab.com/gitlab-org/gitlab/-/blob/v12.7.5-ee/app/assets/javascripts/lib/utils/url_utility.js#L190-207) -- [GlSprintf](https://gitlab-org.gitlab.io/gitlab-ui/?path=/story/utilities-sprintf--default) +- [GlSprintf](https://gitlab-org.gitlab.io/gitlab-ui/?path=/docs/utilities-sprintf--sentence-with-link) #### Content Security Policy @@ -718,13 +718,12 @@ There are some cases where `users` passed in the code is actually referring to a ```ruby def find_user_from_sources - strong_memoize(:find_user_from_sources) do - deploy_token_from_request || - find_user_from_bearer_token || - find_user_from_job_token || - user_from_warden - end + deploy_token_from_request || + find_user_from_bearer_token || + find_user_from_job_token || + user_from_warden end + strong_memoize_attr :find_user_from_sources ``` ### Past Vulnerable Code @@ -1210,7 +1209,7 @@ These types of bugs are often seen in environments which allow multi-threading a ### Examples -**Example 1:** you have a model which accepts a URL as input. When the model is created you verify that the URL's host resolves to a public IP address, to prevent attackers making internal network calls. But DNS records can change ([DNS rebinding](#server-side-request-forgery-ssrf)]). An attacker updates the DNS record to `127.0.0.1`, and when your code resolves those URL's host it results in sending a potentially malicious request to a server on the internal network. The property was valid at the "time of check", but invalid and malicious at "time of use". +**Example 1:** you have a model which accepts a URL as input. When the model is created you verify that the URL host resolves to a public IP address, to prevent attackers making internal network calls. But DNS records can change ([DNS rebinding](#server-side-request-forgery-ssrf)]). An attacker updates the DNS record to `127.0.0.1`, and when your code resolves those URL host it results in sending a potentially malicious request to a server on the internal network. The property was valid at the "time of check", but invalid and malicious at "time of use". GitLab-specific example can be found in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/214401) where, although `Gitlab::UrlBlocker.validate!` was called, the returned value was not used. This made it vulnerable to TOCTOU bug and SSRF protection bypass through [DNS rebinding](#server-side-request-forgery-ssrf). The fix was to [use the validated IP address](https://gitlab.com/gitlab-org/gitlab/-/commit/7af8abd4df9a98f7a1ae7c4ec9840d0a7a8c684d). diff --git a/doc/development/serializing_data.md b/doc/development/serializing_data.md deleted file mode 100644 index aa8b20eded7..00000000000 --- a/doc/development/serializing_data.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/serializing_data.md' -remove_date: '2022-11-06' ---- - -This document was moved to [another location](database/serializing_data.md). - -<!-- This redirect file can be deleted after <2022-11-06>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/service_ping/implement.md b/doc/development/service_ping/implement.md index 5a564b2d83e..70da97502bb 100644 --- a/doc/development/service_ping/implement.md +++ b/doc/development/service_ping/implement.md @@ -181,7 +181,7 @@ The highest encountered error rate is 4.9%. When correctly used, the `estimate_batch_distinct_count` method enables efficient counting over columns that contain non-unique values, which cannot be assured by other counters. -##### estimate_batch_distinct_count method +##### `estimate_batch_distinct_count` method Method: @@ -316,7 +316,7 @@ HyperLogLog (HLL) is a probabilistic algorithm and its **results always includes used HLL implementation is "approximated with a standard error of 0.81%". NOTE: - A user's consent for usage_stats (`User.single_user&.requires_usage_stats_consent?`) is not checked during the data tracking stage due to performance reasons. Keys corresponding to those counters are present in Redis even if `usage_stats_consent` is still required. However, no metric is collected from Redis and reported back to GitLab as long as `usage_stats_consent` is required. + A user's consent for `usage_stats` (`User.single_user&.requires_usage_stats_consent?`) is not checked during the data tracking stage due to performance reasons. Keys corresponding to those counters are present in Redis even if `usage_stats_consent` is still required. However, no metric is collected from Redis and reported back to GitLab as long as `usage_stats_consent` is required. With `Gitlab::UsageDataCounters::HLLRedisCounter` we have available data structures used to count unique values. @@ -509,7 +509,7 @@ We have the following recommendations for [adding new events](#add-new-events): Events are tracked behind optional [feature flags](../feature_flags/index.md) due to concerns for Redis performance and scalability. -For a full list of events and corresponding feature flags see, [known_events](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/) files. +For a full list of events and corresponding feature flags, see the [`known_events/`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/) files. To enable or disable tracking for specific event in <https://gitlab.com> or <https://about.staging.gitlab.com>, run commands such as the following to [enable or disable the corresponding feature](../feature_flags/index.md). @@ -870,7 +870,7 @@ these steps: Only metrics calculated with [Estimated Batch Counters](#estimated-batch-counters) can be persisted for database sourced aggregated metrics. To persist a metric, inject a Ruby block into the -[estimate_batch_distinct_count](#estimate_batch_distinct_count-method) method. +[`estimate_batch_distinct_count`](#estimate_batch_distinct_count-method) method. This block should invoke the `Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll.save_aggregated_metrics` [method](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb#L21), diff --git a/doc/development/service_ping/metrics_dictionary.md b/doc/development/service_ping/metrics_dictionary.md index 3439f581e7f..49f8a5ac465 100644 --- a/doc/development/service_ping/metrics_dictionary.md +++ b/doc/development/service_ping/metrics_dictionary.md @@ -41,7 +41,7 @@ Each metric is defined in a separate YAML file consisting of a number of fields: | `value_type` | yes | `string`; one of [`string`, `number`, `boolean`, `object`](https://json-schema.org/understanding-json-schema/reference/type.html). | | `status` | yes | `string`; [status](#metric-statuses) of the metric, may be set to `active`, `removed`, `broken`. | | `time_frame` | yes | `string`; may be set to a value like `7d`, `28d`, `all`, `none`. | -| `data_source` | yes | `string`; may be set to a value like `database`, `redis`, `redis_hll`, `prometheus`, `system`. | +| `data_source` | yes | `string`; may be set to a value like `database`, `redis`, `redis_hll`, `prometheus`, `system`, `license`. | | `data_category` | yes | `string`; [categories](#data-category) of the metric, may be set to `operational`, `optional`, `subscription`, `standard`. The default value is `optional`.| | `instrumentation_class` | yes | `string`; [the class that implements the metric](metrics_instrumentation.md). | | `distribution` | yes | `array`; may be set to one of `ce, ee` or `ee`. The [distribution](https://about.gitlab.com/handbook/marketing/strategic-marketing/tiers/#definitions) where the tracked feature is available. | @@ -55,7 +55,7 @@ Each metric is defined in a separate YAML file consisting of a number of fields: | `options` | no | `object`: options information needed to calculate the metric value. | | `skip_validation` | no | This should **not** be set. [Used for imported metrics until we review, update and make them valid](https://gitlab.com/groups/gitlab-org/-/epics/5425). | -### Metric key_path +### Metric `key_path` The `key_path` of the metric is the location in the JSON Service Ping payload. @@ -108,7 +108,7 @@ Metric definitions can have one of the following statuses: - `broken`: Metric reports broken data (for example, -1 fallback), or does not report data at all. A metric marked as `broken` must also have the `repair_issue_url` attribute. - `removed`: Metric was removed, but it may appear in Service Ping payloads sent from instances running on older versions of GitLab. -### Metric value_type +### Metric `value_type` Metric definitions can have one of the following values for `value_type`: @@ -120,12 +120,23 @@ In general, we avoid complex objects and prefer one of the `boolean`, `number`, An example of a metric that uses `value_type: object` is `topology` (`/config/metrics/settings/20210323120839_topology.yml`), which has a related schema in `/config/metrics/objects_schemas/topology_schema.json`. -### Metric time_frame - -- `7d`: The metric data applies to the most recent 7-day interval. For example, the following metric counts the number of users that create epics over a 7-day interval: `ee/config/metrics/counts_7d/20210305145820_g_product_planning_epic_created_weekly.yml`. -- `28d`: The metric data applies to the most recent 28-day interval. For example, the following metric counts the number of unique users that create issues over a 28-day interval: `config/metrics/counts_28d/20210216181139_issues.yml`. -- `all`: The metric data applies for the whole time the metric has been active (all-time interval). For example, the following metric counts all users that create issues: `/config/metrics/counts_all/20210216181115_issues.yml`. -- `none`: The metric collects a type of data that's not tracked over time, such as settings and configuration information. Therefore, a time interval is not applicable. For example, `uuid` has no time interval applicable: `config/metrics/license/20210201124933_uuid.yml`. +### Metric `time_frame` + +A metric's time frame is calculated based on the `time_frame` field and the `data_source` of the metric. +For `redis_hll` metrics, the type of aggregation is also taken into consideration. In this context, the term "aggregation" refers to [chosen events data storage interval](implement.md#add-new-events), and is **NOT** related to the Aggregated Metrics feature. +For more information about the aggregation type of each feature, see the [`common.yml` file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/common.yml). Weeks run from Monday to Sunday. + +| data_source | time_frame | aggregation | Description | +|------------------------|------------|----------------|-------------------------------------------------| +| any | `none` | not applicable | A type of data that’s not tracked over time, such as settings and configuration information | +| `database` | `all` | not applicable | The whole time the metric has been active (all-time interval) | +| `database` | `7d` | not applicable | 9 days ago to 2 days ago | +| `database` | `28d` | not applicable | 30 days ago to 2 days ago | +| `redis` | `all` | not applicable | The whole time the metric has been active (all-time interval) | +| `redis_hll` | `7d` | `daily` | Most recent 7 complete days | +| `redis_hll` | `7d` | `weekly` | Most recent complete week | +| `redis_hll` | `28d` | `daily` | Most recent 28 complete days | +| `redis_hll` | `28d` | `weekly` | Most recent 4 complete weeks | ### Data category diff --git a/doc/development/service_ping/metrics_instrumentation.md b/doc/development/service_ping/metrics_instrumentation.md index a9f236819fe..5cc8a7811d7 100644 --- a/doc/development/service_ping/metrics_instrumentation.md +++ b/doc/development/service_ping/metrics_instrumentation.md @@ -464,3 +464,15 @@ This guide describes how to migrate a Service Ping metric from [`lib/gitlab/usag 1. Remove the code from [`lib/gitlab/usage_data.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb) or [`ee/lib/ee/gitlab/usage_data.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/ee/gitlab/usage_data.rb). 1. Remove the tests from [`spec/lib/gitlab/usage_data.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/lib/gitlab/usage_data_spec.rb) or [`ee/spec/lib/ee/gitlab/usage_data.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/spec/lib/ee/gitlab/usage_data_spec.rb). + +## Troubleshoot metrics + +Sometimes metrics fail for reasons that are not immediately clear. The failures can be related to performance issues or other problems. +The following pairing session video gives you an example of an investigation in to a real-world failing metric. + +<div class="video-fallback"> + See the video from: <a href="https://www.youtube.com/watch?v=y_6m2POx2ug">Product Intelligence Office Hours Oct 27th</a> to learn more about the metrics troubleshooting process. +</div> +<figure class="video-container"> + <iframe src="https://www.youtube.com/embed/y_6m2POx2ug" frameborder="0" allowfullscreen="true"> </iframe> +</figure> diff --git a/doc/development/service_ping/troubleshooting.md b/doc/development/service_ping/troubleshooting.md index f8fd45e6062..3b7cd092d97 100644 --- a/doc/development/service_ping/troubleshooting.md +++ b/doc/development/service_ping/troubleshooting.md @@ -30,7 +30,7 @@ For results about an investigation conducted into an unexpected drop in Service Check if the [export jobs](https://gitlab.com/gitlab-services/version-gitlab-com#data-export-using-pipeline-schedules) are successful. -Check [Service Ping errors](https://app.periscopedata.com/app/gitlab/968489/Product-Intelligence---Service-Ping-Health?widget=14609989&udv=0) in the [Service Ping Health Dahsboard](https://app.periscopedata.com/app/gitlab/968489/Product-Intelligence---Service-Ping-Health). +Check [Service Ping errors](https://app.periscopedata.com/app/gitlab/968489/Product-Intelligence---Service-Ping-Health?widget=14609989&udv=0) in the [Service Ping Health Dashboard](https://app.periscopedata.com/app/gitlab/968489/Product-Intelligence---Service-Ping-Health). ### Troubleshoot Google Storage layer diff --git a/doc/development/sha1_as_binary.md b/doc/development/sha1_as_binary.md deleted file mode 100644 index 7f928d09470..00000000000 --- a/doc/development/sha1_as_binary.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/sha1_as_binary.md' -remove_date: '2022-11-06' ---- - -This document was moved to [another location](database/sha1_as_binary.md). - -<!-- This redirect file can be deleted after <2022-11-06>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/sidekiq/compatibility_across_updates.md b/doc/development/sidekiq/compatibility_across_updates.md index cfb709d9110..b417a099228 100644 --- a/doc/development/sidekiq/compatibility_across_updates.md +++ b/doc/development/sidekiq/compatibility_across_updates.md @@ -123,14 +123,62 @@ uses a parameter hash. end ``` -## Removing workers +## Removing worker classes -Try to avoid removing workers and their queues in minor and patch -releases. +To remove a worker class, follow these steps over two minor releases: -During online update instance can have pending jobs and removing the queue can -lead to those jobs being stuck forever. If you can't write migration for those -Sidekiq jobs, please consider removing the worker in a major release only. +### In the first minor release + +1. Remove any code that enqueues the jobs. + + For example, if there is a UI component or an API endpoint that a user can interact with that results in the worker instance getting enqueued, make sure those surface areas are either removed or updated in a way that the worker instance is no longer enqueued. + + This ensures that instances related to the worker class are no longer being enqueued. + +1. Ensure both the frontend and backend code no longer relies on any of the work that used to be done by the worker. +1. In the relevant worker classes, replace the contents of the `perform` method with a no-op, while keeping any arguments in tact. + + For example, if you're working with the following `ExampleWorker`: + + ```ruby + class ExampleWorker + def perform(object_id) + SomeService.run!(object_id) + end + end + ``` + + Implementing the no-op might look like this: + + ```ruby + class ExampleWorker + def perform(object_id); end + end + ``` + + By implementing this no-op, you can avoid unnecessary cycles once any deprecated jobs that are still enqueued eventually get processed. + +### In a subsequent, separate minor release + +1. Delete the worker class file and follow the guidance in our [Sidekiq queues documentation](../sidekiq/index.md#sidekiq-queues) around running Rake tasks to regenerate/update related files. +1. Add a migration (not a post-deployment migration) that uses `sidekiq_remove_jobs`: + + ```ruby + class RemoveMyDeprecatedWorkersJobInstances < Gitlab::Database::Migration[2.0] + DEPRECATED_JOB_CLASSES = %w[ + MyDeprecatedWorkerOne + MyDeprecatedWorkerTwo + ] + + def up + sidekiq_remove_jobs(job_klasses: DEPRECATED_JOB_CLASSES) + end + + def down + # This migration removes any instances of deprecated workers and cannot be undone. + end + end + ``` ## Renaming queues @@ -141,7 +189,7 @@ When renaming queues, use the `sidekiq_queue_migrate` helper migration method in a **post-deployment migration**: ```ruby -class MigrateTheRenamedSidekiqQueue < Gitlab::Database::Migration[2.0] +class MigrateTheRenamedSidekiqQueue < Gitlab::Database::Migration[2.1] restrict_gitlab_migration gitlab_schema: :gitlab_main disable_ddl_transaction! diff --git a/doc/development/sidekiq/idempotent_jobs.md b/doc/development/sidekiq/idempotent_jobs.md index 80c6c403549..1c4f4ba44a8 100644 --- a/doc/development/sidekiq/idempotent_jobs.md +++ b/doc/development/sidekiq/idempotent_jobs.md @@ -156,7 +156,7 @@ end ## Setting the deduplication time-to-live (TTL) -Deduplication depends on an idempotency key that is stored in Redis. This is normally +Deduplication depends on an idempotent key that is stored in Redis. This is normally cleared by the configured deduplication strategy. However, the key can remain until its TTL in certain cases like: @@ -189,7 +189,7 @@ that can tolerate some duplication. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69372) in GitLab 14.3. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/338350) in GitLab 14.4. > - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/338350) in GitLab 14.6. -> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/346598) in GitLab 14.9. [Feature flag preserve_latest_wal_locations_for_idempotent_jobs](https://gitlab.com/gitlab-org/gitlab/-/issues/346598) removed. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/346598) in GitLab 14.9. [Feature flag `preserve_latest_wal_locations_for_idempotent_jobs`](https://gitlab.com/gitlab-org/gitlab/-/issues/346598) removed. The deduplication always take into account the latest binary replication pointer, not the first one. This happens because we drop the same job scheduled for the second time and the Write-Ahead Log (WAL) is lost. diff --git a/doc/development/sidekiq/index.md b/doc/development/sidekiq/index.md index a95e94cdd34..c9d783377bd 100644 --- a/doc/development/sidekiq/index.md +++ b/doc/development/sidekiq/index.md @@ -14,7 +14,7 @@ information on administering GitLab, see [configuring Sidekiq](../../administrat There are pages with additional detail on the following topics: 1. [Compatibility across updates](compatibility_across_updates.md) -1. [Job idempotency and job deduplication](idempotent_jobs.md) +1. [Job idempotence and job deduplication](idempotent_jobs.md) 1. [Limited capacity worker: continuously performing work with a specified concurrency](limited_capacity_worker.md) 1. [Logging](logging.md) 1. [Worker attributes](worker_attributes.md) @@ -27,7 +27,7 @@ There are pages with additional detail on the following topics: All workers should include `ApplicationWorker` instead of `Sidekiq::Worker`, which adds some convenience methods and automatically sets the queue based on -the [routing rules](../../administration/sidekiq/extra_sidekiq_routing.md#queue-routing-rules). +the [routing rules](../../administration/sidekiq/processing_specific_job_classes.md#routing-rules). ## Retries @@ -88,7 +88,7 @@ error rate. Previously, each worker had its own queue, which was automatically set based on the worker class name. For a worker named `ProcessSomethingWorker`, the queue name would be `process_something`. You can now route workers to a specific queue using -[queue routing rules](../../administration/sidekiq/extra_sidekiq_routing.md#queue-routing-rules). +[queue routing rules](../../administration/sidekiq/processing_specific_job_classes.md#routing-rules). In GDK, new workers are routed to a queue named `default`. If you're not sure what queue a worker uses, diff --git a/doc/development/single_table_inheritance.md b/doc/development/single_table_inheritance.md deleted file mode 100644 index da8d48f2a42..00000000000 --- a/doc/development/single_table_inheritance.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/single_table_inheritance.md' -remove_date: '2022-11-06' ---- - -This document was moved to [another location](database/single_table_inheritance.md). - -<!-- This redirect file can be deleted after <2022-11-06>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/snowplow/index.md b/doc/development/snowplow/index.md index 8d05e05e592..32628744a23 100644 --- a/doc/development/snowplow/index.md +++ b/doc/development/snowplow/index.md @@ -17,7 +17,7 @@ Snowplow is an enterprise-grade marketing and Product Intelligence platform that - **Data modeling** joins event-level data with other data sets, aggregates them into smaller data sets, and applies business logic. This produces a clean set of tables for data analysis. We use data models for Redshift and Looker. - **Analytics** are performed on Snowplow events or on aggregate tables. - + ## Enable Snowplow tracking diff --git a/doc/development/snowplow/schemas.md b/doc/development/snowplow/schemas.md index 5a6cdea9fee..da58cd5f2e5 100644 --- a/doc/development/snowplow/schemas.md +++ b/doc/development/snowplow/schemas.md @@ -68,8 +68,8 @@ Page titles are hardcoded as `GitLab` for the same reason. | `doc_charset` | **{dotted-circle}** | string | Web page's character encoding | | `doc_height` | **{dotted-circle}** | string | Web page height | | `doc_width` | **{dotted-circle}** | string | Web page width | -| `domain_sessionid` | **{dotted-circle}** | string | Unique identifier (UUID) for this visit of this user_id to this domain | -| `domain_sessionidx` | **{dotted-circle}** | integer | Index of number of visits that this user_id has made to this domain (The first visit is `1`) | +| `domain_sessionid` | **{dotted-circle}** | string | Unique identifier (UUID) for this visit of this `user_id` to this domain | +| `domain_sessionidx` | **{dotted-circle}** | integer | Index of number of visits that this `user_id` has made to this domain (The first visit is `1`) | | `domain_userid` | **{dotted-circle}** | string | Unique identifier for a user, based on a first party cookie (so domain specific) | | `dvce_created_tstamp` | **{dotted-circle}** | timestamp | Timestamp when event occurred, as recorded by client device | | `dvce_ismobile` | **{dotted-circle}** | boolean | Indicates whether device is mobile | diff --git a/doc/development/snowplow/troubleshooting.md b/doc/development/snowplow/troubleshooting.md index 306040f8c9c..47df3e43d57 100644 --- a/doc/development/snowplow/troubleshooting.md +++ b/doc/development/snowplow/troubleshooting.md @@ -35,7 +35,7 @@ Drop occurring at application layer can be symptom of some issue, but it might b or even a result of a public holiday in some regions of the world with a larger user-base. To verify if there is an underlying problem to solve, you can check following things: 1. Check `about.gitlab.com` website traffic on [Google Analytics](https://analytics.google.com/analytics/web/) to verify if some public holiday might impact overall use of GitLab system - 1. You may require to open an access request for Google Analytics access first eg: [access request internal issue](https://gitlab.com/gitlab-com/team-member-epics/access-requests/-/issues/1772) + 1. You may require to open an access request for Google Analytics access first, for example: [access request internal issue](https://gitlab.com/gitlab-com/team-member-epics/access-requests/-/issues/1772) 1. Plot `select date(dvce_created_tstamp) , event , count(*) from legacy.snowplow_unnested_events_90 where dvce_created_tstamp > '2021-06-15' and dvce_created_tstamp < '2021-07-10' group by 1 , 2 order by 1 , 2` in SiSense to see what type of events was responsible for drop 1. Plot `select date(dvce_created_tstamp) ,se_category , count(*) from legacy.snowplow_unnested_events_90 where dvce_created_tstamp > '2021-06-15' and dvce_created_tstamp < '2021-07-31' and event = 'struct' group by 1 , 2 order by 1, 2` what events recorded the biggest drops in suspected category 1. Check if there was any MR merged that might cause reduction in reported events, pay an attention to ~"product intelligence" and ~"growth experiment" labeled MRs diff --git a/doc/development/software_design.md b/doc/development/software_design.md new file mode 100644 index 00000000000..03cbbb13d9f --- /dev/null +++ b/doc/development/software_design.md @@ -0,0 +1,141 @@ +--- +stage: none ++group: Engineering Productivity +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Software design guides + +## Use ubiquitous language instead of CRUD terminology + +The code should use the same [ubiquitous language](https://about.gitlab.com/handbook/communication/#ubiquitous-language) +as used in the product and user documentation. Failure to use ubiquitous language correctly +can be a major cause of confusion for contributors and customers when there is constant translation +or use of multiple terms. +This also goes against our [communication strategy](https://about.gitlab.com/handbook/communication/#mecefu-terms). + +In the example below, [CRUD](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete) +terminology introduces ambiguity. The name says we are creating an `epic_issues` +association record, but we are adding an existing issue to an epic. The name `epic_issues`, +used from Rails convention, leaks to higher abstractions such as service objects. +The code speaks the framework jargon rather than ubiquitous language. + +```ruby +# Bad +EpicIssues::CreateService +``` + +Using ubiquitous language makes the code clear and doesn't introduce any +cognitive load to a reader trying to translate the framework jargon. + +```ruby +# Good +Epic::AddExistingIssueService +``` + +You can use CRUD when representing simple concepts that are not ambiguous, +like creating a project, and when matching the existing ubiquitous language. + +```ruby +# OK: Matches the product language. +Projects::CreateService +``` + +New classes and database tables should use ubiquitous language. In this case the model name +and table name follow the Rails convention. + +Existing classes that don't follow ubiquitous language should be renamed, when possible. +Some low level abstractions such as the database tables don't need to be renamed. +For example, use `self.table_name=` when the model name diverges from the table name. + +We can allow exceptions only when renaming is challenging. For example, when the naming is used +for STI, exposed to the user, or if it would be a breaking change. + +## Use namespaces to define bounded contexts + +A healthy application is divided into macro and sub components that represent the contexts at play, +whether they are related to business domain or infrastructure code. + +As GitLab code has so many features and components it's hard to see what contexts are involved. +We should expect any class to be defined inside a module/namespace that represents the contexts where it operates. + +When we namespace classes inside their domain: + +- Similar terminology becomes unambiguous as the domain clarifies the meaning: + For example, `MergeRequests::Diff` and `Notes::Diff`. +- Top-level namespaces could be associated to one or more groups identified as domain experts. +- We can better identify the interactions and coupling between components. + For example, several classes inside `MergeRequests::` domain interact more with `Ci::` + domain and less with `ImportExport::`. + +A good guideline for naming a top-level namespace (bounded context) is to use the related +[feature category](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/data/categories.yml). +For example, `Continuous Integration` feature category maps to `Ci::` namespace. + +```ruby +# bad +class JobArtifact +end + +# good +module Ci + class JobArtifact + end +end +``` + +Projects and Groups are generally container concepts because they identify tenants. +They allow features to exist at the project or group level, like repositories or runners, +but do not nest such features under `Projects::` or `Groups::`. + +`Projects::` and `Groups::` namespaces should be used only for concepts that are strictly related to them: +for example `Project::CreateService` or `Groups::TransferService`. + +For controllers we allow `app/controllers/projects` and `app/controllers/groups` to be exceptions. +We use this convention to indicate the scope of a given web endpoint. + +Do not use the [stage or group name](https://about.gitlab.com/handbook/product/categories/#devops-stages) +because a feature category could be reassigned to a different group in the future. + +```ruby +# bad +module Create + class Commit + end +end + +# good +module Repositories + class Commit + end +end +``` + +On the other hand, a feature category may sometimes be too granular. Features tend to be +treated differently according to Product and Marketing, while they may share a lot of +domain models and behavior under the hood. In this case, having too many bounded contexts +could make them shallow and more coupled with other contexts. + +Bounded contexts (or top-level namespaces) can be seen as macro-components in the overall app. +Good bounded contexts should be [deep](https://medium.com/@nakabonne/depth-of-module-f62dac3c2fdb) +so consider having nested namespaces to further break down complex parts of the domain. +For example, `Ci::Config::`. + +For example, instead of having separate and granular bounded contexts like: `ContainerScanning::`, +`ContainerHostSecurity::`, `ContainerNetworkSecurity::`, we could have: + +```ruby +module ContainerSecurity + module HostSecurity + end + + module NetworkSecurity + end + + module Scanning + end +end +``` + +If classes that are defined into a namespace have a lot in common with classes in other namespaces, +chances are that these two namespaces are part of the same bounded context. diff --git a/doc/development/spam_protection_and_captcha/exploratory_testing.md b/doc/development/spam_protection_and_captcha/exploratory_testing.md index f6e3e6814a8..1bcd336ce93 100644 --- a/doc/development/spam_protection_and_captcha/exploratory_testing.md +++ b/doc/development/spam_protection_and_captcha/exploratory_testing.md @@ -353,7 +353,7 @@ GraphQL response: } ``` -### Scenario: allow_possible_spam feature flag enabled +### Scenario: `allow_possible_spam` feature flag enabled With the `allow_possible_spam` feature flag enabled, the API returns a 200 response. Any valid request is successful and no CAPTCHA is presented, even if the request is considered diff --git a/doc/development/sql.md b/doc/development/sql.md index cdc952eb08b..5dbfd8f3ddb 100644 --- a/doc/development/sql.md +++ b/doc/development/sql.md @@ -103,7 +103,7 @@ transaction. Transactions for migrations can be disabled using the following pattern: ```ruby -class MigrationName < Gitlab::Database::Migration[2.0] +class MigrationName < Gitlab::Database::Migration[2.1] disable_ddl_transaction! end ``` @@ -111,7 +111,7 @@ end For example: ```ruby -class AddUsersLowerUsernameEmailIndexes < Gitlab::Database::Migration[2.0] +class AddUsersLowerUsernameEmailIndexes < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up diff --git a/doc/development/swapping_tables.md b/doc/development/swapping_tables.md deleted file mode 100644 index eaa6568dc36..00000000000 --- a/doc/development/swapping_tables.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/swapping_tables.md' -remove_date: '2022-11-04' ---- - -This document was moved to [another location](database/swapping_tables.md). - -<!-- This redirect file can be deleted after <2022-11-04>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index b6bf3c7805a..aee3e2871c2 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -425,11 +425,10 @@ results are available, and not just the first failure. when you need an ID/IID/access level that doesn't actually exists. Using 123, 1234, or even 999 is brittle as these IDs could actually exist in the database in the context of a CI run. -- All top-level `RSpec.describe` blocks should have [`feature_category`](https://about.gitlab.com/categories.json) metadata set. - Consider splitting the file in the case there are identified multiple feature categories in same file. - If no `feature_category` is identified then use `not_owned`. This information is used in flaky test - issues created in order to identify the group owning the feature. - Eg: `RSpec.describe Admin::Geo::SettingsController, :geo, feature_category: :geo_replication do`. + +### Feature category metadata + +You must [set feature category metadata for each RSpec example](../feature_categorization/index.md#rspec-examples). ### Coverage @@ -845,6 +844,8 @@ it 'is overdue' do travel_to(3.days.from_now) do expect(issue).to be_overdue end + + travel_back # Returns the current time back to its original state end ``` @@ -923,6 +924,21 @@ sequence-generated column. To avoid accidental conflicts, specs should also avoid manually specifying any values in these kinds of columns. Instead, leave them unspecified, and look up the value after the row is created. +##### TestProf in migration specs + +Because of what is described above, migration specs can't be run inside +a database transaction. Our test suite uses +[TestProf](https://github.com/test-prof/test-prof) to improve the runtime of the +test suite, but `TestProf` uses database transactions to perform these optimizations. +For this reason, we can't use `TestProf` methods in our migration specs. +These are the methods that should not be used and should be replaced with +default RSpec methods instead: + +- `let_it_be`: use `let` or `let!` instead. +- `let_it_be_with_reload`: use `let` or `let!` instead. +- `let_it_be_with_refind`: use `let` or `let!` instead. +- `before_all`: use `before` or `before(:all)` instead. + #### Redis GitLab stores two main categories of data in Redis: cached items, and Sidekiq @@ -1327,7 +1343,7 @@ Testing query performance allows us to: `QueryRecorder` allows profiling and testing of the number of database queries performed in a given block of code. -See the [`QueryRecorder`](../query_recorder.md) section for more details. +See the [`QueryRecorder`](../database/query_recorder.md) section for more details. #### GitalyClient @@ -1439,7 +1455,7 @@ or [cause the universe to implode](../contributing/verify/index.md#do-not-cause- ### Factories -GitLab uses [factory_bot](https://github.com/thoughtbot/factory_bot) as a test fixture replacement. +GitLab uses [`factory_bot`](https://github.com/thoughtbot/factory_bot) as a test fixture replacement. - Factory definitions live in `spec/factories/`, named using the pluralization of their corresponding model (`User` factories are defined in `users.rb`). @@ -1451,11 +1467,51 @@ GitLab uses [factory_bot](https://github.com/thoughtbot/factory_bot) as a test f resulting record to pass validation. - When instantiating from a factory, don't supply attributes that aren't required by the test. -- Prefer [implicit](https://github.com/thoughtbot/factory_bot/blob/master/GETTING_STARTED.md#implicit-definition), +- Use [implicit](https://github.com/thoughtbot/factory_bot/blob/master/GETTING_STARTED.md#implicit-definition), [explicit](https://github.com/thoughtbot/factory_bot/blob/master/GETTING_STARTED.md#explicit-definition), or [inline](https://github.com/thoughtbot/factory_bot/blob/master/GETTING_STARTED.md#inline-definition) associations - over `create` / `build` for association setup in callbacks. + instead of `create` / `build` for association setup in callbacks. See [issue #262624](https://gitlab.com/gitlab-org/gitlab/-/issues/262624) for further context. + + When creating factories with a [`has_many`](https://github.com/thoughtbot/factory_bot/blob/master/GETTING_STARTED.md#has_many-associations) and `belongs_to` association, use the `instance` method to refer to the object being built. + This prevents [creation of unnecessary records](https://gitlab.com/gitlab-org/gitlab/-/issues/378183) by using [interconnected associations](https://github.com/thoughtbot/factory_bot/blob/master/GETTING_STARTED.md#interconnected-associations). + + For example, if we have the following classes: + + ```ruby + class Car < ApplicationRecord + has_many :wheels, inverse_of: :car, foreign_key: :car_id + end + + class Wheel < ApplicationRecord + belongs_to :car, foreign_key: :car_id, inverse_of: :wheel, optional: false + end + ``` + + We can create the following factories: + + ```ruby + FactoryBot.define do + factory :car do + transient do + wheels_count { 2 } + end + + wheels do + Array.new(wheels_count) do + association(:wheel, car: instance) + end + end + end + end + + FactoryBot.define do + factory :wheel do + car { association :car } + end + end + ``` + - Factories don't have to be limited to `ActiveRecord` objects. [See example](https://gitlab.com/gitlab-org/gitlab-foss/commit/0b8cefd3b2385a21cfed779bd659978c0402766d). - Factories and their traits should produce valid objects that are [verified by specs](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/models/factories_spec.rb). diff --git a/doc/development/testing_guide/contract/index.md b/doc/development/testing_guide/contract/index.md index 8412a260c7d..08a21e58a52 100644 --- a/doc/development/testing_guide/contract/index.md +++ b/doc/development/testing_guide/contract/index.md @@ -26,6 +26,8 @@ The contracts themselves are stored in [`/spec/contracts/contracts`](https://git Before running the consumer tests, go to `spec/contracts/consumer` and run `npm install`. To run all the consumer tests, you just need to run `npm test -- /specs`. Otherwise, to run a specific spec file, replace `/specs` with the specific spec filename. +You can also run tests from the root directory of the project, using the command `yarn jest:contract`. + ### Run the provider tests Before running the provider tests, make sure your GDK (GitLab Development Kit) is fully set up and running. You can follow the setup instructions detailed in the [GDK repository](https://gitlab.com/gitlab-org/gitlab-development-kit/-/tree/main). To run the provider tests, you use Rake tasks that can be found in [`./lib/tasks/contracts`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/tasks/contracts). To get a list of all the Rake tasks related to the provider tests, run `bundle exec rake -T contracts`. For example: diff --git a/doc/development/testing_guide/end_to_end/feature_flags.md b/doc/development/testing_guide/end_to_end/feature_flags.md index 6d826e170f6..e473b158087 100644 --- a/doc/development/testing_guide/end_to_end/feature_flags.md +++ b/doc/development/testing_guide/end_to_end/feature_flags.md @@ -30,7 +30,7 @@ feature flag is under test. - Format: `feature_flag: { name: 'feature_flag_name', scope: :project }` - When `scope` is set to `:global`, the test will be **skipped on all live .com environments**. This is to avoid issues with feature flag changes affecting other tests or users on that environment. -- When `scope` is set to any other value (such as `:project`, `:group` or `:user`), or if no `scope` is specified, the test will only be **skipped on canary, production, and preprod**. +- When `scope` is set to any other value (such as `:project`, `:group` or `:user`), or if no `scope` is specified, the test will only be **skipped on canary, production, and pre-production**. This is due to the fact that administrator access is not available there. **WARNING:** You are strongly advised to first try and [enable feature flags only for a group, project, user](../../feature_flags/index.md#feature-actors), @@ -42,7 +42,7 @@ or [feature group](../../feature_flags/index.md#feature-groups). with administrator access, such as staging. **Note on `requires_admin`:** This tag should still be applied if there are other actions within the test that require administrator access that are unrelated to updating a -feature flag (ex: creating a user via the API). +feature flag (like creating a user via the API). The code below would enable a feature flag named `:feature_flag_name` for the project created by the test: diff --git a/doc/development/testing_guide/end_to_end/index.md b/doc/development/testing_guide/end_to_end/index.md index 8ffe044c4d8..55d725ba4ae 100644 --- a/doc/development/testing_guide/end_to_end/index.md +++ b/doc/development/testing_guide/end_to_end/index.md @@ -238,7 +238,7 @@ Each type of scheduled pipeline generates a static link for the latest test repo - [`production`](https://storage.googleapis.com/gitlab-qa-allure-reports/production-full/master/index.html) - [`production-sanity`](https://storage.googleapis.com/gitlab-qa-allure-reports/production-sanity/master/index.html) -## How do I run the tests? +## How do you run the tests? If you are not [testing code in a merge request](#testing-code-in-merge-requests), there are two main options for running the tests. If you want to run @@ -255,12 +255,12 @@ and the section below. Learn how to perform [tests that require special setup or consideration to run on your local environment](running_tests_that_require_special_setup.md). -## How do I write tests? +## How do you write tests? In order to write new tests, you first need to learn more about GitLab QA architecture. See the [documentation about it](https://gitlab.com/gitlab-org/gitlab-qa/blob/master/docs/architecture.md). -Once you decided where to put [test environment orchestration scenarios](https://gitlab.com/gitlab-org/gitlab-qa/tree/master/lib/gitlab/qa/scenario) and +After you've decided where to put [test environment orchestration scenarios](https://gitlab.com/gitlab-org/gitlab-qa/tree/master/lib/gitlab/qa/scenario) and [instance-level scenarios](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/qa/qa/specs/features), take a look at the [GitLab QA README](https://gitlab.com/gitlab-org/gitlab/-/tree/master/qa/README.md), the [GitLab QA orchestrator README](https://gitlab.com/gitlab-org/gitlab-qa/tree/master/README.md), and [the already existing instance-level scenarios](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/qa/qa/specs/features). @@ -283,7 +283,7 @@ Continued reading: - [Execution context selection](execution_context_selection.md) - [Troubleshooting](troubleshooting.md) -## Where can I ask for help? +## Where can you ask for help? You can ask question in the `#quality` channel on Slack (GitLab internal) or you can find an issue you would like to work on in diff --git a/doc/development/testing_guide/end_to_end/page_objects.md b/doc/development/testing_guide/end_to_end/page_objects.md index 5fbf2ffbcde..6a599ce9a50 100644 --- a/doc/development/testing_guide/end_to_end/page_objects.md +++ b/doc/development/testing_guide/end_to_end/page_objects.md @@ -155,7 +155,7 @@ In our case, `data-qa-selector="login_field"`, `data-qa-selector="password_field Things to note: -- The name of the element and the `qa_selector` must match and be snake_cased +- The name of the element and the `qa_selector` must match and be snake cased - If the element appears on the page unconditionally, add `required: true` to the element. See [Dynamic element validation](dynamic_element_validation.md) - You may see `.qa-selector` classes in existing Page Objects. We should prefer the [`data-qa-selector`](#data-qa-selector-vs-qa-selector) diff --git a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md index a7d1ece77b2..c1389b3ac0e 100644 --- a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md +++ b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md @@ -15,7 +15,8 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec |-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `:elasticsearch` | The test requires an Elasticsearch service. It is used by the [instance-level scenario](https://gitlab.com/gitlab-org/gitlab-qa#definitions) [`Test::Integration::Elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/72b62b51bdf513e2936301cb6c7c91ec27c35b4d/qa/qa/ee/scenario/test/integration/elasticsearch.rb) to include only tests that require Elasticsearch. | | `:except` | The test is to be run in their typical execution contexts _except_ as specified. See [test execution context selection](execution_context_selection.md) for more information. | -| `:feature_flag` | The test uses a feature flag and therefore requires an administrator account to run. When `scope` is set to `:global`, the test will be skipped on all live .com environments. Otherwise, it will be skipped only on Canary, Production, and Preprod. See [testing with feature flags](../../../development/testing_guide/end_to_end/feature_flags.md) for more details. | +| `:feature_flag` | The test uses a feature flag and therefore requires an administrator account to run. When `scope` is set to `:global`, the test will be skipped on all live .com environments. Otherwise, it will be skipped only on Canary, Production, and Pre-production. See [testing with feature flags](../../../development/testing_guide/end_to_end/feature_flags.md) for more details. | +| `:framework` | The test makes sanity assertions around the QA framework itself | | `:geo` | The test requires two GitLab Geo instances - a primary and a secondary - to be spun up. | | `:gitaly_cluster` | The test runs against a GitLab instance where repositories are stored on redundant Gitaly nodes behind a Praefect node. All nodes are [separate containers](../../../administration/gitaly/praefect.md#requirements). Tests that use this tag have a longer setup time since there are three additional containers that need to be started. | | `:github` | The test requires a GitHub personal access token. | diff --git a/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md index 81e1c7d5dc0..4a947e59d5f 100644 --- a/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md +++ b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md @@ -15,8 +15,8 @@ The project also has instructions for forking and building the images automatica Some extra environment variables for the location of the forked repository are also needed. -- `QA_THIRD_PARTY_DOCKER_REGISTRY` (the container registry where the repository/images are hosted, eg `registry.gitlab.com`) -- `QA_THIRD_PARTY_DOCKER_REPOSITORY` (the base repository path where the images are hosted, eg `registry.gitlab.com/<project path>`) +- `QA_THIRD_PARTY_DOCKER_REGISTRY` (the container registry where the repository/images are hosted, for example `registry.gitlab.com`) +- `QA_THIRD_PARTY_DOCKER_REPOSITORY` (the base repository path where the images are hosted, for example `registry.gitlab.com/<project path>`) - `QA_THIRD_PARTY_DOCKER_USER` (a username that has access to the container registry for this repository) - `QA_THIRD_PARTY_DOCKER_PASSWORD` (a password/token for the username to authenticate with) diff --git a/doc/development/testing_guide/end_to_end/troubleshooting.md b/doc/development/testing_guide/end_to_end/troubleshooting.md index e0925cb71f4..b4c47b90f0b 100644 --- a/doc/development/testing_guide/end_to_end/troubleshooting.md +++ b/doc/development/testing_guide/end_to_end/troubleshooting.md @@ -59,7 +59,7 @@ Net::ReadTimeout: Net::ReadTimeout with #<TCPSocket:(closed)> ``` This error can happen if GitLab runs on an address that does not resolve from -`localhost`. For example, if you set GDK's `hostname` +`localhost`. For example, if you set the GDK `hostname` [to a specific local IP address](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/docs/run_qa_against_gdk.md#run-qa-tests-against-your-gdk-setup), you must use that IP address instead of `localhost` in the command. For example, if your IP is `192.168.0.12`: diff --git a/doc/development/testing_guide/flaky_tests.md b/doc/development/testing_guide/flaky_tests.md index cc62a0ebf03..ef5b75d166f 100644 --- a/doc/development/testing_guide/flaky_tests.md +++ b/doc/development/testing_guide/flaky_tests.md @@ -31,7 +31,7 @@ it's reset to a pristine test after each test. inconsistent state, so that following tests might not know about certain columns. - [Example 2](https://gitlab.com/gitlab-org/gitlab/-/issues/368500): A test modifies data that is used by a following test. -- [Example 3](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/103434#note_1172316521): A test for a database query passes in a fresh database, but in a +- [Example 3](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/103434#note_1172316521): A test for a database query passes in a fresh database, but in a CI/CD pipeline where the database is used to process previous test sequences, the test fails. This likely means that the query itself needs to be updated to work in a non-clean database. @@ -56,6 +56,7 @@ the problem. - [Example 1](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10148/diffs): Without specifying `ORDER BY`, database will not give deterministic ordering, or data race happening in the tests. +- [Example 2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106936/diffs). ### Dataset-specific @@ -75,7 +76,7 @@ difficult to achieve locally. any table has more than 500 columns. It could pass in the merge request, but fail later in `master` if the order of tests changes. - [Example 2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91016/diffs): A test asserts - that trying to find a record with an unexisting ID retuns an error message. The test uses an + that trying to find a record with an nonexistent ID returns an error message. The test uses an hardcoded ID that's supposed to not exist (e.g. `42`). If the test is run early in the test suite, it might pass as not enough records were created before it, but as soon as it would run later in the suite, there could be a record that actually has the ID `42`, hence the test would @@ -104,6 +105,7 @@ or the app. **Description:** The DOM selector used in the test is unreliable. **Difficulty to reproduce:** Moderate to difficult. Depending on whether the DOM selector is duplicated, or appears after a delay etc. +Adding a delay in API or controller could help reproducing the issue. **Resolution:** It really depends on the problem here. It could be to wait for requests to finish, to scroll down the page etc. @@ -207,10 +209,10 @@ The `rspec/flaky/report-suite.json` report is: - [Sporadic RSpec failures due to `PG::UniqueViolation`](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/28307#note_24958837): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9846> - Follow-up: <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10688> - [Capybara.reset_session! should be called before requests are blocked](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/33779): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12224> -- FFaker generates funky data that tests are not ready to handle (and tests should be predictable so that's bad!): +- ffaker generates funky data that tests are not ready to handle (and tests should be predictable so that's bad!): - [Make `spec/mailers/notify_spec.rb` more robust](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/20121): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10015> - [Transient failure in `spec/requests/api/commits_spec.rb`](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/27988#note_25342521): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9944> - - [Replace FFaker factory data with sequences](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/29643): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10184> + - [Replace ffaker factory data with sequences](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/29643): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10184> - [Transient failure in spec/finders/issues_finder_spec.rb](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/30211#note_26707685): <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10404> ### Order-dependent flaky tests diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md index 041b0f0a4f4..2fa5fdeab7d 100644 --- a/doc/development/testing_guide/frontend_testing.md +++ b/doc/development/testing_guide/frontend_testing.md @@ -90,7 +90,7 @@ function getFahrenheit(celsius) { } ``` -It does not make sense to test our `getFahrenheit` function because underneath it does nothing else but invoking the library function, and we can expect that one is working as intended. (Simplified, I know) +It does not make sense to test our `getFahrenheit` function because underneath it does nothing else but invoking the library function, and we can expect that one is working as intended. Let's take a short look into Vue land. Vue is a critical part of the GitLab JavaScript codebase. When writing specs for Vue components, a common gotcha is to actually end up testing Vue provided functionality, because it appears to be the easiest thing to test. Here's an example taken from our codebase. @@ -522,7 +522,7 @@ it('waits for an event', () => { ### Ensuring that tests are isolated -Tests are normally architected in a pattern which requires a recurring setup and breakdown of the component under test. This is done by making use of the `beforeEach` and `afterEach` hooks. +Tests are normally architected in a pattern which requires a recurring setup of the component under test. This is often achieved by making use of the `beforeEach` hook. Example @@ -532,16 +532,22 @@ Example beforeEach(() => { wrapper = mount(Component); }); +``` + +With [enableAutoDestroy](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/100389), it is no longer necessary to manually call `wrapper.destroy()`. +However, some mocks, spies, and fixtures do need to be torn down, and we can leverage the `afterEach` hook. + +Example + +```javascript + let wrapper; afterEach(() => { - wrapper.destroy(); + fakeApollo = null; + store = null; }); ``` -When looking at this initially you'd suspect that the component is setup before each test and then broken down afterwards, providing isolation between tests. - -This is however not entirely true as the `destroy` method does not remove everything which has been mutated on the `wrapper` object. For functional components, destroy only removes the rendered DOM elements from the document. - ### Jest best practices > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34209) in GitLab 13.2. @@ -713,7 +719,7 @@ unit testing by mocking out modules that cannot be easily consumed in our test e > Instead, consider using [`jest.mock(..)`](https://jestjs.io/docs/jest-object#jestmockmodulename-factory-options) > (or a similar mocking function) in the relevant spec file. -#### Where should I put manual mocks? +#### Where should you put manual mocks? Jest supports [manual module mocks](https://jestjs.io/docs/manual-mocks) by placing a mock in a `__mocks__/` directory next to the source module (for example, `app/assets/javascripts/ide/__mocks__`). **Don't do this.** We want to keep all of our test-related code in one place (the `spec/` folder). @@ -1159,7 +1165,7 @@ By now you've probably heard of [Jest snapshot tests](https://jestjs.io/docs/sna To use them within GitLab, there are a few guidelines that should be highlighted: - Treat snapshots as code -- Don't think of a snapshot file as a Blackbox +- Don't think of a snapshot file as a black box - Care for the output of the snapshot, otherwise, it's not providing any real value. This will usually involve reading the generated snapshot file as you would read any other piece of code Think of a snapshot test as a simple way to store a raw `String` representation of what you've put into the item being tested. This can be used to evaluate changes in a component, a store, a complex piece of generated output, etc. You can see more in the list below for some recommended `Do's and Don'ts`. @@ -1169,7 +1175,7 @@ Jest provides a great set of docs on [best practices](https://jestjs.io/docs/sna ### How does a snapshot work? -A snapshot is purely a stringified version of what you ask to be tested on the lefthand side of the function call. This means any kind of changes you make to the formatting of the string has an impact on the outcome. This process is done by leveraging serializers for an automatic transform step. For Vue this is already taken care of by leveraging the `vue-jest` package, which offers the proper serializer. +A snapshot is purely a stringified version of what you ask to be tested on the left-hand side of the function call. This means any kind of changes you make to the formatting of the string has an impact on the outcome. This process is done by leveraging serializers for an automatic transform step. For Vue this is already taken care of by leveraging the `vue-jest` package, which offers the proper serializer. Should the outcome of your spec be different from what is in the generated snapshot file, you'll be notified about it by a failing test in your test suite. @@ -1448,13 +1454,10 @@ Before executing any page interaction when navigating or making asynchronous cal #### Elements interaction -There are a lot of different ways to find and interact with elements. For example, you could use the basic `find` method with the `selector` and `text` parameter and then use the `.click` method - -```ruby - find('.gl-tab-nav-item', text: 'Tests').click -``` +There are a lot of different ways to find and interact with elements. +For best practises, refer to the [UI testing](best_practices.md#ui-testing) section. -Alternatively, you could use `click_button` with a string of text that is found within the button, which is a more semantically meaningful way of clicking the element. +To click a button, use `click_button` with the string of text found in the button: ```ruby click_button 'Text inside the button element' @@ -1474,20 +1477,20 @@ You can use `fill_in` to fill input / form elements. The first argument is the s Alternatively, you can use the `find` selector paired with `send_keys` to add keys in a field without removing previous text, or `set` which completely replaces the value of the input element. -All of these are valid selectors and methods. Pick whichever suits your needs and look around as there are many more useful ones! +You can find a more comprehensive list of actions in the [feature tests actions](best_practices.md#actions) documentation. #### Assertions To assert anything in a page, you can always access `page` variable, which is automatically defines and actually means the page document. This means you can expect the `page` to have certain components like selectors or content. Here are a few examples: ```ruby - # Finding an element by ID - expect(page).to have_selector('#js-pipeline-graph') + # Finding a button + expect(page).to have_button('Submit review') ``` ```ruby # Finding by text - expect(page).to have_content('build') + expect(page).to have_text('build') ``` ```ruby @@ -1496,20 +1499,20 @@ To assert anything in a page, you can always access `page` variable, which is au ``` ```ruby - # Finding by CSS selector. This is a last resort. - # For example, when you cannot add attributes on the desired element. - expect(page).to have_css('.js-icon-retry') + # Find by data-testid + # Like CSS selector, this is acceptable when there isn't a specific matcher available. + expect(page).to have_css('[data-testid="pipeline-multi-actions-dropdown"]') ``` ```ruby - # Find by data-testid - # Like CSS selector, this is acceptable when there isn't a specific matcher available. - expect(page).to have_selector('[data-testid="pipeline-multi-actions-dropdown"]') + # Finding by CSS selector. This is a last resort. + # For example, when you cannot add attributes on the desired element. + expect(page).to have_css('.js-icon-retry') ``` ```ruby # You can combine any of these selectors with `not_to` instead - expect(page).not_to have_selector('#js-pipeline-graph') + expect(page).not_to have_button('Submit review') ``` ```ruby @@ -1529,11 +1532,13 @@ You can also create a sub-block to look into, to: - Make sure an element is found within the right boundaries. ```ruby - page.within('#js-pipeline-graph') do + page.within('[data-testid="pipeline-multi-actions-dropdown"]') do ... end ``` +You can find a more comprehensive list of matchers in the [feature tests matchers](best_practices.md#matchers) documentation. + #### Feature flags By default, every feature flag is enabled **regardless of the YAML definition or the flags you've set manually in your GDK**. To test when a feature flag is disabled, you must manually stub the flag, ideally in a `before do` block. diff --git a/doc/development/testing_guide/img/testing_triangle.png b/doc/development/testing_guide/img/testing_triangle.png Binary files differindex 7a9a848c2ee..3ac4955eaff 100644 --- a/doc/development/testing_guide/img/testing_triangle.png +++ b/doc/development/testing_guide/img/testing_triangle.png diff --git a/doc/development/understanding_explain_plans.md b/doc/development/understanding_explain_plans.md deleted file mode 100644 index 72c3df11a96..00000000000 --- a/doc/development/understanding_explain_plans.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/understanding_explain_plans.md' -remove_date: '2022-11-04' ---- - -This document was moved to [another location](database/understanding_explain_plans.md). - -<!-- This redirect file can be deleted after <2022-11-04>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/uploads/index.md b/doc/development/uploads/index.md index 41ec71451fb..b5509f5934e 100644 --- a/doc/development/uploads/index.md +++ b/doc/development/uploads/index.md @@ -40,21 +40,9 @@ When using object storage, administrators can control how those files are moved This move can happen in one of these ways: - [Rails controller upload](#rails-controller-upload). -- [Background upload](#background-upload). - [Direct upload](#direct-upload). -These strategies activate as per the following `<feature>.object_store.*` settings: - -| | `background_upload` = `false` | `background_upload` = `true` | -| ------------------------- | ----------------------------- | ------------------------------- | -| `direct_upload` = `false` | Controller upload | Background upload | -| `direct_upload` = `true` | Direct upload | Direct upload (takes precedence)| - Individual Sidekiq workers might also store files in object storage, which is not something we cover here. -More importantly, `background_upload` does not imply _all files are uploaded by Sidekiq._ -Sidekiq workers that store files in object storage could still exist when this setting is `false`. -Those cases are never user-initiated uploads, but they might occur in response to another user-initiated -action, such as exporting a GitLab repository. Finally, Workhorse assists most user-initiated uploads using an upload buffering mechanism to keep slow work out of Rails controllers. This mechanism is explained in [Workhorse assisted uploads](#workhorse-assisted-uploads), @@ -98,12 +86,12 @@ GitLab to the object store provider. As mentioned above, there are three differe this HTTP request is sent. - [Rails controller upload](#rails-controller-upload). -- [Background upload](#background-upload). - [Direct upload](#direct-upload). +- [Workhorse assisted uploads](#workhorse-assisted-uploads). ### Rails controller upload -When neither background upload nor direct upload are available, Rails uploads the file to object storage +When direct upload is not available, Rails uploads the file to object storage as part of the controller `create` action. Which controller is responsible depends on the kind of file uploaded. A Rails controller upload is very similar to uploading to local storage. The main difference: Rails must @@ -115,25 +103,6 @@ keep some of the costly I/O work out of Ruby and Rails. Direct upload does a bet This strategy is only suitable for small file uploads, as it is subject to Puma's 60 second request timeout. -### Background upload - -WARNING: -This strategy is deprecated in GitLab 14.9 and later, and is scheduled to [be removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/26600). - -With background uploads enabled: - -1. Files are uploaded as if they were to reside in local storage. -1. When Rails saves the upload metadata and the transaction completes, a Sidekiq job is scheduled. -1. The Sidekiq job transfers the file to the object store bucket. - - If the job completes, the upload record is updated to reflect the file's new location. - - If the job fails or gets lost, the upload stays in local storage and has the lifecycle of a normal local storage upload. - -As Rails and Sidekiq must cooperate to move the file to its final destination, it requires shared -storage and as such is unsuitable for CNG installations. We do not use background upload in GitLab SaaS. - -As background upload is an extension of local storage, it benefits from the same [Workhorse assistance](#workhorse-assisted-uploads) to -keep costly I/O work out of Ruby and Rails. - ### Direct upload Direct upload is the recommended way to move large files into object storage in CNG installations like GitLab SaaS. diff --git a/doc/development/uploads/working_with_uploads.md b/doc/development/uploads/working_with_uploads.md index c88762e6bd5..a3951fb4c7e 100644 --- a/doc/development/uploads/working_with_uploads.md +++ b/doc/development/uploads/working_with_uploads.md @@ -8,9 +8,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Recommendations -- When creating an uploader, [make it a subclass](#where-should-i-store-my-files) of `AttachmentUploader` +- When creating an uploader, [make it a subclass](#where-should-you-store-your-files) of `AttachmentUploader` - Add your uploader to the [tables](#tables) in this document -- Do not add [new object storage buckets](#where-should-i-store-my-files) +- Do not add [new object storage buckets](#where-should-you-store-your-files) - Implement [direct upload](#implementing-direct-upload-support) - If you need to process your uploads, decide [where to do that](#processing-uploads) @@ -19,14 +19,14 @@ info: To determine the technical writer assigned to the Stage/Group associated w - [CarrierWave Uploaders](#carrierwave-uploaders) - [GitLab modifications to CarrierWave](#gitlab-modifications-to-carrierwave) -## Where should I store my files? +## Where should you store your files? CarrierWave Uploaders determine where files get stored. When you create a new Uploader class you are deciding where to store the files of your new feature. First of all, ask yourself if you need a new Uploader class. It is OK -to use the same Uploader class for different mountpoints or different +to use the same Uploader class for different mount points or different models. If you do want or need your own Uploader class then you should make it @@ -160,8 +160,8 @@ we modified it. The central concept of CarrierWave is the **Uploader** class. The Uploader defines where files get stored, and optionally contains validation and processing logic. To use an Uploader you must associate -it with a text column on an ActiveRecord model. This called "mounting" -and the column is called the "mountpoint". For example: +it with a text column on an ActiveRecord model. This is called "mounting" +and the column is called `mountpoint`. For example: ```ruby class Project < ApplicationRecord @@ -169,8 +169,8 @@ class Project < ApplicationRecord end ``` -Now if I upload an avatar called `tanuki.png` the idea is that in the -`projects.avatar` column for my project, CarrierWave stores the string +Now if you upload an avatar called `tanuki.png` the idea is that in the +`projects.avatar` column for your project, CarrierWave stores the string `tanuki.png`, and that the AttachmentUploader class contains the configuration data and directory schema. For example if the project ID is 123, the actual file may be in @@ -179,12 +179,12 @@ The directory `/var/opt/gitlab/gitlab-rails/uploads/-/system/project/avatar/123/` was chosen by the Uploader using among others configuration (`/var/opt/gitlab/gitlab-rails/uploads`), the model name (`project`), -the model ID (`123`) and the mountpoint (`avatar`). +the model ID (`123`) and the mount point (`avatar`). > The Uploader determines the individual storage directory of your -> upload. The mountpoint column in your model contains the filename. +> upload. The `mountpoint` column in your model contains the filename. -You never access the mountpoint column directly because CarrierWave +You never access the `mountpoint` column directly because CarrierWave defines a getter and setter on your model that operates on file handle objects. @@ -213,7 +213,7 @@ CarrierWave has 2 storage engines: |CarrierWave class|GitLab name|Description| |---|---|---| -|`CarrierWave::Storage::File`|`ObjectStorage::Store::LOCAL` |Local files, accessed through the Ruby stdlib| +|`CarrierWave::Storage::File`|`ObjectStorage::Store::LOCAL` |Local files, accessed through the Ruby `stdlib` | | `CarrierWave::Storage::Fog`|`ObjectStorage::Store::REMOTE`|Cloud files, accessed through the [Fog gem](https://github.com/fog/fog)| GitLab uses both of these engines, depending on configuration. @@ -227,8 +227,8 @@ storage engine file by file. An Uploader is associated with two storage areas: regular storage and cache storage. Each has its own storage engine. If you assign a file -to a mountpoint setter (`project.avatar = -File.open('/tmp/tanuki.png')`) you will copy/move the file to cache +to a mount point setter (`project.avatar = File.open('/tmp/tanuki.png')`) +you will copy/move the file to cache storage as a side effect via the `cache!` method. To persist the file you must somehow call the `store!` method. This either happens via [ActiveRecord callbacks](https://github.com/carrierwaveuploader/carrierwave/blob/v1.3.2/lib/carrierwave/orm/activerecord.rb#L55) diff --git a/doc/development/utilities.md b/doc/development/utilities.md index 551834670b3..58954101890 100644 --- a/doc/development/utilities.md +++ b/doc/development/utilities.md @@ -181,20 +181,6 @@ Refer to [`strong_memoize.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/maste include Gitlab::Utils::StrongMemoize def result - strong_memoize(:result) do - search - end - end - end - ``` - - Alternatively, use the `strong_memoize_attr` helper to memoize the method for you: - - ```ruby - class Find - include Gitlab::Utils::StrongMemoize - - def result search end strong_memoize_attr :result @@ -206,6 +192,26 @@ Refer to [`strong_memoize.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/maste end ``` + Using `strong_memoize_attr` on methods with parameters is not supported. + It does not work when combined with [`override`](#override) and might memoize wrong results. + + Use `strong_memoize_with` instead. + + ```ruby + # bad + def expensive_method(arg) + # ... + end + strong_memoize_attr :expensive_method + + # good + def expensive_method(arg) + strong_memoize_with(:expensive_method, arg) + # ... + end + end + ``` + There's also `strong_memoize_with` to help memoize methods that take arguments. This should be used for methods that have a low number of possible values as arguments or with consistent repeating arguments in a loop. diff --git a/doc/development/value_stream_analytics.md b/doc/development/value_stream_analytics.md index 0d321133705..2d5f33b5dae 100644 --- a/doc/development/value_stream_analytics.md +++ b/doc/development/value_stream_analytics.md @@ -71,7 +71,7 @@ which are always available to the end-users regardless of the subscription. ### Value streams Value streams are container objects for the stages. There can be multiple value streams per group -focusing on different aspects of the Dev Ops lifecycle. +focusing on different aspects of the DevOps lifecycle. ### Events @@ -89,7 +89,8 @@ They're responsible for defining a timestamp expression that is used in the calc #### Implementing an `Event` class -There are a few methods that are required to be implemented, the `StageEvent` base class describes them in great detail. The most important ones are: +You must implement a few methods, as described in the `StageEvent` base class. +The most important methods are: - `object_type` - `timestamp_projection` diff --git a/doc/development/value_stream_analytics/value_stream_analytics_aggregated_backend.md b/doc/development/value_stream_analytics/value_stream_analytics_aggregated_backend.md index a07998550bf..5bcadc6f39b 100644 --- a/doc/development/value_stream_analytics/value_stream_analytics_aggregated_backend.md +++ b/doc/development/value_stream_analytics/value_stream_analytics_aggregated_backend.md @@ -54,9 +54,9 @@ database with a minimal development effort. ### Example configuration - + -In this example, there are two independent value streams set up for two teams that are using +In this example, two independent value streams are set up for two teams that are using different development workflows within the `Test Group` (top-level namespace). The first value stream uses standard timestamp-based events for defining the stages. The second @@ -102,7 +102,7 @@ High-level overview for each top-level namespace with Premium or Ultimate licens 1. `INSERT` or `UPDATE` the data into the VSA database tables. The data loading is implemented within the [`Analytics::CycleAnalytics::DataLoaderService`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/services/analytics/cycle_analytics/data_loader_service.rb) -class. There are groups containing a lot of data, so to avoid overloading the primary database, +class. Some groups contain a lot of data, so to avoid overloading the primary database, the service performs operations in batches and enforces strict application limits: - Load records in batches. diff --git a/doc/development/verifying_database_capabilities.md b/doc/development/verifying_database_capabilities.md deleted file mode 100644 index 0217eb96e5a..00000000000 --- a/doc/development/verifying_database_capabilities.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'database/verifying_database_capabilities.md' -remove_date: '2022-11-06' ---- - -This document was moved to [another location](database/verifying_database_capabilities.md). - -<!-- This redirect file can be deleted after <2022-11-06>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/work_items.md b/doc/development/work_items.md index a417e1d1349..90e454bec85 100644 --- a/doc/development/work_items.md +++ b/doc/development/work_items.md @@ -54,7 +54,7 @@ To avoid confusion and ensure communication is efficient, we will use the follow | work item view | The new frontend view that renders work items of any type | | | | legacy issue view | The existing view used to render issues and incidents | | | | issue | The existing issue model | | | -| issuable | Any model currently using the issueable module (issues, epics and MRs) | _Incidents are an **issuable**_ | _Incidents are a **work item type**_ | +| issuable | Any model currently using the issuable module (issues, epics and MRs) | _Incidents are an **issuable**_ | _Incidents are a **work item type**_ | | widget | A UI element to present or allow interaction with specific work item data | | | Some terms have been used in the past but have since become confusing and are now discouraged. @@ -92,7 +92,7 @@ NOTE: At first, defining a WIT will only be possible at the root-level group, which would then be inherited by subgroups. We will investigate the possibility of defining new WITs at subgroup levels at a later iteration. -### Introducing work_item_types table +### Introducing `work_item_types` table For example, suppose there are three root-level groups with IDs: `11`, `12`, and `13`. Also, assume the following base types: `issue: 0`, `incident: 1`, `test_case: 2`. @@ -228,7 +228,7 @@ So, migrating epics to a work item type requires providing feature parity betwee The main missing features are: -- Get WIs to the group level. This is dependent on [Consolidate Groups and Projects](https://gitlab.com/gitlab-org/architecture/tasks/-/issues/7) +- Get work items to the group level. This is dependent on [Consolidate Groups and Projects](https://gitlab.com/gitlab-org/architecture/tasks/-/issues/7) initiative. - A hierarchy widget: the ability to structure work items into hierarchies. - Inherited date widget. diff --git a/doc/development/work_items_widgets.md b/doc/development/work_items_widgets.md index 89602d969e6..ba15a3f0163 100644 --- a/doc/development/work_items_widgets.md +++ b/doc/development/work_items_widgets.md @@ -80,7 +80,7 @@ mutation { ``` -### Widget's responsibility and structure +### Widget responsibility and structure A widget is responsible for displaying and updating a single attribute, such as title, description, or labels. Widgets must support any type of work item. diff --git a/doc/development/workhorse/configuration.md b/doc/development/workhorse/configuration.md index 82e44a6f995..9fc106b8f04 100644 --- a/doc/development/workhorse/configuration.md +++ b/doc/development/workhorse/configuration.md @@ -162,7 +162,7 @@ addr = "localhost:9229" max_version = "tls1.3" ``` -## Interaction of authBackend and authSocket +## Interaction of `authBackend` and `authSocket` The interaction between `authBackend` and `authSocket` can be confusing. If `authSocket` is set, it overrides the host portion of `authBackend`, but not @@ -170,7 +170,7 @@ the relative path. In table form: -| authBackend | authSocket | Workhorse connects to | Rails relative URL | +| `authBackend` | `authSocket` | Workhorse connects to | Rails relative URL | |--------------------------------|-------------------|-----------------------|--------------------| | unset | unset | `localhost:8080` | `/` | | `http://localhost:3000` | unset | `localhost:3000` | `/` | diff --git a/doc/gitlab-basics/start-using-git.md b/doc/gitlab-basics/start-using-git.md index 25ef094b2a7..61b06fb106a 100644 --- a/doc/gitlab-basics/start-using-git.md +++ b/doc/gitlab-basics/start-using-git.md @@ -172,7 +172,7 @@ add your namespace (username or group) to the path: Clone with HTTPS using a token if: - You want to use 2FA. -- You want to have a revokable set of credentials scoped to one or more repositories. +- You want to have a revocable set of credentials scoped to one or more repositories. You can use any of these tokens to authenticate when cloning over HTTPS: diff --git a/doc/index.md b/doc/index.md index c359ec7b639..88b8c653aae 100644 --- a/doc/index.md +++ b/doc/index.md @@ -20,23 +20,16 @@ description: 'Learn how to use and administer GitLab, the most scalable Git-base # GitLab Docs -Welcome to [GitLab](https://about.gitlab.com/) documentation. +Welcome to the GitLab documentation! -Here you can access the complete documentation for GitLab, the single application for the -[entire DevOps lifecycle](#the-entire-devops-lifecycle). - -## Overview - -No matter how you use GitLab, we have documentation for you. - -| Essential documentation | Essential documentation | +| | | |:------------------------|:------------------------| -| [**User documentation**](user/index.md)<br>Discover features and concepts for GitLab users. | [**Administrator documentation**](administration/index.md)<br/>Everything GitLab self-managed administrators need to know. | -| [**Contributing to GitLab**](#contributing-to-gitlab)<br/>At GitLab, everyone can contribute! | [**New to Git and GitLab?**](tutorials/index.md)<br/>We have the resources to get you started. | -| [**Build an integration with GitLab**](#build-an-integration-with-gitlab)<br/>Consult our integration documentation. | [**Coming to GitLab from another platform?**](#coming-to-gitlab-from-another-platform)<br/>Consult our guides. | -| [**Install GitLab**](https://about.gitlab.com/install/)<br/>Installation options for different platforms. | [**Customers**](subscriptions/index.md)<br/>Information for new and existing customers. | -| [**Update GitLab**](update/index.md)<br/>Update your GitLab self-managed instance to the latest version. | [**Reference Architectures**](administration/reference_architectures/index.md)<br/>GitLab reference architectures. | -| [**GitLab releases**](https://about.gitlab.com/releases/)<br/>What's new in GitLab. | | +| [**Use GitLab**](user/index.md)<br>Get started with GitLab features and functionality. | [**Administer GitLab**](administration/index.md)<br/>Administer a self-managed GitLab instance. | +| [**New to Git and GitLab?**](tutorials/index.md)<br/>Start learning about Git and GitLab. | [**Contribute to GitLab development**](#contributing-to-gitlab)<br/>Create new GitLab functionality and documentation. | +| [**Coming to GitLab from another platform?**](#coming-to-gitlab-from-another-platform)<br/>Learn how to move to GitLab. | [**Build an integration with GitLab**](#build-an-integration-with-gitlab)<br/>Integrate with Jira and other common applications. | +| [**Choose a subscription**](subscriptions/index.md)<br/>Determine which subscription tier makes sense for you. | [**Install GitLab**](https://about.gitlab.com/install/)<br/>Install GitLab on different platforms. | +| [**Reference architectures**](administration/reference_architectures/index.md)<br/>View recommended deployments at scale. | [**Update GitLab**](update/index.md)<br/>Update your GitLab self-managed instance to the latest version. | +| [**GitLab releases**](https://about.gitlab.com/releases/)<br/>See what's new in GitLab. | | ## Popular topics diff --git a/doc/install/aws/eks_clusters_aws.md b/doc/install/aws/eks_clusters_aws.md index 03f7cd19ed5..191d0f93382 100644 --- a/doc/install/aws/eks_clusters_aws.md +++ b/doc/install/aws/eks_clusters_aws.md @@ -33,7 +33,7 @@ Using `eksctl` enables the following when building an EKS Cluster: Read more about Amazon EKS architecture quick start guide: -- [Landing page](https://aws.amazon.com/quickstart/architecture/amazon-eks/) +- [Landing page](https://aws.amazon.com/solutions/implementations/amazon-eks/) - [Reference guide](https://aws-quickstart.github.io/quickstart-amazon-eks/) - [Reference guide deployment steps](https://aws-quickstart.github.io/quickstart-amazon-eks/#_deployment_steps) - [Reference guide parameter reference](https://aws-quickstart.github.io/quickstart-amazon-eks/#_parameter_reference) diff --git a/doc/install/aws/gitlab_hybrid_on_aws.md b/doc/install/aws/gitlab_hybrid_on_aws.md index 7ae4391dde0..ef7d4ac0f69 100644 --- a/doc/install/aws/gitlab_hybrid_on_aws.md +++ b/doc/install/aws/gitlab_hybrid_on_aws.md @@ -14,8 +14,8 @@ Amazon provides a managed Kubernetes service offering known as [Amazon Elastic K ## Tested AWS Bill of Materials by reference architecture size -| GitLab Cloud Native Hybrid Ref Arch | GitLab Baseline Perf Test Results Omnibus on Instances | AWS Bill of Materials (BOM) for CNH | AWS Build Performance Testing Results for [CNH](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/5K/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128_results.txt) | CNH Cost Estimate 3 AZs* | -| ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | +| GitLab Cloud Native Hybrid Ref Arch | GitLab Baseline Performance Test Results Omnibus on Instances | AWS Bill of Materials (BOM) for CNH | AWS Build Performance Testing Results for [CNH](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/5K/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128_results.txt) | CNH Cost Estimate 3 AZs* | +| ------------------------------------------------------------ | ------------------------------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | | [2K Omnibus](../../administration/reference_architectures/2k_users.md) | [2K Baseline](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/2k) | [2K Cloud Native Hybrid on EKS](#2k-cloud-native-hybrid-on-eks) | GPT Test Results | [1 YR Ec2 Compute Savings + 1 YR RDS & ElastiCache RIs](https://calculator.aws/#/estimate?id=544bcf1162beae6b8130ad257d081cdf9d4504e3)<br />(2 AZ Cost Estimate is in BOM Below) | | [3K](../../administration/reference_architectures/3k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) | [3k Baseline](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/3k) | [3K Cloud Native Hybrid on EKS](#3k-cloud-native-hybrid-on-eks) | [3K Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/3K/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216_results.txt)<br /><br />[3K Elastic Auto Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/3K/3k-QuickStart-AutoScale-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_194200/3k-QuickStart-AutoScale-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_194200_results.txt) | [1 YR Ec2 Compute Savings + 1 YR RDS & ElastiCache RIs](https://calculator.aws/#/estimate?id=f1294fec554e21be999711cddcdab9c5e7f83f14)<br />(2 AZ Cost Estimate is in BOM Below) | | [5K](../../administration/reference_architectures/5k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) | [5k Baseline](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/5k) | [5K Cloud Native Hybrid on EKS](#5k-cloud-native-hybrid-on-eks) | [5K Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/5K/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128_results.txt)<br /><br />[5K AutoScale from 25% GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/5K/5k-QuickStart-AutoScale-From-25Percent-ARM-RDS-Redis_v13-12-3-ee_2021-07-24_102717/5k-QuickStart-AutoScale-From-25Percent-ARM-RDS-Redis_v13-12-3-ee_2021-07-24_102717_results.txt) | [1 YR Ec2 Compute Savings + 1 YR RDS & ElastiCache RIs](https://calculator.aws/#/estimate?id=330ee43c5b14662db5df6e52b34898d181a09e16) | @@ -46,7 +46,7 @@ The Beta version deploys Aurora PostgreSQL, but the release version will deploy | | [AWS Quick Start for GitLab Cloud Native Hybrid on EKS](https://aws-quickstart.github.io/quickstart-eks-gitlab/) | [GitLab Environment Toolkit (GET)](https://gitlab.com/gitlab-org/gitlab-environment-toolkit) | | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | -| Overview and Vision | [AWS Quick Start](https://aws.amazon.com/quickstart/architecture/amazon-eks/) | [GitLab Environment Toolkit](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/README.md) | +| Overview and Vision | [AWS Quick Start](https://aws.amazon.com/solutions/implementations/amazon-eks/) | [GitLab Environment Toolkit](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/README.md) | | Licensing | [Open Source (Apache 2.0)](https://github.com/aws-quickstart/quickstart-eks-gitlab/blob/main/LICENSE.txt) | [GitLab Enterprise Edition license](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/LICENSE) ([GitLab Premium tier](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/README.md)) | | GitLab Support | [GitLab Beta Support](../../policy/alpha-beta-support.md#beta-features) | [GitLab GA Support](../../policy/alpha-beta-support.md#generally-available-ga) | | GitLab Reference Architecture Compliant | Yes | Yes | @@ -166,12 +166,12 @@ If EKS node autoscaling is employed, it is likely that your average loading will | Non-Kubernetes Compute | Ref Arch Raw Total | AWS BOM<br />(Directly Usable in AWS Quick Start) | Example Cost<br />US East, 3 AZ | Example Cost<br />US East, 2 AZ | | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------- | ------------------------------- | ------------------------------- | -| **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for perf. testing | | | +| **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for performance testing | | | | **PostgreSQL**<br />AWS Amazon RDS PostgreSQL Nodes Configuration (GPT tested) | 2vCPU, 7.5 GB<br />Tested with Graviton ARM | **db.r6g.large** x 3 nodes <br />(6vCPU, 48 GB) | 3 nodes x $0.26 = $0.78/hr | 3 nodes x $0.26 = $0.78/hr | | **Redis** | 1vCPU, 3.75GB<br />(across 12 nodes for Redis Cache, Redis Queues/Shared State, Sentinel Cache, Sentinel Queues/Shared State) | **cache.m6g.large** x 3 nodes<br />(6vCPU, 19GB) | 3 nodes x $0.15 = $0.45/hr | 2 nodes x $0.15 = $0.30/hr | | **<u>Gitaly Cluster</u>** [Details](gitlab_sre_for_aws.md#gitaly-sre-considerations) | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | | | | | Gitaly Instances (in ASG) | 12 vCPU, 45GB<br />(across 3 nodes) | **m5.xlarge** x 3 nodes<br />(48 vCPU, 180 GB) | $0.192 x 3 = $0.58/hr | $0.192 x 3 = $0.58/hr | -| | The GitLab Reference architecture for 2K is not Highly Available and therefore has a single Gitaly no Praefect. AWS Quick Starts MUST be HA, so it implements Prafect from the 3K Ref Architecture to meet that requirement | | | | +| | The GitLab Reference architecture for 2K is not Highly Available and therefore has a single Gitaly no Praefect. AWS Quick Starts MUST be HA, so it implements Praefect from the 3K Ref Architecture to meet that requirement | | | | | Praefect (Instances in ASG with load balancer) | 6 vCPU, 10 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | **c5.large** x 3 nodes<br />(6 vCPU, 12 GB) | $0.09 x 3 = $0.21/hr | $0.09 x 3 = $0.21/hr | | Praefect PostgreSQL(1) (AWS RDS) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | Not applicable; reuses GitLab PostgreSQL | $0 | $0 | | Internal Load Balancing Node | 2 vCPU, 1.8 GB | AWS ELB | $0.10/hr | $0.10/hr | @@ -221,7 +221,7 @@ If EKS node autoscaling is employed, it is likely that your average loading will | Non-Kubernetes Compute | Ref Arch Raw Total | AWS BOM<br />(Directly Usable in AWS Quick Start) | Example Cost<br />US East, 3 AZ | Example Cost<br />US East, 2 AZ | | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------- | ------------------------------- | ------------------------------------------------------------ | -| **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for perf. testing | | | +| **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for performance testing | | | | **PostgreSQL**<br />Amazon RDS PostgreSQL Nodes Configuration (GPT tested) | 18vCPU, 36 GB <br />(across 9 nodes for PostgreSQL, PgBouncer, Consul)<br />Tested with Graviton ARM | **db.r6g.xlarge** x 3 nodes <br />(12vCPU, 96 GB) | 3 nodes x $0.52 = $1.56/hr | 3 nodes x $0.52 = $1.56/hr | | **Redis** | 6vCPU, 18GB<br />(across 6 nodes for Redis Cache, Sentinel) | **cache.m6g.large** x 3 nodes<br />(6vCPU, 19GB) | 3 nodes x $0.15 = $0.45/hr | 2 nodes x $0.15 = $0.30/hr | | **<u>Gitaly Cluster</u>** [Details](gitlab_sre_for_aws.md#gitaly-sre-considerations) | | | | | @@ -275,7 +275,7 @@ If EKS node autoscaling is employed, it is likely that your average loading will | Non-Kubernetes Compute | Ref Arch Raw Total | AWS BOM<br />(Directly Usable in AWS Quick Start) | Example Cost<br />US East, 3 AZ | Example Cost<br />US East, 2 AZ | | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------- | ------------------------------- | ------------------------------------------------------------ | -| **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for perf. testing | | | +| **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for performance testing | | | | **PostgreSQL**<br />Amazon RDS PostgreSQL Nodes Configuration (GPT tested) | 21vCPU, 51 GB <br />(across 9 nodes for PostgreSQL, PgBouncer, Consul)<br />Tested with Graviton ARM | **db.r6g.2xlarge** x 3 nodes <br />(24vCPU, 192 GB) | 3 nodes x $1.04 = $3.12/hr | 3 nodes x $1.04 = $3.12/hr | | **Redis** | 9vCPU, 27GB<br />(across 6 nodes for Redis, Sentinel) | **cache.m6g.xlarge** x 3 nodes<br />(12vCPU, 39GB) | 3 nodes x $0.30 = $0.90/hr | 2 nodes x $0.30 = $0.60/hr | | **<u>Gitaly Cluster</u>** [Details](gitlab_sre_for_aws.md#gitaly-sre-considerations) | | | | | @@ -328,7 +328,7 @@ If EKS node autoscaling is employed, it is likely that your average loading will | Non-Kubernetes Compute | Ref Arch Raw Total | AWS BOM | Example Cost<br />US East, 3 AZ | Example Cost<br />US East, 2 AZ | | ------------------------------------------------------------ | ------------------------------ | ------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | -| **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for perf. testing | | | +| **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for performance testing | | | | **PostgreSQL**<br />Amazon RDS PostgreSQL Nodes Configuration (GPT tested) | 36vCPU, 102 GB <br />(across 9 nodes for PostgreSQL, PgBouncer, Consul) | **db.r6g.2xlarge** x 3 nodes <br />(24vCPU, 192 GB) | 3 nodes x $1.04 = $3.12/hr | 3 nodes x $1.04 = $3.12/hr | | **Redis** | 30vCPU, 114GB<br />(across 12 nodes for Redis Cache, Redis Queues/Shared State, Sentinel Cache, Sentinel Queues/Shared State) | **cache.m5.2xlarge** x 3 nodes<br />(24vCPU, 78GB) | 3 nodes x $0.62 = $1.86/hr | 2 nodes x $0.62 = $1.24/hr | | **<u>Gitaly Cluster</u>** [Details](gitlab_sre_for_aws.md#gitaly-sre-considerations) | | | | | @@ -381,7 +381,7 @@ If EKS node autoscaling is employed, it is likely that your average loading will | Non-Kubernetes Compute | Ref Arch Raw Total | AWS BOM | Example Cost<br />US East, 3 AZ | Example Cost<br />US East, 2 AZ | | ------------------------------------------------------------ | ------------------------------------------------------------ | --------------------------------------------------------- | ------------------------------- | ------------------------------------------------------------ | -| **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for perf. testing | | | +| **Bastion Host (Quick Start)** | 1 HA instance in ASG | **t2.micro** for prod, **m4.2xlarge** for performance testing | | | | **PostgreSQL**<br />Amazon RDS PostgreSQL Nodes Configuration (GPT tested) | 96vCPU, 360 GB <br />(across 3 nodes) | **db.r6g.8xlarge** x 3 nodes <br />(96vCPU, 768 GB total) | 3 nodes x $4.15 = $12.45/hr | 3 nodes x $4.15 = $12.45/hr | | **Redis** | 30vCPU, 114GB<br />(across 12 nodes for Redis Cache, Redis Queues/Shared State, Sentinel Cache, Sentinel Queues/Shared State) | **cache.m6g.2xlarge** x 3 nodes<br />(24vCPU, 78GB total) | 3 nodes x $0.60 = $1.80/hr | 2 nodes x $0.60 = $1.20/hr | | **<u>Gitaly Cluster</u>** [Details](gitlab_sre_for_aws.md#gitaly-sre-considerations) | | | | | diff --git a/doc/install/aws/gitlab_sre_for_aws.md b/doc/install/aws/gitlab_sre_for_aws.md index c55ce993cfc..e68aea00b36 100644 --- a/doc/install/aws/gitlab_sre_for_aws.md +++ b/doc/install/aws/gitlab_sre_for_aws.md @@ -56,7 +56,7 @@ All recommendations are for production configurations, including performance tes - Use only SSD storage and the [class of Elastic Block Store (EBS) storage](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) that suites your durability and speed requirements. - When not using provisioned EBS IO, EBS volume size determines the I/O level, so provisioning volumes that are much larger than needed can be the least expensive way to improve EBS IO. -- If Gitaly performance monitoring shows signs of disk stress then one of the provisioned IOPs levels can be chosen. EBS IOPs levels also have enhanced durability which may be appealing for some implementations aside from performance considerations. +- If Gitaly performance monitoring shows signs of disk stress then one of the provisioned IOPS levels can be chosen. EBS IOPS levels also have enhanced durability which may be appealing for some implementations aside from performance considerations. **To accommodate:** diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index 93a966b69fb..05db439ffcd 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -70,7 +70,7 @@ Because any given GitLab upgrade might involve data disk updates or database sch - [GitLab Enterprise Edition](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#Images:visibility=public-images;ownerAlias=782774275127;search=GitLab%20EE;sort=desc:name): If you want to unlock the enterprise features, a license is needed. - [GitLab Community Edition](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#Images:visibility=public-images;ownerAlias=782774275127;search=GitLab%20CE;sort=desc:name): The open source version of GitLab. - - [GitLab Premium or Ultimate Marketplace (Prelicensed)](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#Images:visibility=public-images;source=Marketplace;search=GitLab%20EE;sort=desc:name): 5 user license built into per-minute billing. + - [GitLab Premium or Ultimate Marketplace (pre-licensed)](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#Images:visibility=public-images;source=Marketplace;search=GitLab%20EE;sort=desc:name): 5 user license built into per-minute billing. 1. AMI IDs are unique per region, so after you've loaded one of the above, select the desired target region in the upper right of the console to see the appropriate AMIs. 1. After the console is loaded, you can add additional search criteria to narrow further. For instance, type `13.` to find only 13.x versions. @@ -104,7 +104,7 @@ Implementation patterns enable platform-specific terminology, best practice arch ### Platform as a Service (PaaS) specification and usage -Platform as a Service options are a huge portion of the value provided by Cloud Platforms as they simplify operational complexity and reduce the SRE and security skilling required to operate advanced, highly available technology services. Implementation patterns can be prequalified against the partner PaaS options. +Platform as a Service options are a huge portion of the value provided by Cloud Platforms as they simplify operational complexity and reduce the SRE and security skilling required to operate advanced, highly available technology services. Implementation patterns can be pre-qualified against the partner PaaS options. - Implementation patterns help implementers understand what PaaS options are known to work and how to choose between PaaS solutions when a single platform has more than one PaaS option for the same GitLab role. - For instance, where reference architectures do not have a specific recommendation on what technology is leveraged for GitLab outbound email services or what the sizing should be - a Reference Implementation may advise using a cloud providers Email as a Service (PaaS) and possibly even with specific settings. diff --git a/doc/install/docker.md b/doc/install/docker.md index 11525842c6e..2464f43dc23 100644 --- a/doc/install/docker.md +++ b/doc/install/docker.md @@ -263,7 +263,7 @@ Here's an example that deploys GitLab with four runners as a [stack](https://doc 1. Create a `root_password.txt` file: ```plaintext - MySuperSecretAndSecurePass0rd! + MySuperSecretAndSecurePassw0rd! ``` 1. Make sure you are in the same directory as `docker-compose.yml` and run: @@ -407,8 +407,8 @@ port `2289`: ``` NOTE: - The format for publishing ports is `hostPort:containerPort`. Read more in - Docker's documentation about + The format for publishing ports is `hostPort:containerPort`. Read more in the + Docker documentation about [exposing incoming ports](https://docs.docker.com/engine/reference/run/#/expose-incoming-ports). 1. Enter the running container: @@ -720,7 +720,7 @@ purpose. ### Docker containers exhausts space due to the `json-file` -Docker's [default logging driver is `json-file`](https://docs.docker.com/config/containers/logging/configure/#configure-the-default-logging-driver), which performs no log rotation by default. As a result of this lack of rotation, log files stored by the `json-file` driver can consume a significant amount of disk space for containers that generate a lot of output. This can lead to disk space exhaustion. To address this, use [`journald`](https://docs.docker.com/config/containers/logging/journald/) as the logging driver when available, or [another supported driver](https://docs.docker.com/config/containers/logging/configure/#supported-logging-drivers) with native rotation support. +Docker uses the [`json-file` default logging driver](https://docs.docker.com/config/containers/logging/configure/#configure-the-default-logging-driver), which performs no log rotation by default. As a result of this lack of rotation, log files stored by the `json-file` driver can consume a significant amount of disk space for containers that generate a lot of output. This can lead to disk space exhaustion. To address this, use [`journald`](https://docs.docker.com/config/containers/logging/journald/) as the logging driver when available, or [another supported driver](https://docs.docker.com/config/containers/logging/configure/#supported-logging-drivers) with native rotation support. ### Buffer overflow error when starting Docker diff --git a/doc/install/installation.md b/doc/install/installation.md index c00a959e037..1191b32181e 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -32,7 +32,7 @@ Because an installation from source is a lot of work and error prone we strongly One reason the Omnibus package is more reliable is its use of runit to restart any of the GitLab processes in case one crashes. On heavily used GitLab instances the memory usage of the Sidekiq background worker grows over time. -Omnibus packages solve this by [letting the Sidekiq terminate gracefully](../administration/operations/sidekiq_memory_killer.md) if it uses too much memory. +Omnibus packages solve this by [letting the Sidekiq terminate gracefully](../administration/sidekiq/sidekiq_memory_killer.md) if it uses too much memory. After this termination runit detects Sidekiq is not running and starts it. Because installations from source don't use runit for process supervision, Sidekiq can't be terminated and its memory usage grows over time. @@ -47,11 +47,11 @@ If the highest number stable branch is unclear, check the [GitLab blog](https:// ## Software requirements | Software | Minimum version | Notes | -| ------------------ | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| ------------------ | --------------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [Ruby](#2-ruby) | `2.7` | From GitLab 13.6, Ruby 2.7 is required. Ruby 3.0 is not supported yet (see [the relevant epic](https://gitlab.com/groups/gitlab-org/-/epics/5149) for the current status). You must use the standard MRI implementation of Ruby. We love [JRuby](https://www.jruby.org/) and [Rubinius](https://github.com/rubinius/rubinius#the-rubinius-language-platform), but GitLab needs several Gems that have native extensions. | | [Go](#3-go) | `1.18` | From GitLab 15.6, Go 1.18 or later is required. | | [Git](#git) | `2.37.x` | From GitLab 15.6, Git 2.37.x and later is required. It's highly recommended that you use the [Git version provided by Gitaly](#git). | -| [Node.js](#4-node) | `14.15.0` | GitLab uses [webpack](https://webpack.js.org/) to compile frontend assets. Node.js 16.x is recommended, as it's faster. You can check which version you're running with `node -v`. You must update it to a newer version if needed. | +| [Node.js](#4-node) | `16.15.0` | From GitLab 15.7, Node.js 16.15.0 or later is required. | ## GitLab directory structure @@ -263,7 +263,8 @@ GitLab requires the use of Node to compile JavaScript assets, and Yarn to manage JavaScript dependencies. The current minimum requirements for these are: -- `node` >= v14.15.0. (We recommend node 16.x as it is faster) +- `node` 16.x releases (v16.15.0 or later). + [Other LTS versions of Node.js](https://github.com/nodejs/release#release-schedule) might be able to build assets, but we only guarantee Node.js 16.x. - `yarn` = v1.22.x (Yarn 2 is not supported yet) In many distributions, @@ -616,6 +617,7 @@ Install the gems (if you want to use Kerberos for user authentication, omit ```shell sudo -u git -H bundle config set --local deployment 'true' sudo -u git -H bundle config set --local without 'development test mysql aws kerberos' +sudo -u git -H bundle config path /home/git/gitlab/vendor/bundle sudo -u git -H bundle install ``` diff --git a/doc/install/openshift_and_gitlab/index.md b/doc/install/openshift_and_gitlab/index.md index a620540369f..6086716be1c 100644 --- a/doc/install/openshift_and_gitlab/index.md +++ b/doc/install/openshift_and_gitlab/index.md @@ -29,11 +29,11 @@ The GitLab Operator does not include the GitLab Runner. To install and manage a ## Unsupported GitLab features -### Secure and Protect +### Secure -- License Compliance -- Code Quality scanning -- Cluster Image Scanning +- [License Compliance](../../user/compliance/license_compliance/index.md) +- [Code Quality scanning](../../ci/testing/code_quality.md) +- [Operational Container Scanning](../../user/clusters/agent/vulnerabilities.md) (Note: Pipeline [Container Scanning](../../user/application_security/container_scanning/index.md) is supported) ### Docker-in-Docker diff --git a/doc/install/requirements.md b/doc/install/requirements.md index f581a1c50f9..7fcb371c178 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -77,8 +77,7 @@ process, such as PostgreSQL, which can have disastrous consequences. PostgreSQL is the only supported database, which is bundled with the Omnibus GitLab package. You can also use an [external PostgreSQL database](https://docs.gitlab.com/omnibus/settings/database.html#using-a-non-packaged-postgresql-database-management-server). -Support for MySQL was removed in GitLab 12.1. Existing users using GitLab with -MySQL/MariaDB are advised to [migrate to PostgreSQL](../update/mysql_to_postgresql.md) before upgrading. +Support for MySQL was removed in [GitLab 12.1](../update/index.md#1210). ### PostgreSQL Requirements diff --git a/doc/integration/advanced_search/elasticsearch.md b/doc/integration/advanced_search/elasticsearch.md index 94493aa6958..a55a56b30d6 100644 --- a/doc/integration/advanced_search/elasticsearch.md +++ b/doc/integration/advanced_search/elasticsearch.md @@ -46,13 +46,7 @@ If you are using a compatible version and after connecting to OpenSearch, you ge Elasticsearch requires additional resources to those documented in the [GitLab system requirements](../../install/requirements.md). -Memory, CPU, and storage resource amounts vary depending on the amount of data you index into the Elasticsearch cluster. Heavily used Elasticsearch clusters may require more resources. According to -[Elasticsearch official guidelines](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory), -each node should have: - -- [Memory](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory): 8 GiB (minimum). -- [CPU](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_cpus): Modern processor with multiple cores. GitLab has minimal CPU requirements for Elasticsearch. Multiple cores provide extra concurrency, which is more beneficial than faster CPUs. -- [Storage](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_disks): Use SSD storage. The total storage size of all Elasticsearch nodes is about 50% of the total size of your Git repositories. It includes one primary and one replica. The [`estimate_cluster_size`](#gitlab-advanced-search-rake-tasks) Rake task ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221177) in GitLab 13.10) uses total repository size to estimate the Advanced Search storage requirements. +Memory, CPU, and storage resource amounts vary depending on the amount of data you index into the Elasticsearch cluster. Heavily used Elasticsearch clusters may require more resources. The [`estimate_cluster_size`](#gitlab-advanced-search-rake-tasks) Rake task ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221177) in GitLab 13.10) uses the total repository size to estimate the Advanced Search storage requirements. ## Install Elasticsearch @@ -239,6 +233,85 @@ Sidekiq performance. Return them to their default values if you see increased `s in your Sidekiq logs. For more information, see [issue 322147](https://gitlab.com/gitlab-org/gitlab/-/issues/322147). +### Access requirements for self-managed AWS OpenSearch Service using fine-grained access control + +To use the self-managed AWS OpenSearch Service with GitLab using fine-grained access control, try one of the +[recommended configurations](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html#fgac-recommendations). + +Configure your instance's domain access policies to allow `es:ESHttp*` actions. You can customize +the following example configuration to limit principals or resources. +See [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": [ + "*" + ] + }, + "Action": [ + "es:ESHttp*" + ], + "Resource": "domain-arn/*" + } + ] +} +``` + +#### Connecting with a master user in the internal database + +When using fine-grained access control with a user in the internal database, you should use HTTP basic +authentication to connect to OpenSearch. You can provide the master username and password as part of the +OpenSearch URL or in the **Username** and **Password** text boxes in the Advanced Search settings. See +[Tutorial: Internal user database and HTTP basic authentication](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac-walkthrough-basic.html) for details. + +#### Connecting with an IAM user + +When using fine-grained access control with IAM credentials, you can provide the credentials in the **AWS OpenSearch IAM credentials** section in the Advanced Search settings. + +#### Permissions for fine-grained access control + +The following permissions are required for Advanced Search. See [Creating roles](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html#fgac-roles) for details. + +```json +{ + "cluster_permissions": [ + "cluster_composite_ops", + "cluster_monitor" + ], + "index_permissions": [ + { + "index_patterns": [ + "gitlab*" + ], + "allowed_actions": [ + "data_access", + "manage_aliases", + "search", + "create_index", + "delete", + "manage" + ] + }, + { + "index_patterns": [ + "*" + ], + "allowed_actions": [ + "indices:admin/aliases/get", + "indices:monitor/stats" + ] + } + ] +} +``` + +The index pattern `*` requires a few permissions for Advanced Search to work. + ### Limit the number of namespaces and projects that can be indexed If you check checkbox `Limit the number of namespaces and projects that can be indexed` @@ -486,7 +559,7 @@ The following are some available Rake tasks: | Task | Description | |:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [`sudo gitlab-rake gitlab:elastic:info`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Outputs debugging information for the Advanced Search intergation. | +| [`sudo gitlab-rake gitlab:elastic:info`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Outputs debugging information for the Advanced Search integration. | | [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enables Elasticsearch indexing and run `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. | | [`sudo gitlab-rake gitlab:elastic:pause_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Pauses Elasticsearch indexing. Changes are still tracked. Useful for cluster/index migrations. | | [`sudo gitlab-rake gitlab:elastic:resume_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Resumes Elasticsearch indexing. | @@ -783,8 +856,8 @@ additional process dedicated to indexing a set of queues (or queue group). This ensure that indexing queues always have a dedicated worker, while the rest of the queues have another dedicated worker to avoid contention. -For this purpose, use the [queue selector](../../administration/sidekiq/extra_sidekiq_processes.md#queue-selector) -option that allows a more general selection of queue groups using a [worker matching query](../../administration/sidekiq/extra_sidekiq_routing.md#worker-matching-query). +For this purpose, use the [queue selectors](../../administration/sidekiq/processing_specific_job_classes.md#queue-selectors) +option that allows a more general selection of queue groups using a [worker matching query](../../administration/sidekiq/processing_specific_job_classes.md#worker-matching-query). To handle these two queue groups, we generally recommend one of the following two options. You can either: @@ -804,8 +877,8 @@ To create both an indexing and a non-indexing Sidekiq process in one node: ```ruby sidekiq['enable'] = true - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ + sidekiq['queue_selector'] = true + sidekiq['queue_groups'] = [ "feature_category=global_search", "feature_category!=global_search" ] diff --git a/doc/integration/advanced_search/elasticsearch_troubleshooting.md b/doc/integration/advanced_search/elasticsearch_troubleshooting.md index aa6613d6f1a..7e2edf10c90 100644 --- a/doc/integration/advanced_search/elasticsearch_troubleshooting.md +++ b/doc/integration/advanced_search/elasticsearch_troubleshooting.md @@ -96,7 +96,7 @@ Here are some common pitfalls and how to overcome them. a Lucene index. - **Replicas**: Failover mechanisms that duplicate indices. -## How can I verify that my GitLab instance is using Elasticsearch? +## How can you verify that your GitLab instance is using Elasticsearch? There are a couple of ways to achieve that: @@ -184,13 +184,13 @@ If reindexing the project shows: - Elasticsearch errors or doesn't present any errors at all, reach out to your Elasticsearch administrator to check the instance. -### I updated GitLab and now I can't find anything +### You updated GitLab and now you can't find anything We continuously make updates to our indexing strategies and aim to support newer versions of Elasticsearch. When indexing changes are made, it may be necessary for you to [reindex](elasticsearch.md#zero-downtime-reindexing) after updating GitLab. -### I indexed all the repositories but I can't get any hits for my search term in the UI +### You indexed all the repositories but you can't get any hits for your search term in the UI Make sure you [indexed all the database data](elasticsearch.md#enable-advanced-search). @@ -220,7 +220,7 @@ The above instructions are not to be used for scenarios that only index a [subse See [Elasticsearch Index Scopes](elasticsearch.md#advanced-search-index-scopes) for more information on searching for specific types of data. -### I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything +### You indexed all the repositories but then switched Elasticsearch servers and now you can't find anything You must re-run all the Rake tasks to reindex the database, repositories, and wikis. @@ -228,11 +228,11 @@ You must re-run all the Rake tasks to reindex the database, repositories, and wi The more data present in your GitLab instance, the longer the indexing process takes. -### There are some projects that weren't indexed, but I don't know which ones +### There are some projects that weren't indexed, but you don't know which ones You can run `sudo gitlab-rake gitlab:elastic:projects_not_indexed` to display projects that aren't indexed. -### No new data is added to the Elasticsearch index when I push code +### No new data is added to the Elasticsearch index when you push code NOTE: This was [fixed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35936) in GitLab 13.2 and the Rake task is not available for versions greater than that. @@ -248,7 +248,7 @@ sudo gitlab-rake gitlab:elastic:clear_locked_projects If `ElasticCommitIndexerWorker` Sidekiq workers are failing with this error during indexing, it usually means that Elasticsearch is unable to keep up with the concurrency of indexing request. To address change the following settings: - To decrease the indexing throughput you can decrease `Bulk request concurrency` (see [Advanced Search settings](elasticsearch.md#advanced-search-configuration)). This is set to `10` by default, but you change it to as low as 1 to reduce the number of concurrent indexing operations. -- If changing `Bulk request concurrency` didn't help, you can use the [queue selector](../../administration/sidekiq/extra_sidekiq_processes.md#queue-selector) option to [limit indexing jobs only to specific Sidekiq nodes](elasticsearch.md#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests. +- If changing `Bulk request concurrency` didn't help, you can use the [queue selector](../../administration/sidekiq/processing_specific_job_classes.md#queue-selectors) option to [limit indexing jobs only to specific Sidekiq nodes](elasticsearch.md#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests. ### Indexing is very slow or fails with `rejected execution of coordinating operation` messages @@ -447,24 +447,9 @@ however, searches only surface results that can be viewed by the user. Advanced Search honors all permission checks in the application by filtering out projects that a user does not have access to at search time. -## Access requirements for the self-managed AWS OpenSearch Service +### Role mapping when using fine-grained access control with AWS Elasticsearch or OpenSearch -To use the self-managed AWS OpenSearch Service with GitLab, configure your instance's domain access policies -to contain the actions below. -See [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details. - -```plaintext -es:ESHttpDelete -es:ESHttpGet -es:ESHttpHead -es:ESHttpPost -es:ESHttpPut -es:ESHttpPatch -``` - -## Role-mapping when using AWS Elasticsearch or AWS OpenSearch fine-grained access control - -When using fine-grained access control with an IAM role, you might encounter the following error: +When using fine-grained access control with an IAM role or a role created using OpenSearch Dashboards, you might encounter the following error: ```plaintext {"error":{"root_cause":[{"type":"security_exception","reason":"no permissions for [indices:data/write/bulk] and User [name=arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE, backend_roles=[arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE], requestedTenant=null]"}],"type":"security_exception","reason":"no permissions for [indices:data/write/bulk] and User [name=arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE, backend_roles=[arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE], requestedTenant=null]"},"status":403} diff --git a/doc/integration/arkose.md b/doc/integration/arkose.md index aa27e3ba4a4..09a7defcff8 100644 --- a/doc/integration/arkose.md +++ b/doc/integration/arkose.md @@ -35,8 +35,8 @@ improve their risk prediction model. NOTE: Enabling the `arkose_labs_prevent_login` feature flag results in sessions with a `High` risk -score being denied access. So far, we have kept this feature flag disabled to evaluate Arkose -Protect's predictions and to make sure we are not preventing legitimate users from signing in. +score being denied access. So far, we have kept this feature flag disabled to evaluate Arkose Protect +predictions and to make sure we are not preventing legitimate users from signing in. That said, we have seen that interactive challenges are effective in preventing some malicious sign-in attempts as not completing them prevents attackers from moving on to the next sign-in step. diff --git a/doc/integration/datadog.md b/doc/integration/datadog.md index 31e254658c1..1f20bccf083 100644 --- a/doc/integration/datadog.md +++ b/doc/integration/datadog.md @@ -38,9 +38,11 @@ project, group, or instance level: Used only in advanced scenarios. 1. Optional. If you use more than one GitLab instance, provide a unique **Service** name to differentiate between your GitLab instances. +<!-- vale gitlab.Spelling = NO --> 1. Optional. If you use groups of GitLab instances (such as staging and production environments), provide an **Env** name. This value is attached to each span the integration generates. +<!-- vale gitlab.Spelling = YES --> 1. Optional. To define any custom tags for all spans at which the integration is being configured, enter one tag per line in **Tags**. Each line must be in the format `key:value`. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79665) in GitLab 14.8.) 1. Optional. Select **Test settings** to test your integration. @@ -51,4 +53,4 @@ section of your Datadog account. ## Related topics -- [Datadog's CI Visibility](https://docs.datadoghq.com/continuous_integration/) documentation. +- [Datadog CI Visibility](https://docs.datadoghq.com/continuous_integration/) documentation. diff --git a/doc/integration/glab/img/glabgettingstarted.gif b/doc/integration/glab/img/glabgettingstarted.gif Binary files differnew file mode 100644 index 00000000000..cf335294e41 --- /dev/null +++ b/doc/integration/glab/img/glabgettingstarted.gif diff --git a/doc/integration/glab/index.md b/doc/integration/glab/index.md new file mode 100644 index 00000000000..3951f38dfab --- /dev/null +++ b/doc/integration/glab/index.md @@ -0,0 +1,80 @@ +--- +stage: Create +group: Code Review +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# GitLab CLI - `glab` + +GLab is an open source GitLab CLI tool. It brings GitLab to your terminal: +next to where you are already working with Git and your code, without +switching between windows and browser tabs. + +- Work with issues. +- Work with merge requests. +- Watch running pipelines directly from your CLI. + + + +The GitLab CLI uses commands structured like `glab <command> <subcommand> [flags]` +to perform many of the actions you normally do from the GitLab user interface: + +```shell +# Sign in +glab auth login --stdin < token.txt + +# View a list of issues +glab issue list + +# Create merge request for issue 123 +glab mr for 123 + +# Check out the branch for merge request 243 +glab mr checkout 243 + +# Watch the pipeline in progress +glab pipeline ci view + +# View, approve, and merge the merge request +glab mr view +glab mr approve +glab mr merge +``` + +## Core commands + +- `glab alias` +- `glab api` +- `glab auth` +- `glab ci` +- `glab issue` +- `glab label` +- `glab mr` +- `glab project` +- `glab release` +- `glab snippet` +- `glab ssh-key` +- `glab user` +- `glab variable` + +## Install the CLI + +Installation instructions are available in the GLab +[`README`](https://gitlab.com/gitlab-org/cli/#installation). + +## Authenticate with GitLab + +To authenticate with your GitLab account, run `glab auth login`. +`glab` respects tokens set using `GITLAB_TOKEN`. + +## Report issues + +Open an issue in the [`gitlab-org/cli` repository](https://gitlab.com/gitlab-org/cli/issues/new) +to send us feedback. + +## Related topics + +- [Install the CLI](https://gitlab.com/gitlab-org/cli/-/blob/main/README.md#installation) +- [Documentation](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source) +- The extension source code is available in the + [`cli`](https://gitlab.com/gitlab-org/cli/) project. diff --git a/doc/integration/index.md b/doc/integration/index.md index b2a4201e88c..bdf6475b6d2 100644 --- a/doc/integration/index.md +++ b/doc/integration/index.md @@ -11,7 +11,7 @@ You can integrate GitLab with external services for enhanced functionality. ## Services -Services such as Campfire, Flowdock, Jira, Pivotal Tracker, and Slack +Services such as Campfire, Jira, Pivotal Tracker, and Slack are available as [integrations](../user/project/integrations/index.md). ## Issue trackers diff --git a/doc/integration/jenkins.md b/doc/integration/jenkins.md index 8a438dde52e..53a2fb8bbdd 100644 --- a/doc/integration/jenkins.md +++ b/doc/integration/jenkins.md @@ -179,7 +179,7 @@ If you get this error message while configuring GitLab, the following are possib - The Jenkins instance is at a local address and is not included in the [GitLab installation's allowlist](../security/webhooks.md#create-an-allowlist-for-local-requests). - The credentials for the Jenkins instance do not have sufficient access or are invalid. -- The **Enable authentication for `/project` end-point** checkbox is not selected in your [Jenkin's plugin configuration](#configure-the-jenkins-server). +- The **Enable authentication for `/project` end-point** checkbox is not selected in your [Jenkins plugin configuration](#configure-the-jenkins-server). ### Error in merge requests - "Could not connect to the CI server" diff --git a/doc/integration/jira/connect-app.md b/doc/integration/jira/connect-app.md index 171c1cbe484..513877a7b71 100644 --- a/doc/integration/jira/connect-app.md +++ b/doc/integration/jira/connect-app.md @@ -72,6 +72,55 @@ for details. If the app requires additional permissions, [the update must first be manually approved in Jira](https://developer.atlassian.com/platform/marketplace/upgrading-and-versioning-cloud-apps/#changes-that-require-manual-customer-approval). +## Connect the GitLab.com for Jira Cloud app for self-managed instances **(FREE SELF)** + +> - Introduced in GitLab 15.6 [with flags](../../administration/feature_flags.md) named [`jira_connect_oauth_self_managed_setting`](https://gitlab.com/gitlab-org/gitlab/-/issues/377679), [`jira_connect_oauth`](https://gitlab.com/gitlab-org/gitlab/-/issues/355048), and [`jira_connect_oauth_self_managed`](https://gitlab.com/gitlab-org/gitlab/-/issues/359940). Disabled by default. +> - Feature flag `jira_connect_oauth_self_managed_setting` [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105070) in GitLab 15.7. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, +ask an administrator to [enable the feature flags](../../administration/feature_flags.md) named +`jira_connect_oauth` and `jira_connect_oauth_self_managed`. On GitLab.com, this feature +is not available. The feature is not ready for production use. + +Prerequisites: + +- GitLab.com must serve as a proxy for the instance. +- The instance must be publicly available. +- The instance must be on version 15.7 or later. + +You can link self-managed instances after installing the GitLab.com for Jira Cloud app from the marketplace. +Jira apps can only link to one URL per marketplace listing. The official listing links to GitLab.com. + +### Set up your instance + +To set up your self-managed instance for the GitLab.com for Jira Cloud app in GitLab 15.7 or later: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Applications** (`/admin/applications`). +1. Select **New application**. +1. In **Redirect URI**, enter `https://gitlab.com/-/jira_connect/oauth_callbacks`. +1. Ensure the **Trusted** and **Confidential** checkboxes are cleared. +<!-- markdownlint-disable MD044 --> +1. In **Scopes**, select the **api** checkbox only. +<!-- markdownlint-enable MD044 --> +1. Select **Save application**. +1. Copy the **Application ID** value. +1. On the left sidebar, select **Settings > General** (`/admin/application_settings/general`). +1. Expand the **GitLab for Jira App** section. +1. Paste the **Application ID** value into **Jira Connect Application ID**. +1. In **Jira Connect Proxy URL**, enter `https://gitlab.com`. +1. Select **Save changes**. + +### Link your instance + +To link your self-managed instance to the GitLab.com for Jira Cloud app: + +1. Install the [GitLab.com for Jira Cloud app](https://marketplace.atlassian.com/apps/1221011/gitlab-com-for-jira-cloud?tab=overview&hosting=cloud). +1. Select **GitLab (self-managed)**. +1. Enter your GitLab instance URL. +1. Select **Save**. + ## Install the GitLab.com for Jira Cloud app for self-managed instances **(FREE SELF)** If your GitLab instance is self-managed, you must follow some diff --git a/doc/integration/jira/dvcs.md b/doc/integration/jira/dvcs.md index f33536b7b91..982c8203904 100644 --- a/doc/integration/jira/dvcs.md +++ b/doc/integration/jira/dvcs.md @@ -19,6 +19,9 @@ are accessible. - **Jira Server**: Your network must allow access to your instance. - **Jira Cloud**: Your instance must be accessible through the internet. +NOTE: +When using GitLab 15.0 and later (including GitLab.com) with Jira Server, you might experience a [session token bug in Jira](https://jira.atlassian.com/browse/JSWSERVER-21389). As a workaround, ensure Jira Server is version 9.1.0 and later or 8.20.11 and later. + ## Smart Commits When connecting GitLab with Jira with DVCS, you can process your Jira issues using diff --git a/doc/integration/kerberos.md b/doc/integration/kerberos.md index c7cbc4389f5..a0441b79490 100644 --- a/doc/integration/kerberos.md +++ b/doc/integration/kerberos.md @@ -357,6 +357,38 @@ to a larger value in [the NGINX configuration](https://nginx.org/en/docs/http/ng ## Troubleshooting +### Test connectivity between the GitLab and Kerberos servers + +You can use utilities like [`kinit`](https://web.mit.edu/kerberos/krb5-1.12/doc/user/user_commands/kinit.html) and [`klist`](https://web.mit.edu/kerberos/krb5-1.12/doc/user/user_commands/klist.html) to test connectivity between the GitLab server +and the Kerberos server. How you install these depends on your specific OS. + +Use `klist` to see the service principal names (SPN) available in your `keytab` file and the encryption type for each SPN: + +```shell +klist -ke /etc/http.keytab +``` + +On an Ubuntu server, the output would look similar to the following: + +```shell +Keytab name: FILE:/etc/http.keytab +KVNO Principal +---- -------------------------------------------------------------------------- + 3 HTTP/my.gitlab.domain@MY.REALM (des-cbc-crc) + 3 HTTP/my.gitlab.domain@MY.REALM (des-cbc-md5) + 3 HTTP/my.gitlab.domain@MY.REALM (arcfour-hmac) + 3 HTTP/my.gitlab.domain@MY.REALM (aes256-cts-hmac-sha1-96) + 3 HTTP/my.gitlab.domain@MY.REALM (aes128-cts-hmac-sha1-96) +``` + +Use `kinit` in verbose mode to test whether GitLab can use the keytab file to connect to the Kerberos server: + +```shell +KRB5_TRACE=/dev/stdout kinit -kt /etc/http.keytab HTTP/my.gitlab.domain@MY.REALM +``` + +This command shows a detailed output of the authentication process. + ### Unsupported GSSAPI mechanism With Kerberos SPNEGO authentication, the browser is expected to send a list of diff --git a/doc/integration/mattermost/index.md b/doc/integration/mattermost/index.md index 04b0157b737..df6130a7540 100644 --- a/doc/integration/mattermost/index.md +++ b/doc/integration/mattermost/index.md @@ -123,7 +123,7 @@ http://mattermost.example.com/signup/gitlab/complete http://mattermost.example.com/login/gitlab/complete ``` -Note that you do not need to select any options under **Scopes**. Choose **Save application**. +Make sure to select the **Trusted** and **Confidential** settings. Under **Scopes**, select `read_user`. Then, choose **Save application**. Once the application is created you are provided with an `Application ID` and `Secret`. One other piece of information needed is the URL of GitLab instance. Return to the server running GitLab Mattermost and edit the `/etc/gitlab/gitlab.rb` configuration file as follows using the values you received above: @@ -132,7 +132,7 @@ Return to the server running GitLab Mattermost and edit the `/etc/gitlab/gitlab. mattermost['gitlab_enable'] = true mattermost['gitlab_id'] = "12345656" mattermost['gitlab_secret'] = "123456789" -mattermost['gitlab_scope'] = "" +mattermost['gitlab_scope'] = "read_user" mattermost['gitlab_auth_endpoint'] = "http://gitlab.example.com/oauth/authorize" mattermost['gitlab_token_endpoint'] = "http://gitlab.example.com/oauth/token" mattermost['gitlab_user_api_endpoint'] = "http://gitlab.example.com/api/v4/user" diff --git a/doc/integration/oauth2_generic.md b/doc/integration/oauth2_generic.md index a337873a67e..c51400113d4 100644 --- a/doc/integration/oauth2_generic.md +++ b/doc/integration/oauth2_generic.md @@ -6,8 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Generic OAuth 2.0 provider **(FREE SELF)** -The `omniauth-oauth2-generic` gem allows single sign-on (SSO) between GitLab -and your OAuth 2.0 provider, or any OAuth 2.0 provider compatible with this gem). +The [`omniauth-oauth2-generic` gem](https://gitlab.com/satorix/omniauth-oauth2-generic) allows single sign-on (SSO) between GitLab +and your OAuth 2.0 provider, or any OAuth 2.0 provider compatible with this gem. This strategy allows for the configuration of this OmniAuth SSO process: @@ -48,62 +48,149 @@ To configure the provider: appear is different for each provider. This may also be called application ID and application secret. -1. On your GitLab server, open the appropriate configuration file. - - For Omnibus GitLab: - - ```shell - sudo editor /etc/gitlab/gitlab.rb - ``` - - For installations from source: - - ```shell - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` - -1. See [Configure initial settings](omniauth.md#configure-initial-settings) for - initial settings. - -1. Add the provider-specific configuration for your provider. For example: - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - name: "oauth2_generic", - label: "Provider name", # optional label for login button, defaults to "Oauth2 Generic" - app_id: "<your_app_client_id>", - app_secret: "<your_app_client_secret>", - args: { - client_options: { - site: "<your_auth_server_url>", - user_info_url: "/oauth2/v1/userinfo", - authorize_url: "/oauth2/v1/authorize", - token_url: "/oauth2/v1/token" - }, - user_response_structure: { - root_path: [], - id_path: ["sub"], - attributes: { - email: "email", - name: "name" - } - }, - authorize_params: { - scope: "openid profile email" - }, - strategy_class: "OmniAuth::Strategies::OAuth2Generic" - } - } - ] - ``` - - For more information about these settings, see the [gem's README](https://gitlab.com/satorix/omniauth-oauth2-generic#gitlab-config-example). - -1. Save the configuration file. - -1. For the changes to take effect, [restart GitLab](../administration/restart_gitlab.md#installations-from-source). +1. On your GitLab server, complete the following steps. + + ::Tabs + + :::TabTitle Linux package (Omnibus) + + 1. [Configure the initial settings](omniauth.md#configure-initial-settings). + 1. Edit `/etc/gitlab/gitlab.rb` to add the configuration for your provider. For example: + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + name: "oauth2_generic", + label: "Provider name", # optional label for login button, defaults to "Oauth2 Generic" + app_id: "<your_app_client_id>", + app_secret: "<your_app_client_secret>", + args: { + client_options: { + site: "<your_auth_server_url>", + user_info_url: "/oauth2/v1/userinfo", + authorize_url: "/oauth2/v1/authorize", + token_url: "/oauth2/v1/token" + }, + user_response_structure: { + root_path: [], + id_path: ["sub"], + attributes: { + email: "email", + name: "name" + } + }, + authorize_params: { + scope: "openid profile email" + }, + strategy_class: "OmniAuth::Strategies::OAuth2Generic" + } + } + ] + ``` + + 1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + + :::TabTitle Helm chart (Kubernetes) + + 1. [Configure the initial settings](omniauth.md#configure-initial-settings). + 1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + + 1. Edit `gitlab_values.yaml`. + + NOTE: + The following example exposes the `app_secret` value in the main YAML file. + You're strongly advised to use + [Helm secrets](https://docs.gitlab.com/charts/installation/secrets.html) + instead. + + ```yaml + global: + appConfig: + omniauth: + enabled: true + providers: + - name: "oauth2_generic" + label: "Provider name" # optional label for login button defaults to "Oauth2 Generic" + app_id: "<your_app_client_id>" + app_secret: "<your_app_client_secret>" + args: + client_options: + site: "<your_auth_server_url>" + user_info_url: "/oauth2/v1/userinfo" + authorize_url: "/oauth2/v1/authorize" + token_url: "/oauth2/v1/token" + user_response_structure: + root_path: [] + id_path: ["sub"] + attributes: + email: "email" + name: "name" + authorize_params: + scope: "openid profile email" + strategy_class: "OmniAuth::Strategies::OAuth2Generic" + ``` + + 1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + + :::TabTitle Self-compiled (source) + + 1. [Configure the initial settings](omniauth.md#configure-initial-settings). + 1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: &base + omniauth: + providers: + - { name: "oauth2_generic", + label: "Provider name", # optional label for login button, defaults to "Oauth2 Generic" + app_id: "<your_app_client_id>", + app_secret: "<your_app_client_secret>", + args: { + client_options: { + site: "<your_auth_server_url>", + user_info_url: "/oauth2/v1/userinfo", + authorize_url: "/oauth2/v1/authorize", + token_url: "/oauth2/v1/token" + }, + user_response_structure: { + root_path: [], + id_path: ["sub"], + attributes: { + email: "email", + name: "name" + } + }, + authorize_params: { + scope: "openid profile email" + }, + strategy_class: "OmniAuth::Strategies::OAuth2Generic" + } + } + ``` + + 1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + + ::EndTabs On the sign-in page there should now be a new icon below the regular sign-in form. Select that icon to begin your provider's authentication process. This diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index af039c8a009..2dd8505b558 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -38,7 +38,7 @@ GitLab supports the following OmniAuth providers. | [SAML](saml.md) | `saml` | | [Twitter](twitter.md) | `twitter` | -## Configure initial settings +## Initial settings Before you configure the OmniAuth provider, configure the settings that are common for all providers. @@ -49,13 +49,15 @@ Omnibus, Docker, and source | Helm chart | Description | Default value `auto_link_ldap_user` | `autoLinkLdapUser` | Creates an LDAP identity in GitLab for users that are created through an OmniAuth provider. You can enable this setting if you have [LDAP integration](../administration/auth/ldap/index.md) enabled. Requires the `uid` of the user to be the same in both LDAP and the OmniAuth provider. | `false` `block_auto_created_users` | `blockAutoCreatedUsers` | Blocks users that are automatically created from signing in until they are approved by an administrator. | `true`. If you set the value to `false`, make sure you define providers that you can control, like SAML or Google. Otherwise, any user on the internet can sign in to GitLab without an administrator's approval. -To change these settings: +### Configure initial settings + +To change the OmniAuth settings: ::Tabs - :::TabTitle Omnibus + :::TabTitle Linux package (Omnibus) - 1. Edit `/etc/gitlab/gitlab.rb` and update the following section: + 1. Edit `/etc/gitlab/gitlab.rb`: ```ruby # CAUTION! @@ -67,13 +69,13 @@ To change these settings: gitlab_rails['omniauth_block_auto_created_users'] = true ``` - 1. Reconfigure GitLab: + 1. Save the file and reconfigure GitLab: ```shell sudo gitlab-ctl reconfigure ``` - :::TabTitle Helm chart + :::TabTitle Helm chart (Kubernetes) 1. Export the Helm values: @@ -96,22 +98,15 @@ To change these settings: For more details, see the [globals documentation](https://docs.gitlab.com/charts/charts/globals.html#omniauth). - 1. Apply the new values: + 1. Save the file and apply the new values: ```shell helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab ``` - :::TabTitle Source - - 1. Open the configuration file: - - ```shell - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + :::TabTitle Self-compiled (source) - 1. Update the following section: + 1. Edit `/home/git/gitlab/config/gitlab.yml`: ```yaml ## OmniAuth settings @@ -132,9 +127,13 @@ To change these settings: block_auto_created_users: true ``` - 1. Restart GitLab: + 1. Save the file and restart GitLab: ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init sudo service gitlab restart ``` @@ -283,7 +282,7 @@ for the OpenID Connect provider and the Twitter OAuth provider. This method of enabling automatic linking works for all providers [except SAML](https://gitlab.com/gitlab-org/gitlab/-/issues/338293). -To enable automatic linking for SAML, see the [SAML setup instructions](saml.md#general-setup). +To enable automatic linking for SAML, see the [SAML setup instructions](saml.md#configure-saml-support-in-gitlab). ## Create an external providers list diff --git a/doc/integration/saml.md b/doc/integration/saml.md index fd01e9e0e56..84879b7c4c7 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -5,41 +5,29 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# SAML OmniAuth Provider **(FREE SELF)** +# SAML SSO for self-managed GitLab instances **(FREE SELF)** -This page describes instance-wide SAML for self-managed GitLab instances. For -SAML on GitLab.com, see [SAML SSO for GitLab.com groups](../user/group/saml_sso/index.md). +This page describes how to set up instance-wide SAML single sign on (SSO) for +self-managed GitLab instances. -You should also reference the [OmniAuth documentation](omniauth.md) for general -settings that apply to all OmniAuth providers. +You can configure GitLab to act as a SAML service provider (SP). This allows +GitLab to consume assertions from a SAML identity provider (IdP), such as +Okta, to authenticate users. -## Glossary of common terms - -| Term | Description | -|--------------------------------|-------------| -| Identity provider (IdP) | The service which manages your user identities, such as Okta or OneLogin. | -| Service provider (SP) | GitLab can be configured as a SAML 2.0 SP. | -| Assertion | A piece of information about a user's identity, such as their name or role. Also known as claims or attributes. | -| Single Sign-On (SSO) | Name of authentication scheme. | -| Assertion consumer service URL | The callback on GitLab where users are redirected after successfully authenticating with the identity provider. | -| Issuer | How GitLab identifies itself to the identity provider. Also known as a "Relying party trust identifier". | -| Certificate fingerprint | Used to confirm that communications over SAML are secure by checking that the server is signing communications with the correct certificate. Also known as a certificate thumbprint. | +To set up SAML on GitLab.com, see [SAML SSO for GitLab.com groups](../user/group/saml_sso/index.md). -## General Setup +For more information on: -GitLab can be configured to act as a SAML 2.0 Service Provider (SP). This allows -GitLab to consume assertions from a SAML 2.0 Identity Provider (IdP), such as -Okta to authenticate users. +- OmniAuth provider settings, see the [OmniAuth documentation](omniauth.md). +- Commonly-used terms, see the [glossary of common terms](#glossary-of-common-terms). -First configure SAML 2.0 support in GitLab, then register the GitLab application -in your SAML IdP: +## Configure SAML support in GitLab -1. Make sure GitLab is configured with HTTPS. - See [Using HTTPS](../install/installation.md#using-https) for instructions. +1. Make sure GitLab is [configured with HTTPS](../install/installation.md#using-https). 1. On your GitLab server, open the configuration file. - For Omnibus package: + For Omnibus installations: ```shell sudo editor /etc/gitlab/gitlab.rb @@ -53,11 +41,12 @@ in your SAML IdP: sudo -u git -H editor config/gitlab.yml ``` -1. See [Configure initial settings](omniauth.md#configure-initial-settings) for initial settings. +1. Edit the initial [configuration settings](omniauth.md#configure-initial-settings). + 1. To allow your users to use SAML to sign up without having to manually create - an account first, add the following values to your configuration: + an account first, add the following values to your configuration. - For Omnibus package: + For Omnibus installations: ```ruby gitlab_rails['omniauth_allow_single_sign_on'] = ['saml'] @@ -73,10 +62,10 @@ in your SAML IdP: block_auto_created_users: false ``` -1. You can also automatically link SAML users with existing GitLab users if their - email addresses match by adding the following setting: +1. Optional. You can automatically link SAML users with existing GitLab users if their + email addresses match by adding the following setting. - For Omnibus package: + For Omnibus installations: ```ruby gitlab_rails['omniauth_auto_link_saml_user'] = true @@ -88,12 +77,21 @@ in your SAML IdP: auto_link_saml_user: true ``` -1. Ensure that the SAML [`NameID`](../user/group/saml_sso/index.md#nameid) and email address are fixed for each user, -as described in the section on [Security](#security). Otherwise, your users are able to sign in as other authorized users. + Alternatively, a user can manually link their SAML identity to an existing GitLab + account by [enabling OmniAuth for an existing user](omniauth.md#enable-omniauth-for-an-existing-user). -1. Add the provider configuration: +1. Configure the following attributes so your SAML users cannot change them: - For Omnibus package: + - [`NameID`](../user/group/saml_sso/index.md#nameid). + - `Email` when used with `omniauth_auto_link_saml_user`. + + If users can change these attributes, they can sign in as other authorized users. + See your SAML IdP documentation for information on how to make these attributes + unchangeable. + +1. Add the provider configuration. + + For Omnibus installations: ```ruby gitlab_rails['omniauth_providers'] = [ @@ -129,63 +127,118 @@ as described in the section on [Security](#security). Otherwise, your users are } ``` -1. Change the value for `assertion_consumer_service_url` to match the HTTPS endpoint - of GitLab (append `users/auth/saml/callback` to the HTTPS URL of your GitLab - installation to generate the correct value). +1. Match the value for `assertion_consumer_service_url` to the HTTPS endpoint + of GitLab. To generate the correct value, append `users/auth/saml/callback` to the + HTTPS URL of your GitLab installation. -1. Change the values of `idp_cert_fingerprint`, `idp_sso_target_url`, - `name_identifier_format` to match your IdP. If a fingerprint is used it must - be a SHA1 fingerprint; check - [the OmniAuth SAML documentation](https://github.com/omniauth/omniauth-saml) - for more details on these options. - See the [notes on configuring your identity provider](#notes-on-configuring-your-identity-provider) for more information. +1. Change the following values to match your IdP: + - `idp_cert_fingerprint`. + - `idp_sso_target_url`. + - `name_identifier_format`. + If you use a `idp_cert_fingerprint`, it must be a SHA1 fingerprint. For more + information on these values, see the + [OmniAuth SAML documentation](https://github.com/omniauth/omniauth-saml). + For more information on other configuration settings, see + [configuring SAML on your IdP](#configure-saml-on-your-idp). 1. Change the value of `issuer` to a unique name, which identifies the application to the IdP. -1. For the changes to take effect: - - If you installed via Omnibus, [reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure). - - If you installed from source, [restart GitLab](../administration/restart_gitlab.md#installations-from-source). +1. For the changes to take effect, if you installed: + - Using Omnibus, [reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure). + - From source, [restart GitLab](../administration/restart_gitlab.md#installations-from-source). -1. Register the GitLab SP in your SAML 2.0 IdP, using the application name specified - in `issuer`. +### Register GitLab in your SAML IdP -To ease configuration, most IdP accept a metadata URL for the application to provide -configuration information to the IdP. To build the metadata URL for GitLab, append -`users/auth/saml/metadata` to the HTTPS URL of your GitLab installation, for instance: +1. Register the GitLab SP in your SAML IdP, using the application name specified in `issuer`. -```plaintext -https://gitlab.example.com/users/auth/saml/metadata -``` +1. To provide configuration information to the IdP, build a metadata URL for the + application. To build the metadata URL for GitLab, append `users/auth/saml/metadata` + to the HTTPS URL of your GitLab installation. For example: -At a minimum the IdP *must* provide a claim containing the user's email address using `email` or `mail`. -See [the assertions list](#assertions) for other available claims. + ```plaintext + https://gitlab.example.com/users/auth/saml/metadata + ``` + + At a minimum the IdP **must** provide a claim containing the user's email address + using `email` or `mail`. For more information on other available claims, see + [configuring assertions](#configure-assertions). + +1. On the sign in page there should now be a SAML icon below the regular sign in form. + Select the icon to begin the authentication process. If authentication is successful, + you are returned to GitLab and signed in. + +### Configure SAML on your IdP -On the sign in page there should now be a SAML button below the regular sign in form. -Select the icon to begin the authentication process. If everything goes well the user -is returned to GitLab and signed in. +To configure a SAML application on your IdP, you need at least the following information: -### Use multiple SAML identity providers +- Assertion consumer service URL. +- Issuer. +- [`NameID`](../user/group/saml_sso/index.md#nameid). +- [Email address claim](#configure-assertions). + +For an example configuration, see [set up identity providers](#set-up-identity-providers). + +Your IdP may need additional configuration. For more information, see +[additional configuration for SAML apps on your IdP](#additional-configuration-for-saml-apps-on-your-idp). + +### Configure GitLab to use multiple SAML IdPs > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14361) in GitLab 14.6. -You can configure GitLab to use multiple SAML identity providers if: +You can configure GitLab to use multiple SAML IdPs if: -- Each provider has a unique name set that matches a name set in `args`. At least one provider **must** have the name `saml` to mitigate a - [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/366450) in GitLab 14.6 and newer. -- The providers' names are: - - Used in OmniAuth configuration for properties based on the provider name. For example, `allowBypassTwoFactor`, `allowSingleSignOn`, and - `syncProfileFromProvider`. - - Used for association to each existing user as an additional identity. +- Each provider has a unique name set that matches a name set in `args`. At least + one provider must have the name `saml` to mitigate a + [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/366450) in GitLab + 14.6 and newer. +- The providers' names are used: + - In OmniAuth configuration for properties based on the provider name. For example, + `allowBypassTwoFactor`, `allowSingleSignOn`, and `syncProfileFromProvider`. + - For association to each existing user as an additional identity. - The `assertion_consumer_service_url` matches the provider name. -- The `strategy_class` is explicitly set because it cannot be inferred from provider name. +- The `strategy_class` is explicitly set because it cannot be inferred from provider + name. -Example multiple providers configuration for Omnibus GitLab: +Example provider's configuration for installations from source: + +```yaml +omniauth: + providers: + - { + name: 'saml', # This must match the following name configuration parameter + args: { + name: 'saml', # This is mandatory and must match the provider name + strategy_class: 'OmniAuth::Strategies::SAML', + assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml_1/callback', # URL must match the name of the provider + ... # Put here all the required arguments similar to a single provider + }, + label: 'Provider 1' # Differentiate the two buttons and providers in the UI + } + - { + name: 'saml1', # This must match the following name configuration parameter + args: { + name: 'saml1', # This is mandatory and must match the provider name + strategy_class: 'OmniAuth::Strategies::SAML', + assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml_2/callback', # URL must match the name of the provider + ... # Put here all the required arguments similar to a single provider + }, + label: 'Provider 2' # Differentiate the two buttons and providers in the UI + } +``` + +Example provider's configuration for Omnibus GitLab installations: + +To allow your users to use SAML to sign up without having to manually create an account from either of the providers, add the following values to your configuration. + +```ruby +gitlab_rails['omniauth_allow_single_sign_on'] = ['saml', 'saml1'] +``` ```ruby gitlab_rails['omniauth_providers'] = [ { - name: 'saml', + name: 'saml', # This must match the following name configuration parameter args: { name: 'saml', # This is mandatory and must match the provider name strategy_class: 'OmniAuth::Strategies::SAML', @@ -195,7 +248,7 @@ gitlab_rails['omniauth_providers'] = [ label: 'Provider 1' # Differentiate the two buttons and providers in the UI }, { - name: 'saml1', + name: 'saml1', # This must match the following name configuration parameter args: { name: 'saml1', # This is mandatory and must match the provider name strategy_class: 'OmniAuth::Strategies::SAML', @@ -207,87 +260,126 @@ gitlab_rails['omniauth_providers'] = [ ] ``` -Example providers configuration for installations from source: +To allow your users to use SAML to sign up without having to manually create an +account from either of the providers, add the following values to your configuration. -```yaml -omniauth: - providers: - - { - name: 'saml', - args: { - name: 'saml', # This is mandatory and must match the provider name - strategy_class: 'OmniAuth::Strategies::SAML', - assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml_1/callback', # URL must match the name of the provider - ... # Put here all the required arguments similar to a single provider - }, - label: 'Provider 1' # Differentiate the two buttons and providers in the UI - } - - { - name: 'saml1', - args: { - name: 'saml1', # This is mandatory and must match the provider name - strategy_class: 'OmniAuth::Strategies::SAML', - assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml_2/callback', # URL must match the name of the provider - ... # Put here all the required arguments similar to a single provider - }, - label: 'Provider 2' # Differentiate the two buttons and providers in the UI - } +```ruby +gitlab_rails['omniauth_allow_single_sign_on'] = ['saml', 'saml1'] ``` -### Notes on configuring your identity provider +## Set up identity providers -When configuring a SAML app on the IdP, you need at least: +GitLab support of SAML means you can sign in to GitLab through a wide range +of IdPs. -- Assertion consumer service URL -- Issuer -- [`NameID`](../user/group/saml_sso/index.md#nameid) -- [Email address claim](#assertions) +GitLab provides the following content on setting up the Okta and Google Workspace +IdPs for guidance only. If you have any questions on configuring either of these +IdPs, contact your provider's support. -Your identity provider may require additional configuration, such as the following: +### Set up Okta -| Field | Value | Notes | -|-------|-------|-------| -| SAML profile | Web browser SSO profile | GitLab uses SAML to sign users in through their browser. No requests are made directly to the identity provider. | -| SAML request binding | HTTP Redirect | GitLab (the service provider) redirects users to your identity provider with a base64 encoded `SAMLRequest` HTTP parameter. | -| SAML response binding | HTTP POST | Specifies how the SAML token is sent by your identity provider. Includes the `SAMLResponse`, which a user's browser submits back to GitLab. | -| Sign SAML response | Required | Prevents tampering. | -| X.509 certificate in response | Required | Signs the response and checks against the provided fingerprint. | -| Fingerprint algorithm | SHA-1 | GitLab uses a SHA-1 hash of the certificate to sign the SAML Response. | -| Signature algorithm | SHA-1/SHA-256/SHA-384/SHA-512 | Determines how a response is signed. Also known as the digest method, this can be specified in the SAML response. | -| Encrypt SAML assertion | Optional | Uses TLS between your identity provider, the user's browser, and GitLab. | -| Sign SAML assertion | Optional | Validates the integrity of a SAML assertion. When active, signs the whole response. | -| Check SAML request signature | Optional | Checks the signature on the SAML response. | -| Default RelayState | Optional | Specifies the URL users should end up on after successfully signing in through SAML at your identity provider. | -| NameID format | Persistent | See [NameID format details](../user/group/saml_sso/index.md#nameid-format). | -| Additional URLs | Optional | May include the issuer (or identifier) or the assertion consumer service URL in other fields on some providers. | +1. In the Okta administrator section choose **Applications**. +1. On the app screen, select **Create App Integration** and then select + **SAML 2.0** on the next screen. +1. Optional. Choose and add a logo from [GitLab Press](https://about.gitlab.com/press/). + You must crop and resize the logo. +1. Complete the SAML general configuration. Enter: + - `"Single sign-on URL"`: Use the assertion consumer service URL. + - `"Audience URI"`: Use the issuer. + - [`NameID`](../user/group/saml_sso/index.md#nameid). + - [Assertions](#configure-assertions). +1. In the feedback section, enter that you're a customer and creating an + app for internal use. +1. At the top of your new app's profile, select **SAML 2.0 configuration instructions**. +1. Note the **Identity Provider Single Sign-On URL**. Use this URL for the + `idp_sso_target_url` on your GitLab configuration file. +1. Before you sign out of Okta, make sure you add your user and groups if any. + +### Set up Google Workspace + +Prerequisites: -For example configurations, see the [notes on specific providers](#providers). +- Make sure you have access to a +[Google Workspace Super Admin account](https://support.google.com/a/answer/2405986#super_admin). -### Assertions +1. Use the following information, and follow the instructions in +[Set up your own custom SAML application in Google Workspace](https://support.google.com/a/answer/6087519?hl=en). -| Field | Supported keys | -|-----------------|----------------| -| Email (required)| `email`, `mail` | -| Full Name | `name` | + | | Typical value | Description | + |------------------|--------------------------------------------------|----------------------------------------------------------| + | Name of SAML App | GitLab | Other names OK. | + | ACS URL | `https://<GITLAB_DOMAIN>/users/auth/saml/callback` | Assertion Consumer Service URL. | + | GITLAB_DOMAIN | `gitlab.example.com` | Your GitLab instance domain. | + | Entity ID | `https://gitlab.example.com` | A value unique to your SAML application. Set it to the `issuer` in your GitLab configuration. | + | Name ID format | EMAIL | Required value. Also known as `name_identifier_format`. | + | Name ID | Primary email address | Your email address. Make sure someone receives content sent to that address. | + | First name | `first_name` | First name. Required value to communicate with GitLab. | + | Last name | `last_name` | Last name. Required value to communicate with GitLab. | + +1. Set up the following SAML attribute mappings: + + | Google Directory attributes | App attributes | + |-----------------------------------|----------------| + | Basic information > Email | `email` | + | Basic Information > First name | `first_name` | + | Basic Information > Last name | `last_name` | + + You might use some of this information when you + [configure SAML support in GitLab](#configure-saml-support-in-gitlab). + +When configuring the Google Workspace SAML application, record the following information: + +| | Value | Description | +|-------------|--------------|-----------------------------------------------------------------------------------| +| SSO URL | Depends | Google Identity Provider details. Set to the GitLab `idp_sso_target_url` setting. | +| Certificate | Downloadable | Run `openssl x509 -in <your_certificate.crt> -noout -fingerprint` to generate the SHA1 fingerprint that can be used in the `idp_cert_fingerprint` setting. | + +Google Workspace Administrator also provides the IdP metadata, Entity ID, and SHA-256 +fingerprint. However, GitLab does not need this information to connect to the +Google Workspace SAML application. + +### Set up other IdPs + +Some IdPs have documentation on how to use them as the IdP in SAML configurations. +For example: + +- [Active Directory Federation Services (ADFS)](https://learn.microsoft.com/en-us/windows-server/identity/ad-fs/operations/create-a-relying-party-trust) +- [Auth0](https://auth0.com/docs/authenticate/protocols/saml/saml-sso-integrations/configure-auth0-saml-identity-provider) + +If you have any questions on configuring your IdP in a SAML configuration, contact +your provider's support. + +### Configure assertions + +| Field | Supported default keys | +|-----------------|------------------------| +| Email (required)| `email`, `mail` | +| Full Name | `name` | | First Name | `first_name`, `firstname`, `firstName` | -| Last Name | `last_name`, `lastname`, `lastName` | +| Last Name | `last_name`, `lastname`, `lastName` | + +See [`attribute_statements`](#map-saml-response-attribute-names) for: -See [`attribute_statements`](#attribute_statements) for examples on how custom -assertions are configured. This section also describes how to configure custom -username attributes. +- Custom assertion configuration examples. +- How to configure custom username attributes. -Please refer to [the OmniAuth SAML gem](https://github.com/omniauth/omniauth-saml/blob/master/lib/omniauth/strategies/saml.rb) -for a full list of supported assertions. +For a full list of supported assertions, see the [OmniAuth SAML gem](https://github.com/omniauth/omniauth-saml/blob/master/lib/omniauth/strategies/saml.rb) -## SAML Groups +## Configure users based on SAML group membership -You can require users to be members of a certain group, or assign users [external](../user/permissions.md#external-users), administrator or [auditor](../user/permissions.md#auditor-users) roles based on group membership. -These groups are checked on each SAML login and user attributes updated as necessary. -This feature **does not** allow you to -automatically add users to GitLab [Groups](../user/group/index.md). +You can: -Support for these groups depends on your [subscription](https://about.gitlab.com/pricing/) -and whether you've installed [GitLab Enterprise Edition (EE)](https://about.gitlab.com/install/). +- Require users to be members of a certain group. +- Assign users [external](../user/admin_area/external_users.md), administrator or [auditor](../administration/auditor_users.md) roles based on group membership. + +GitLab checks these groups on each SAML sign in and updates user attributes as necessary. +This feature **does not** allow you to automatically add users to GitLab +[Groups](../user/group/index.md). + +Support for these groups depends on: + +- Your [subscription](https://about.gitlab.com/pricing/). +- Whether you've installed [GitLab Enterprise Edition (EE)](https://about.gitlab.com/install/). | Group | Tier | GitLab Enterprise Edition (EE) Only? | |------------------------------|--------------------|--------------------------------------| @@ -296,11 +388,11 @@ and whether you've installed [GitLab Enterprise Edition (EE)](https://about.gitl | [Admin](#administrator-groups) | **(FREE SELF)** | Yes | | [Auditor](#auditor-groups) | **(PREMIUM SELF)** | Yes | -### Requirements +### Prerequisites -First tell GitLab where to look for group information. For this, you -must make sure that your IdP server sends a specific `AttributeStatement` along -with the regular SAML response. Here is an example: +You must tell GitLab where to look for group information. To do this, make sure +that your IdP server sends a specific `AttributeStatement` along with the regular +SAML response. For example: ```xml <saml:AttributeStatement> @@ -313,22 +405,25 @@ with the regular SAML response. Here is an example: </saml:AttributeStatement> ``` -The name of the attribute can be anything you like, but it must contain the groups -to which a user belongs. To tell GitLab where to find these groups, you need -to add a `groups_attribute:` element to your SAML settings. +The name of the attribute must contain the groups that a user belongs to. +To tell GitLab where to find these groups, add a `groups_attribute:` +element to your SAML settings. ### Required groups -Your IdP passes Group information to the SP (GitLab) in the SAML Response. -To use this response, configure GitLab to identify: +Your IdP passes group information to GitLab in the SAML response. To use this +response, configure GitLab to identify: + +- Where to look for the groups in the SAML response, using the `groups_attribute` setting. +- Information about a group or user, using a group setting. -- Where to look for the groups in the SAML response via the `groups_attribute` setting -- Which group membership is requisite to sign in via the `required_groups` setting +Use the `required_groups` setting to configure GitLab to identify which group +membership is required to sign in. -When `required_groups` is empty or not set, anyone with proper authentication -is able to use the service. +If you do not set `required_groups` or leave the setting empty, anyone with proper +authentication can use the service. -Example: +Example configuration: ```yaml { name: 'saml', @@ -346,9 +441,17 @@ Example: ### External groups -SAML login supports the automatic identification of a user as an -[external user](../user/permissions.md#external-users). This is based on the user's group -membership in the SAML identity provider. +Your IdP passes group information to GitLab in the SAML response. To use this +response, configure GitLab to identify: + +- Where to look for the groups in the SAML response, using the `groups_attribute` setting. +- Information about a group or user, using a group setting. + +SAML can automatically identify a user as an +[external user](../user/admin_area/external_users.md), based on the `external_groups` +setting. + +Example configuration: ```yaml { name: 'saml', @@ -366,11 +469,16 @@ membership in the SAML identity provider. ### Administrator groups -The requirements are the same as the previous settings: +Your IdP passes group information to GitLab in the SAML response. To use this +response, configure GitLab to identify: + +- Where to look for the groups in the SAML response, using the `groups_attribute` setting. +- Information about a group or user, using a group setting. -- The IdP must pass Group information to GitLab. -- GitLab must know where to look for the groups in the SAML response, as well as - which groups grant the user administrator access. +Use the `admin_groups` setting to configure GitLab to identify which groups grant +the user administrator access. + +Example configuration: ```yaml { name: 'saml', @@ -390,11 +498,16 @@ The requirements are the same as the previous settings: > Introduced in GitLab 11.4. -The requirements are the same as the previous settings: +Your IdP passes group information to GitLab in the SAML response. To use this +response, configure GitLab to identify: + +- Where to look for the groups in the SAML response, using the `groups_attribute` setting. +- Information about a group or user, using a group setting. + +Use the `auditor_groups` setting to configure GitLab to identify which groups include +users with [auditor access](../administration/auditor_users.md). -- The IdP must pass Group information to GitLab. -- GitLab should know where to look for the groups in the SAML response, as well as which - groups include users with the [Auditor role](../user/permissions.md#auditor-users). +Example configuration: ```yaml { name: 'saml', @@ -410,17 +523,17 @@ The requirements are the same as the previous settings: } } ``` -## Group Sync +## Automatically manage SAML Group Sync For information on automatically managing GitLab group membership, see [SAML Group Sync](../user/group/saml_sso/group_sync.md). -## Bypass two factor authentication +## Bypass two-factor authentication -If you want some SAML authentication methods to count as 2FA on a per session -basis, you can register them in the `upstream_two_factor_authn_contexts` list. +To configure a SAML authentication method to count as two-factor authentication +(2FA) on a per session basis, register that method in the `upstream_two_factor_authn_contexts` +list. -In addition to the changes in GitLab, make sure that your IdP is returning the -`AuthnContext`. For example: +1. Make sure that your IdP is returning the `AuthnContext`. For example: ```xml <saml:AuthnStatement> @@ -430,7 +543,11 @@ In addition to the changes in GitLab, make sure that your IdP is returning the </saml:AuthnStatement> ``` -**For Omnibus installations:** +1. Edit your installation configuration to register the SAML authentication method + in the `upstream_two_factor_authn_contexts` list. How you edit your configuration + will differ depending on your installation type. + +### Omnibus GitLab installations 1. Edit `/etc/gitlab/gitlab.rb`: @@ -456,11 +573,10 @@ In addition to the changes in GitLab, make sure that your IdP is returning the ] ``` -1. Save the file and [reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. - ---- +1. Save the file and [reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) + GitLab for the changes to take effect. -**For installations from source:** +### Installations from source 1. Edit `config/gitlab.yml`: @@ -486,17 +602,62 @@ In addition to the changes in GitLab, make sure that your IdP is returning the } ``` -1. Save the file and [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect +1. Save the file and [restart GitLab](../administration/restart_gitlab.md#installations-from-source) + for the changes to take effect. + +## Validate response signatures -## Customization +IdPs must sign SAML responses to ensure that the assertions are not tampered with. -### `auto_sign_in_with_provider` +This prevents user impersonation and privilege escalation when specific group +membership is required. + +You configure the response signature validation using `idp_cert_fingerprint`. +An example configuration: + +```yaml +args: { + assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', + idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', + idp_sso_target_url: 'https://login.example.com/idp', + issuer: 'https://gitlab.example.com', + name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent', +} +``` -You can add this setting to your GitLab configuration to automatically redirect you -to your SAML server for authentication. This removes the requirement to select a button -before actually signing in. +If your IdP does not support configuring this using `idp_cert_fingerprint`, you +can instead configure GitLab directly using `idp_cert`. An example configuration: -For Omnibus package: +```yaml +args: { + assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', + idp_cert: '-----BEGIN CERTIFICATE----- + <redacted> + -----END CERTIFICATE-----', + idp_sso_target_url: 'https://login.example.com/idp', + issuer: 'https://gitlab.example.com', + name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent', +} +``` + +If you have configured the response signature validation incorrectly, you might see +error messages such as: + +- A key validation error. +- Digest mismatch. +- Fingerprint mismatch. + +For more information on solving these errors, see the [troubleshooting SAML guide](../user/group/saml_sso/troubleshooting.md). + +## Customize SAML settings + +### Redirect users to SAML server for authentication + +You can add the `auto_sign_in_with_provider` setting to your GitLab configuration +to automatically redirect you to your SAML server for authentication. This removes +the requirement to select an element before actually signing in. + +For Omnibus GitLab installations: ```ruby gitlab_rails['omniauth_auto_sign_in_with_provider'] = 'saml' @@ -509,31 +670,28 @@ omniauth: auto_sign_in_with_provider: saml ``` -Keep in mind that every sign in attempt redirects to the SAML server; -you cannot sign in using local credentials. Ensure at least one of the -SAML users has administrator access. +Every sign in attempt redirects to the SAML server, so you cannot sign in using +local credentials. Make sure at least one of the SAML users has administrator access. -You may also bypass the auto sign-in feature by browsing to +You can also bypass the auto sign-in feature by `https://gitlab.example.com/users/sign_in?auto_sign_in=false`. -### `attribute_statements` **(FREE SELF)** - -NOTE: -This setting should be used only to map attributes that are part of the OmniAuth -`info` hash schema. +### Map SAML response attribute names **(FREE SELF)** -`attribute_statements` is used to map Attribute Names in a SAMLResponse to entries +You can use `attribute_statements` to map attribute names in a SAML response to entries in the OmniAuth [`info` hash](https://github.com/omniauth/omniauth/wiki/Auth-Hash-Schema#schema-10-and-later). -For example, if your SAMLResponse contains an Attribute called `EmailAddress`, +NOTE: +Only use this setting to map attributes that are part of the OmniAuth `info` hash schema. + +For example, if your `SAMLResponse` contains an Attribute called `EmailAddress`, specify `{ email: ['EmailAddress'] }` to map the Attribute to the corresponding key in the `info` hash. URI-named Attributes are also supported, for example, `{ email: ['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress'] }`. -This setting allows you tell GitLab where to look for certain attributes required -to create an account. Like mentioned above, if your IdP sends the user's email -address as `EmailAddress` instead of `email`, let GitLab know by setting it on -your configuration: +Use this setting to tell GitLab where to look for certain attributes required +to create an account. If your IdP sends the user's email address as `EmailAddress` +instead of `email`, let GitLab know by setting it on your configuration: ```yaml args: { @@ -566,7 +724,7 @@ args: { This also sets the `username` attribute in your SAML Response to the username in GitLab. -### `allowed_clock_drift` +### Allow for clock drift The clock of the Identity Provider may drift slightly ahead of your system clocks. To allow for a small amount of clock drift, you can use `allowed_clock_drift` in @@ -585,9 +743,11 @@ args: { } ``` -### `uid_attribute` +### Designate a unique attribute for the `uid` -By default, the `uid` is set as the `name_id` in the SAML response. If you'd like to designate a unique attribute for the `uid`, you can set the `uid_attribute`. In the example below, the value of `uid` attribute in the SAML response is set as the `uid_attribute`. +By default, the `uid` is set as the `name_id` in the SAML response. To designate +a unique attribute for the `uid`, you can set the `uid_attribute`. In the following +example, the value of `uid` attribute in the SAML response is set as the `uid_attribute`. ```yaml args: { @@ -600,59 +760,19 @@ args: { } ``` -Make sure you read the [Security](#security) section before changing this value. - -## Response signature validation (required) - -We require Identity Providers to sign SAML responses to ensure that the assertions are -not tampered with. - -This prevents user impersonation and prevents privilege escalation when specific group -membership is required. Typically this: +Before setting the `uid` to a unique attribute, make sure that you have configured +the following attributes so your SAML users cannot change them: -- Is configured using `idp_cert_fingerprint`. -- Includes the full certificate in the response, although if your Identity Provider - doesn't support this, you can directly configure GitLab using the `idp_cert` option. +- [`NameID`](../user/group/saml_sso/index.md#nameid). +- `Email` when used with `omniauth_auto_link_saml_user`. -Example configuration with `idp_cert_fingerprint`: - -```yaml -args: { - assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', - idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', - idp_sso_target_url: 'https://login.example.com/idp', - issuer: 'https://gitlab.example.com', - name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent', -} -``` +If users can change these attributes, they can sign in as other authorized users. +See your SAML IdP documentation for information on how to make these attributes +unchangeable. -Example configuration with `idp_cert`: +## Assertion encryption (optional) -```yaml -args: { - assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', - idp_cert: '-----BEGIN CERTIFICATE----- - <redacted> - -----END CERTIFICATE-----', - idp_sso_target_url: 'https://login.example.com/idp', - issuer: 'https://gitlab.example.com', - name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent', -} -``` - -If the response signature validation is configured incorrectly, you can see error messages -such as: - -- A key validation error. -- Digest mismatch. -- Fingerprint mismatch. - -Refer to the [troubleshooting section](#troubleshooting) for more information on -debugging these errors. - -## Assertion Encryption (optional) - -GitLab requires the use of TLS encryption with SAML, but in some cases there can be a +GitLab requires the use of TLS encryption with SAML 2.0, but in some cases there can be a need for additional encryption of the assertions. This may be the case, for example, if you terminate TLS encryption early at a load @@ -679,7 +799,7 @@ Your Identity Provider encrypts the assertion with the public certificate of Git NOTE: This integration uses the `certificate` and `private_key` settings for both assertion encryption and request signing. -## Request signing (optional) +## Sign SAML authentication requests (optional) Another optional configuration is to sign SAML authentication requests. GitLab SAML Requests use the SAML redirect binding, so this isn't necessary (unlike the @@ -713,27 +833,21 @@ args: { GitLab signs the request with the provided private key. GitLab includes the configured public x500 certificate in the metadata for your Identity Provider to validate the signature of the received request with. For more information on this option, see the [Ruby SAML gem documentation](https://github.com/onelogin/ruby-saml/tree/v1.7.0). The Ruby SAML gem is used by the [OmniAuth SAML gem](https://github.com/omniauth/omniauth-saml) to implement the client side of the SAML authentication. -## Security - -Avoid user control of the following attributes: +## Password generation for users created through SAML -- [`NameID`](../user/group/saml_sso/index.md#nameid) -- `Email` when used with `omniauth_auto_link_saml_user` - -These attributes define the SAML user. If users can change these attributes, they can impersonate others. - -Refer to the documentation for your SAML Identity Provider for information on how to fix these attributes. +The [Generated passwords for users created through integrated authentication](../security/passwords_for_integrated_authentication_methods.md) guide provides an overview of how GitLab generates and sets passwords for users created via SAML. -## Passwords for users created via SAML +Users authenticated with SSO or SAML must not use a password for Git operations over HTTPS. These users can do one of the following instead: -The [Generated passwords for users created through integrated authentication](../security/passwords_for_integrated_authentication_methods.md) guide provides an overview of how GitLab generates and sets passwords for users created via SAML. +- Set up a [personal access token](../user/profile/personal_access_tokens.md). +- Use the [Git Credential Manager](../user/profile/account/two_factor_authentication.md#git-credential-manager) which securely authenticates using OAuth. ## Link SAML identity for an existing user A user can manually link their SAML identity to an existing GitLab account by following the steps in [Enable OmniAuth for an existing user](omniauth.md#enable-omniauth-for-an-existing-user). -## Configuring Group SAML on a self-managed GitLab instance **(PREMIUM SELF)** +## Group SAML on a self-managed GitLab instance **(PREMIUM SELF)** For information on the GitLab.com implementation, please see the [SAML SSO for GitLab.com groups page](../user/group/saml_sso). @@ -741,10 +855,10 @@ Group SAML SSO helps if you have to allow access via multiple SAML identity prov To proceed with configuring Group SAML SSO instead, enable the `group_saml` OmniAuth provider. This can be done from: -- `gitlab.rb` for [Omnibus GitLab installations](#omnibus-installations). -- `gitlab/config/gitlab.yml` for [source installations](#source-installations). +- `gitlab.rb` for Omnibus GitLab installations. +- `gitlab/config/gitlab.yml` for source installations. -### Limitations +### Self-managed instance group SAML limitations Group SAML on a self-managed instance is limited when compared to the recommended [instance-wide SAML](../user/group/saml_sso/index.md). The recommended solution allows you to take advantage of: @@ -755,7 +869,7 @@ Group SAML on a self-managed instance is limited when compared to the recommende - [Administrator groups](#administrator-groups). - [Auditor groups](#auditor-groups). -### Omnibus installations +For Omnibus installations: 1. Make sure GitLab is [configured with HTTPS](../install/installation.md#using-https). @@ -766,7 +880,7 @@ Group SAML on a self-managed instance is limited when compared to the recommende gitlab_rails['omniauth_providers'] = [{ name: 'group_saml' }] ``` -### Source installations +For installations from source: 1. Make sure GitLab is [configured with HTTPS](../install/installation.md#using-https). @@ -779,78 +893,39 @@ Group SAML on a self-managed instance is limited when compared to the recommende - { name: 'group_saml' } ``` -## Providers - -GitLab support of SAML means that you can sign in to GitLab with a wide range of identity providers. -Your identity provider may have additional documentation. Some identity providers include -documentation on how to use SAML to sign in to GitLab. - -Examples: - -- [ADFS (Active Directory Federation Services)](https://learn.microsoft.com/en-us/windows-server/identity/ad-fs/operations/create-a-relying-party-trust) -- [Auth0](https://auth0.com/docs/authenticate/protocols/saml/saml-sso-integrations/configure-auth0-saml-identity-provider) - -GitLab provides the following setup notes for guidance only. -If you have any questions on configuring the SAML app, please contact your provider's support. - -### Okta setup notes - -1. In the Okta administrator section choose **Applications**. -1. When the app screen comes up you see another button to **Create App Integration** and - choose SAML 2.0 on the next screen. -1. Optionally, you can add a logo - (you can choose it from <https://about.gitlab.com/press/>). You must - crop and resize it. -1. Next, fill in the SAML general configuration with - the assertion consumer service URL as "Single sign-on URL" and - the issuer as "Audience URI" along with the [NameID](../user/group/saml_sso/index.md#nameid) and [assertions](#assertions). -1. The last part of the configuration is the feedback section where you can - just say you're a customer and creating an app for internal use. -1. When you have your app you can see a few tabs on the top of the app's - profile. Select the SAML 2.0 configuration instructions button. -1. On the screen that comes up take note of the - **Identity Provider Single Sign-On URL** which you can use for the - `idp_sso_target_url` on your GitLab configuration file. -1. **Before you leave Okta, make sure you add your user and groups if any.** - -### Google workspace setup notes - -The following guidance is based on this Google Workspace article, on how to [Set up your own custom SAML application](https://support.google.com/a/answer/6087519?hl=en): - -Make sure you have access to a Google Workspace [Super Admin](https://support.google.com/a/answer/2405986#super_admin) account. - Use the information below and follow the instructions in the linked Google Workspace article. - -| | Typical value | Description | -|------------------|--------------------------------------------------|----------------------------------------------------------| -| Name of SAML App | GitLab | Other names OK. | -| ACS URL | `https://<GITLAB_DOMAIN>/users/auth/saml/callback` | ACS is short for Assertion Consumer Service. | -| GITLAB_DOMAIN | `gitlab.example.com` | Set to the domain of your GitLab instance. | -| Entity ID | `https://gitlab.example.com` | A value unique to your SAML app, set it to the `issuer` in your GitLab configuration. | -| Name ID format | EMAIL | Required value. Also known as `name_identifier_format` | -| Name ID | Primary email address | Make sure someone receives content sent to that address | -| First name | `first_name` | Required value to communicate with GitLab. | -| Last name | `last_name` | Required value to communicate with GitLab. | +## Additional configuration for SAML apps on your IdP -You also must setup the following SAML attribute mappings: +When configuring a SAML app on the IdP, your identity provider may require additional configuration, such as the following: -| Google Directory attributes | App attributes | -|-----------------------------------|----------------| -| Basic information > Email | `email` | -| Basic Information > First name | `first_name` | -| Basic Information > Last name | `last_name` | - -You may also use some of this information when you [configure GitLab](#general-setup). +| Field | Value | Notes | +|-------|-------|-------| +| SAML profile | Web browser SSO profile | GitLab uses SAML to sign users in through their browser. No requests are made directly to the identity provider. | +| SAML request binding | HTTP Redirect | GitLab (the service provider) redirects users to your identity provider with a base64 encoded `SAMLRequest` HTTP parameter. | +| SAML response binding | HTTP POST | Specifies how the SAML token is sent by your identity provider. Includes the `SAMLResponse`, which a user's browser submits back to GitLab. | +| Sign SAML response | Required | Prevents tampering. | +| X.509 certificate in response | Required | Signs the response and checks against the provided fingerprint. | +| Fingerprint algorithm | SHA-1 | GitLab uses a SHA-1 hash of the certificate to sign the SAML Response. | +| Signature algorithm | SHA-1/SHA-256/SHA-384/SHA-512 | Determines how a response is signed. Also known as the digest method, this can be specified in the SAML response. | +| Encrypt SAML assertion | Optional | Uses TLS between your identity provider, the user's browser, and GitLab. | +| Sign SAML assertion | Optional | Validates the integrity of a SAML assertion. When active, signs the whole response. | +| Check SAML request signature | Optional | Checks the signature on the SAML response. | +| Default RelayState | Optional | Specifies the URL users should end up on after successfully signing in through SAML at your identity provider. | +| NameID format | Persistent | See [NameID format details](../user/group/saml_sso/index.md#nameid-format). | +| Additional URLs | Optional | May include the issuer (or identifier) or the assertion consumer service URL in other fields on some providers. | -When configuring the Google Workspace SAML app, be sure to record the following information: +For example configurations, see the [notes on specific providers](#set-up-identity-providers). -| | Value | Description | -|-------------|--------------|-----------------------------------------------------------------------------------| -| SSO URL | Depends | Google Identity Provider details. Set to the GitLab `idp_sso_target_url` setting. | -| Certificate | Downloadable | Run `openssl x509 -in <your_certificate.crt> -noout -fingerprint` to generate the SHA1 fingerprint that can be used in the `idp_cert_fingerprint` setting. | +## Glossary of common terms -While the Google Workspace Administrator provides IdP metadata, Entity ID, and SHA-256 -fingerprint, they are not required. GitLab does not need that information to -connect to the Google Workspace SAML app. +| Term | Description | +|--------------------------------|-------------| +| Identity provider (IdP) | The service which manages your user identities, such as Okta or OneLogin. | +| Service provider (SP) | GitLab can be configured as a SAML 2.0 SP. | +| Assertion | A piece of information about a user's identity, such as their name or role. Also known as claims or attributes. | +| Single Sign-On (SSO) | Name of authentication scheme. | +| Assertion consumer service URL | The callback on GitLab where users are redirected after successfully authenticating with the identity provider. | +| Issuer | How GitLab identifies itself to the identity provider. Also known as a "Relying party trust identifier". | +| Certificate fingerprint | Used to confirm that communications over SAML are secure by checking that the server is signing communications with the correct certificate. Also known as a certificate thumbprint. | ## Troubleshooting diff --git a/doc/operations/error_tracking.md b/doc/operations/error_tracking.md index 85b64eb7b3e..d0a208c995b 100644 --- a/doc/operations/error_tracking.md +++ b/doc/operations/error_tracking.md @@ -127,17 +127,14 @@ Marking an error as resolved indicates that the error has stopped firing events. If another event occurs, the error reverts to unresolved. -## Integrated error tracking +## Integrated error tracking **(FREE SAAS)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/329596) in GitLab 14.4. > - [Disabled](https://gitlab.com/gitlab-org/gitlab/-/issues/353639) in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `integrated_error_tracking`. Disabled by default. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/7586) in GitLab 15.6. -FLAG: -By default this feature is not available. To make it available on self-managed GitLab, ask an -administrator to [enable the feature flag](../administration/feature_flags.md) -named `integrated_error_tracking`. The feature is not ready for production use. -On GitLab.com, this feature is available. +NOTE: +Available only on GitLab.com. This feature is currently in [Open Beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#open-beta). Integrated error tracking is a lightweight alternative to Sentry backend. You still use Sentry SDK with your application. But you don't need to deploy Sentry @@ -149,9 +146,8 @@ settings. By using a GitLab-provided DSN, your application connects to GitLab to Those errors are stored in the GitLab database and rendered by the GitLab UI, in the same way as Sentry integration. -In GitLab 15.6 and later, the integrated error tracking is available as an -[open Beta](../policy/alpha-beta-support.md#beta-features). -It now uses a new backend based on the ClickHouse database that enables better scalability. +In GitLab 15.6 and later, the integrated error tracking +uses a new backend based on the ClickHouse database that enables better scalability. Integrated error tracking remains limited in comparison to the Sentry backend, as only a small subset of the Sentry API is implemented. diff --git a/doc/operations/incident_management/alerts.md b/doc/operations/incident_management/alerts.md index d6293cf1479..0dffa15351c 100644 --- a/doc/operations/incident_management/alerts.md +++ b/doc/operations/incident_management/alerts.md @@ -39,7 +39,7 @@ The alert list displays the following information: NOTE: Check out a live example available from the -[`tanuki-inc` project page](https://gitlab-examples-ops-incident-setup-everyone-tanuki-inc.34.69.64.147.nip.io/) +[`tanuki-inc` project page](https://gitlab.com/gitlab-examples/ops/incident-setup/everyone/tanuki-inc) in GitLab to examine alerts in action. ## Alert severity @@ -113,7 +113,7 @@ timeline of the alert's investigation and assignment history. The following actions result in a system note: - [Updating the status of an alert](#change-an-alerts-status) -- [Creating an incident based on an alert](#create-an-incident-from-an-alert) +- [Creating an incident based on an alert](manage_incidents.md#from-an-alert) - [Assignment of an alert to a user](#assign-an-alert) - [Escalation of an alert to on-call responders](paging.md#escalating-an-alert) @@ -149,7 +149,7 @@ To change an alert's status: 1. On the right sidebar, select **Edit**. 1. Select a status. -To stop email notifications for alert reoccurrences in projects with [email notifications enabled](paging.md#email-notifications-for-alerts), +To stop email notifications for alert recurrences in projects with [email notifications enabled](paging.md#email-notifications-for-alerts), change the alert's status away from **Triggered**. #### Resolve an alert by closing the linked incident @@ -158,8 +158,8 @@ Prerequisites: - You must have at least the Reporter role. -When you close an [incident](incidents.md) that is linked to an alert, -the linked alert's status changes to **Resolved**. +When you [close an incident](manage_incidents.md#close-an-incident) that is linked to an alert, +GitLab [changes the alert's status](#change-an-alerts-status) to **Resolved**. You are then credited with the alert's status change. #### As an on-call responder **(PREMIUM)** @@ -173,25 +173,10 @@ Changing the status has the following effects: - To **Resolved**: silences all on-call pages for the alert. - From **Resolved** to **Triggered**: restarts the alert escalating. -In GitLab 15.1 and earlier, updating the status of an [alert with an associated incident](alerts.md#create-an-incident-from-an-alert) +In GitLab 15.1 and earlier, updating the status of an [alert with an associated incident](manage_incidents.md#from-an-alert) also updates the incident status. In [GitLab 15.2 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/356057), the incident status is independent and does not update when the alert status changes. -### Create an incident from an alert - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217745) in GitLab 13.1. - -The Alert detail view enables you to create an issue with a -description populated from an alert. To create the issue, -select the **Create Issue** button. You can then view the issue from the -alert by selecting the **View Issue** button. - -You can also [create incidents for alerts automatically](incidents.md#create-incidents-automatically). - -Closing a GitLab issue associated with an alert [changes the alert's status](#change-an-alerts-status) to -**Resolved**. See [Alert List](#alert-list) for more details -about alert statuses. - ### Assign an alert > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. @@ -225,18 +210,7 @@ and clear the user from the list of assignees, or select **Unassigned**. > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3066) in GitLab 13.1. -You can manually create [To-Do list items](../../user/todos.md) for yourself -from the Alert details screen, and view them later on your **To-Do List**. To -add a to-do item: - -1. Display the list of current alerts: - - 1. On the top bar, select **Main menu > Projects** and find your project. - 1. On the left sidebar, select **Monitor > Alerts**. - -1. Select your desired alert to display its **Alert Management Details View**. -1. On the right sidebar, select **Add a to do**: - -  +You can manually create a [to-do item](../../user/todos.md) for yourself +from an alert, and view it later on your **To-Do List**. -To view your To-Do List, on the top bar, select **To-Do List** (**{todo-done}**). +To add a to-do item, on the right sidebar, select **Add a to do**. diff --git a/doc/operations/incident_management/img/alert_detail_add_todo_v13_9.png b/doc/operations/incident_management/img/alert_detail_add_todo_v13_9.png Binary files differdeleted file mode 100644 index 5beb1cd0bfb..00000000000 --- a/doc/operations/incident_management/img/alert_detail_add_todo_v13_9.png +++ /dev/null diff --git a/doc/operations/incident_management/img/incident_list_create_v13_3.png b/doc/operations/incident_management/img/incident_list_create_v13_3.png Binary files differdeleted file mode 100644 index a000c849099..00000000000 --- a/doc/operations/incident_management/img/incident_list_create_v13_3.png +++ /dev/null diff --git a/doc/operations/incident_management/img/incident_list_v14_9.png b/doc/operations/incident_management/img/incident_list_v14_9.png Binary files differdeleted file mode 100644 index 4cf6f0e6522..00000000000 --- a/doc/operations/incident_management/img/incident_list_v14_9.png +++ /dev/null diff --git a/doc/operations/incident_management/img/incident_list_v15_6.png b/doc/operations/incident_management/img/incident_list_v15_6.png Binary files differnew file mode 100644 index 00000000000..fe2a91e2eba --- /dev/null +++ b/doc/operations/incident_management/img/incident_list_v15_6.png diff --git a/doc/operations/incident_management/img/new_incident_create_v13_4.png b/doc/operations/incident_management/img/new_incident_create_v13_4.png Binary files differdeleted file mode 100644 index 458532736bd..00000000000 --- a/doc/operations/incident_management/img/new_incident_create_v13_4.png +++ /dev/null diff --git a/doc/operations/incident_management/img/pagerduty_incidents_integration_v13_3.png b/doc/operations/incident_management/img/pagerduty_incidents_integration_v13_3.png Binary files differdeleted file mode 100644 index 08a45d001c2..00000000000 --- a/doc/operations/incident_management/img/pagerduty_incidents_integration_v13_3.png +++ /dev/null diff --git a/doc/operations/incident_management/img/timeline_view_toggle_v14_10.png b/doc/operations/incident_management/img/timeline_view_toggle_v14_10.png Binary files differdeleted file mode 100644 index 90f1d205fed..00000000000 --- a/doc/operations/incident_management/img/timeline_view_toggle_v14_10.png +++ /dev/null diff --git a/doc/operations/incident_management/incident_timeline_events.md b/doc/operations/incident_management/incident_timeline_events.md index 58448222356..4af5a815929 100644 --- a/doc/operations/incident_management/incident_timeline_events.md +++ b/doc/operations/incident_management/incident_timeline_events.md @@ -78,13 +78,15 @@ The comment is shown on the incident timeline as a timeline event. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/375280) in GitLab 15.6. -A new timeline event is created when someone [changes the severity](incidents.md#change-severity) +A new timeline event is created when someone [changes the severity](manage_incidents.md#change-severity) of an incident.  ## Delete an event +> Ability to delete an event when editing it [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/372265) in GitLab 15.7. + You can also delete timeline events. Prerequisites: @@ -99,3 +101,9 @@ To delete a timeline event: 1. Select the **Timeline** tab. 1. On the right of a timeline event, select **More actions** (**{ellipsis_v}**) and then select **Delete**. 1. To confirm, select **Delete Event**. + +Alternatively: + +1. On the right of a timeline event, select **More actions** (**{ellipsis_v}**) and then select **Edit**. +1. Select **Delete**. +1. To confirm, select **Delete event**. diff --git a/doc/operations/incident_management/incidents.md b/doc/operations/incident_management/incidents.md index a5d38b1a27c..5cfb8a77fc9 100644 --- a/doc/operations/incident_management/incidents.md +++ b/doc/operations/incident_management/incidents.md @@ -6,165 +6,78 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Incidents **(FREE)** -Incidents are critical entities in incident management workflows. They represent -a service disruption or outage that needs to be restored urgently. GitLab provides -tools for the triage, response, and remediation of incidents. +An incident is a service disruption or outage that needs to be restored urgently. +Incidents are critical in incident management workflows. +Use GitLab to triage, respond, and remediate incidents. -## Incident creation +## Incidents list -You can create an incident manually or automatically. +When you [view the incidents list](manage_incidents.md#view-incidents-list), it contains the following: -### Create incidents manually - -> - [Moved](https://gitlab.com/gitlab-org/monitor/monitor/-/issues/24) to GitLab Free in 13.3. -> - [Permission changed](https://gitlab.com/gitlab-org/gitlab/-/issues/336624) from Guest to Reporter in GitLab 14.5. -> - Automatic application of the `incident` label [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/290964) in GitLab 14.8. - -If you have at least Reporter [permissions](../../user/permissions.md), -you can create an incident manually from the Incidents List or the Issues List. - -To create an incident from the Incidents List: - -1. Navigate to **Monitor > Incidents** and select **Create Incident**. -1. Create a new issue using the `incident` template. - - - -To create an incident from the Issues List: - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230857) in GitLab 13.4. - -1. Go to **Issues > List**, and select **New issue**. -1. In the **Type** dropdown list, select **Incident**. Only fields relevant to - incidents are displayed on the page. -1. Create the incident as needed, and select **Create issue** to save the - incident. - - - -### Create incidents automatically **(ULTIMATE)** +- **State**: To filter incidents by their state, select **Open**, **Closed**, + or **All** above the incident list. +- **Search**: Search for incident titles and descriptions or [filter the list](#filter-the-incidents-list). +- **Severity**: Severity of a particular incident, which can be one of the following + values: -With at least the Maintainer role, you can enable -GitLab to create incident automatically whenever an alert is triggered: + - **{severity-critical}** Critical - S1 + - **{severity-high}** High - S2 + - **{severity-medium}** Medium - S3 + - **{severity-low}** Low - S4 + - **{severity-unknown}** Unknown -1. Navigate to **Settings > Monitor > Incidents** and expand **Incidents**. -1. Check the **Create an incident** checkbox. -1. To customize the incident, select an - [issue template](../../user/project/description_templates.md#create-an-issue-template), - to include in the [incident summary](#summary). -1. To send [an email notification](paging.md#email-notifications-for-alerts) to users - with the Developer role, select - **Send a separate email notification to Developers**. Email notifications are - also sent to users with the **Maintainer** and **Owner** roles. -1. Select **Save changes**. +- **Incident**: The title of the incident, which attempts to capture the + most meaningful information. +- **Status**: The status of the incident, which can be one of the following values: -### Create incidents via the PagerDuty webhook + - Triggered + - Acknowledged + - Resolved -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/119018) in GitLab 13.3. + In the Premium or Ultimate tier, this field is also linked to [on-call escalation](paging.md#escalating-an-incident) for the incident. -You can set up a webhook with PagerDuty to automatically create a GitLab incident -for each PagerDuty incident. This configuration requires you to make changes -in both PagerDuty and GitLab: +- **Date created**: How long ago the incident was created. This field uses the + standard GitLab pattern of `X time ago`. Hover over this value to see the exact date and time formatted according to your locale. +- **Assignees**: The user assigned to the incident. +- **Published**: Whether the incident is published to a [status page](status_page.md). -1. Sign in as a user with the Maintainer role. -1. Navigate to **Settings > Monitor > Incidents** and expand **Incidents**. -1. Select the **PagerDuty integration** tab: + -  +For an example of the incident list in action, visit this +[demo project](https://gitlab.com/gitlab-org/monitor/monitor-sandbox/-/incidents). -1. Activate the integration, and save the changes in GitLab. -1. Copy the value of **Webhook URL** for use in a later step. -1. Follow the steps described in the - [PagerDuty documentation](https://support.pagerduty.com/docs/webhooks) - to add the webhook URL to a PagerDuty webhook integration. +### Sort the incident list -To confirm the integration is successful, trigger a test incident from PagerDuty to -confirm that a GitLab incident is created from the incident. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229534) in GitLab 13.3: incidents are sorted by created date by default. -## Incident list +The incident list shows incidents sorted by incident created date, showing the newest first. -Whether you can view an incident depends on the [project visibility level](../../user/public_access.md) and -the incident's confidentiality status: +To sort by another column, or to change the sorting order, select the column. -- Public project and a non-confidential incident: You don't have to be a member of the project. -- Private project and non-confidential incident: You must have at least the Guest role for the project. -- Confidential incident (regardless of project visibility): You must have at least the Reporter. +The columns you can sort by: -The Incident list is available at **Monitor > Incidents** -in your project's sidebar. The list contains the following metrics: +- Severity +- Status +- Time to SLA +- Published - +### Filter the incidents list -- **State** - To filter incidents by their state, select **Open**, **Closed**, - or **All** above the incident list. -- **Search** - The Incident list supports a simple free text search, which filters - on the **Title** and **Incident** fields. -- **Severity** - Severity of a particular incident, which can be one of the following - values: - - **{severity-critical}** **Critical - S1** - - **{severity-high}** **High - S2** - - **{severity-medium}** **Medium - S3** - - **{severity-low}** **Low - S4** - - **{severity-unknown}** **Unknown** - - [Editing incident severity](#change-severity) on the incident details page was - [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229402) in GitLab 13.4. - -- **Incident** - The description of the incident, which attempts to capture the - most meaningful data. -- **Status** - The status of the incident, which can be one of the following values: - - **Triggered** - - **Acknowledged** - - **Resolved** - - In GitLab Premium, this field is also linked to [on-call escalation](paging.md#escalating-an-incident) for the incident. - -- **Date created** - How long ago the incident was created. This field uses the - standard GitLab pattern of `X time ago`, but is supported by a granular date/time - tooltip depending on the user's locale. -- **Assignees** - The user assigned to the incident. -- **Published** - Displays a green check mark (**{check-circle}**) if the incident is published - to a [Status Page](status_page.md). - -The Incident list displays incidents sorted by incident created date. -([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229534) in GitLab 13.3.) -To see if a column is sortable, point your mouse at the header. Sortable columns -display an arrow next to the column name. - -Incidents share the [Issues API](../../api/issues.md). - -NOTE: -For a live example of the incident list in action, visit this -[demo project](https://gitlab.com/gitlab-examples/ops/incident-setup/everyone/tanuki-inc/-/incidents). +To filter the incident list by author or assignee, enter these values in the search box. ## Incident details -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230847) in GitLab 13.4. - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Monitor > Incidents**. -1. Select an incident from the list. -When you take any of these actions on an incident, GitLab logs a system note and -displays it in the Incident Details view: - -- Updating the severity of an incident - ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42358) in GitLab 13.5.) - -For live examples of GitLab incidents, visit the `tanuki-inc` project's -[incident list page](https://gitlab.com/gitlab-examples/ops/incident-setup/everyone/tanuki-inc/-/incidents). -Select any incident in the list to display its incident details page. - ### Summary -The summary section for incidents provides both critical details about the -incident and the contents of the issue template (if applicable). The highlighted +The summary section for incidents provides critical details about the +incident and the contents of the issue template (if [selected](../metrics/alerts.md#trigger-actions-from-alerts)). The highlighted bar at the top of the incident displays from left to right: - The link to the original alert. - The alert start time. - The event count. -Beneath the highlight bar, GitLab displays a summary that includes the following fields: +Below the highlight bar, a summary includes the following fields: - Start time - Severity @@ -172,15 +85,18 @@ Beneath the highlight bar, GitLab displays a summary that includes the following - Monitoring tool The incident summary can be further customized using -[GitLab Flavored Markdown](../../user/markdown.md). If the corresponding alert -[provided Markdown for the incident](../metrics/alerts.md#trigger-actions-from-alerts), -then the Markdown is appended to the summary after the above alert fields. If an -[incident template](#create-incidents-automatically) is configured for the -project, then the template content is appended at the end. +[GitLab Flavored Markdown](../../user/markdown.md). + +If an incident is [created from an alert](../metrics/alerts.md#trigger-actions-from-alerts) +that provided Markdown for the incident, then the Markdown is appended to the summary. +If an incident template is configured for the project, then the template content is appended at the end. Comments are displayed in threads, but can be displayed chronologically [by toggling on the recent updates view](#recent-updates-view). +When you make changes to an incident, GitLab creates system notes and +displays them below the summary. + ### Metrics **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235994) in GitLab 13.8. @@ -198,6 +114,8 @@ If you add a link, you can access the original graph by selecting the hyperlink ### Alert details +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230847) in GitLab 13.4. + Incidents show the details of linked alerts in a separate tab. To populate this tab, the incident must have been created with a linked alert. Incidents created automatically from alerts have this @@ -216,144 +134,60 @@ Read more about [timeline events](incident_timeline_events.md) and how to enable > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227836) in GitLab 13.5. -To quickly see the latest updates on an incident, select -**{history}** **Turn recent updates view on** in the comment bar to display comments -un-threaded and ordered chronologically, newest to oldest: - - +To see the latest updates on an incident, select +**Turn recent updates view on** (**{history}**) on the comment bar. Comments display +un-threaded and chronologically, newest to oldest. ### Service Level Agreement countdown timer **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241663) in GitLab 13.5. You can enable the Service Level Agreement Countdown timer on incidents to track -the Service Level Agreements (SLAs) you hold with your customers. The timer is +the Service Level Agreements (SLA) you hold with your customers. The timer is automatically started when the incident is created, and shows the time remaining before the SLA period expires. The timer is also dynamically updated every 15 minutes so you do not have to refresh the page to see the time remaining. + +Prerequisites: + +- You must have at least the Maintainer role for the project. + To configure the timer: -1. Navigate to **Settings > Monitor**. -1. Scroll to **Incidents** and select **Expand**, then select the - **Incident settings** tab. +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Monitor**. +1. Expand the **Incidents** section, then select the **Incident settings** tab. 1. Select **Activate "time to SLA" countdown timer**. 1. Set a time limit in increments of 15 minutes. 1. Select **Save changes**. -After you enable the SLA countdown timer, the **Time to SLA** attribute is displayed -as a column in the Incidents List, and as a field on newly created Incidents. If +After you enable the SLA countdown timer, the **Time to SLA** column is available in the +incidents list and as a field on new incidents. If the incident isn't closed before the SLA period ends, GitLab adds a `missed::SLA` label to the incident. -## Assign incidents - -Assign incidents to users that are actively responding. Select **Edit** in the -right-hand side bar to select or clear assignees. - -## Associate a milestone - -Associate an incident to a milestone by selecting **Edit** next to the milestone feature in the right-hand side bar. - -## Change severity - -See [Incident List](#incident-list) for a full description of the severity levels available. -Select **Edit** in the right-hand side bar to change the severity of an incident. - -You can also change the severity using the [`/severity` quick action](../../user/project/quick_actions.md). - -## Add a to-do item - -Add a to-do for incidents that you want to track in your to-do list. Select -**Add a to do** at the top of the right-hand side bar to add a to-do item. - -## Change incident status - -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5716) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `incident_escalations`. Disabled by default. -> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/345769) in GitLab 14.10. -> - [Feature flag `incident_escalations`](https://gitlab.com/gitlab-org/gitlab/-/issues/345769) removed in GitLab 15.1. - -For users with the Developer role or higher, select **Edit** in the **Status** section of the -right-hand side bar of an incident, then select a status. **Triggered** is the default status for -new incidents. - -In projects with GitLab Premium, on-call responders can respond to [incident pages](paging.md#escalating-an-incident) -by changing the status. Setting the status to: - -- **Resolved** silences on-call pages for the alert. -- **Acknowledged** limits on-call pages based on the selected [escalation policy](#change-escalation-policy). -- **Triggered** from **Resolved** restarts the incident escalating from the beginning. - -In GitLab 15.1 and earlier, updating the status of an [incident created from an alert](alerts.md#create-an-incident-from-an-alert) -also updates the alert status. In [GitLab 15.2 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/356057), -the alert status is independent and does not update when the incident status changes. - -## Change escalation policy **(PREMIUM)** - -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5716) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `incident_escalations`. Disabled by default. -> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/345769) in GitLab 14.10. -> - [Feature flag `incident_escalations`](https://gitlab.com/gitlab-org/gitlab/-/issues/345769) removed in GitLab 15.1. - -For users with the Developer role or higher, select **Edit** in the **Escalation policy** section of -the right-hand side bar of an incident, then select a policy. By default, new incidents do not have -an escalation policy selected. - -Selecting an escalation policy updates the incident status to **Triggered** and begins -[escalating the incident to on-call responders](paging.md#escalating-an-incident). -Deselecting an escalation policy halts escalation. Refer to the [incident status](#change-incident-status) -to manage on-call paging once escalation has begun. - -In GitLab 15.1 and earlier, the escalation policy for [incidents created from alerts](alerts.md#create-an-incident-from-an-alert) -reflects the alert's escalation policy and cannot be changed. In [GitLab 15.2 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/356057), -the incident escalation policy is independent and can be changed. - -## Manage incidents from Slack - -Slack slash commands allow you to control GitLab and view GitLab content without leaving Slack. - -Learn how to [set up Slack slash commands](../../user/project/integrations/slack_slash_commands.md) -and how to [use the available slash commands](../../integration/slash_commands.md). - -## Associate Zoom calls - -GitLab enables you to [associate a Zoom meeting with an issue](../../user/project/issues/associate_zoom_meeting.md) -for synchronous communication during incident management. After starting a Zoom -call for an incident, you can associate the conference call with an issue. Your -team members can join the Zoom call without requesting a link. - -## Linked resources - -In an incident, you can add [links to various resources](linked_resources.md), -for example: - -- The incident Slack channel -- Zoom meeting -- Resources for resolving the incidents - -## Embed metrics in incidents - -You can embed metrics anywhere [GitLab Markdown](../../user/markdown.md) is -used, such as descriptions, comments on issues, and merge requests. Embedding -metrics helps you share them when discussing incidents or performance issues. -You can output the dashboard directly into any issue, merge request, epic, or -any other Markdown text field in GitLab by -[copying and pasting the link to the metrics dashboard](../metrics/embed.md#embedding-gitlab-managed-kubernetes-metrics). - -You can embed both [GitLab-hosted metrics](../metrics/embed.md) and -[Grafana metrics](../metrics/embed_grafana.md) in incidents and issue -templates. - -## Automatically close incidents via recovery alerts - -> - [Introduced for Prometheus Integrations](https://gitlab.com/gitlab-org/gitlab/-/issues/13401) in GitLab 12.5. -> - [Introduced for HTTP Integrations](https://gitlab.com/gitlab-org/gitlab/-/issues/13402) in GitLab 13.4. - -With at least the Maintainer role, you can enable - GitLab to close an incident automatically when a **Recovery Alert** is received: - -1. Navigate to **Settings > Monitor > Incidents** and expand **Incidents**. -1. Check the **Automatically close associated Incident** checkbox. -1. Select **Save changes**. - -When GitLab receives a **Recovery Alert**, it closes the associated incident. -This action is recorded as a system message on the incident indicating that it -was closed automatically by the GitLab Alert bot. +## Related topics + +- [Create an incident](manage_incidents.md#create-an-incident) +- [Create an incident automatically](../metrics/alerts.md#trigger-actions-from-alerts) + whenever an alert is triggered +- [View incidents list](manage_incidents.md#view-incidents-list) +- [Assign to a user](manage_incidents.md#assign-to-a-user) +- [Change incident severity](manage_incidents.md#change-severity) +- [Change incident status](manage_incidents.md#change-status) +- [Change escalation policy](manage_incidents.md#change-escalation-policy) +- [Embed metrics](manage_incidents.md#embed-metrics) +- [Close an incident](manage_incidents.md#close-an-incident) +- [Automatically close incidents via recovery alerts](manage_incidents.md#automatically-close-incidents-via-recovery-alerts) +- [Add a to-do item](../../user/todos.md#create-a-to-do-item) +- [Add labels](../../user/project/labels.md) +- [Assign a milestone](../../user/project/milestones/index.md) +- [Make an incident confidential](../../user/project/issues/confidential_issues.md) +- [Set a due date](../../user/project/issues/due_dates.md) +- [Toggle notifications](../../user/profile/notifications.md#edit-notification-settings-for-issues-merge-requests-and-epics) +- [Track spent time](../../user/project/time_tracking.md) +- [Add a Zoom meeting to an incident](../../user/project/issues/associate_zoom_meeting.md) the same + way you add it to an issue. +- [Linked resources in incidents](linked_resources.md) +- Create incidents and receive incident notifications [directly from Slack](slack.md). +- Use the [Issues API](../../api/issues.md) to interact with incidents. diff --git a/doc/operations/incident_management/integrations.md b/doc/operations/incident_management/integrations.md index a54556bd3a2..fdcfbe0cb2c 100644 --- a/doc/operations/incident_management/integrations.md +++ b/doc/operations/incident_management/integrations.md @@ -164,7 +164,7 @@ curl --request POST \ The authorization key can be used as the `password`. The `username` is left blank: - username: `<blank>` -- password: authorization_key +- password: `<authorization_key>` ```shell curl --request POST \ @@ -249,7 +249,7 @@ receives a payload with the end time of the alert set. For HTTP Endpoints without [custom mappings](#map-fields-in-custom-alerts), the expected field is `end_time`. With custom mappings, you can select the expected field. -You can also configure the associated [incident to be closed automatically](../incident_management/incidents.md#automatically-close-incidents-via-recovery-alerts) when the alert resolves. +You can also configure the associated [incident to be closed automatically](../incident_management/manage_incidents.md#automatically-close-incidents-via-recovery-alerts) when the alert resolves. ## Link to your Opsgenie Alerts **(PREMIUM)** diff --git a/doc/operations/incident_management/linked_resources.md b/doc/operations/incident_management/linked_resources.md index 40b2bbdc757..eb289076424 100644 --- a/doc/operations/incident_management/linked_resources.md +++ b/doc/operations/incident_management/linked_resources.md @@ -15,9 +15,9 @@ you can add linked resources to an incident issue. Resources you might want link to: -- Zoom meetings -- Slack channels or threads -- Google Docs +- The incident Slack channel +- Zoom meeting +- Resources for resolving the incidents ## View linked resources of an incident diff --git a/doc/operations/incident_management/manage_incidents.md b/doc/operations/incident_management/manage_incidents.md new file mode 100644 index 00000000000..75826c2c55e --- /dev/null +++ b/doc/operations/incident_management/manage_incidents.md @@ -0,0 +1,263 @@ +--- +stage: Monitor +group: Respond +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Manage incidents **(FREE)** + +This page collects instructions for all the things you can do with [incidents](incidents.md) or in relation to them. + +## Create an incident + +You can create an incident manually or automatically. + +### From the incidents list + +> - [Moved](https://gitlab.com/gitlab-org/monitor/monitor/-/issues/24) to GitLab Free in 13.3. +> - [Permission changed](https://gitlab.com/gitlab-org/gitlab/-/issues/336624) from Guest to Reporter in GitLab 14.5. +> - Automatic application of the `incident` label [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/290964) in GitLab 14.8. + +Prerequisites: + +- You must have at least the Reporter role for the project. + +To create an incident from the incidents list: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Monitor > Incidents**. +1. Select **Create incident**. + +### From the issues list + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230857) in GitLab 13.4. + +Prerequisites: + +- You must have at least the Reporter role for the project. + +To create an incident from the issues list: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Issues > List**, and select **New issue**. +1. From the **Type** dropdown list, select **Incident**. Only fields relevant to + incidents are available on the page. +1. Select **Create issue**. + +### From an alert + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217745) in GitLab 13.1. + +Create an incident issue when viewing an [alert](alerts.md). +The incident description is populated from the alert. + +Prerequisites: + +- You must have at least the Developer role for the project. + +To create an incident from an alert: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Monitor > Alerts**. +1. Select your desired alert. +1. Select **Create incident**. + +After an incident is created, to view it from the alert, select **View incident**. + +When you [close an incident](#close-an-incident) linked to an alert, GitLab +[changes the alert's status](alerts.md#change-an-alerts-status) to **Resolved**. +You are then credited with the alert's status change. + +### Automatically, when an alert is triggered **(ULTIMATE)** + +In the project settings, you can turn on [creating an incident automatically](../metrics/alerts.md#trigger-actions-from-alerts) +whenever an alert is triggered. + +### Using the PagerDuty webhook + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/119018) in GitLab 13.3. +> - [PagerDuty V3 Webhook](https://support.pagerduty.com/docs/webhooks) support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/383029) in GitLab 15.7. + +You can set up a webhook with PagerDuty to automatically create a GitLab incident +for each PagerDuty incident. This configuration requires you to make changes +in both PagerDuty and GitLab. + +Prerequisites: + +- You must have at least the Maintainer role for the project. + +To set up a webhook with PagerDuty: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Monitor** +1. Expand **Incidents**. +1. Select the **PagerDuty integration** tab. +1. Turn on the **Active** toggle. +1. Select **Save integration**. +1. Copy the value of **Webhook URL** for use in a later step. +1. To add the webhook URL to a PagerDuty webhook integration, follow the steps described in the [PagerDuty documentation](https://support.pagerduty.com/docs/webhooks#manage-v3-webhook-subscriptions). + +To confirm the integration is successful, trigger a test incident from PagerDuty to +check if a GitLab incident is created from the incident. + +## View incidents list + +To view the [incidents list](incidents.md#incidents-list): + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Monitor > Incidents**. + +To view an incident's [details page](incidents.md#incident-details), select it from the list. + +### Who can view an incident + +Whether you can view an incident depends on the [project visibility level](../../user/public_access.md) and +the incident's confidentiality status: + +- Public project and a non-confidential incident: You don't have to be a member of the project. +- Private project and non-confidential incident: You must have at least the Guest role for the project. +- Confidential incident (regardless of project visibility): You must have at least the Reporter role for the project. + +## Assign to a user + +Assign incidents to users that are actively responding. + +Prerequisites: + +- You must have at least the Reporter role for the project. + +To assign a user: + +1. In an incident, on the right sidebar, next to **Assignees**, select **Edit**. +1. From the dropdown list, select one or [multiple users](../../user/project/issues/multiple_assignees_for_issues.md) to add as **assignees**. +1. Select any area outside the dropdown list. + +## Change severity + +> Editing severity on incident details page was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229402) in GitLab 13.4. + +See [incident list](incidents.md#incidents-list) for a full description of the severity levels available. + +Prerequisites: + +- You must have at least the Reporter role for the project. + +To change an incident's severity: + +1. In an incident, on the right sidebar, next to **Severity**, select **Edit**. +1. From the dropdown list, select the new severity. + +You can also change the severity using the `/severity` [quick action](../../user/project/quick_actions.md). + +## Change status + +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5716) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `incident_escalations`. Disabled by default. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/345769) in GitLab 14.10. +> - [Feature flag `incident_escalations`](https://gitlab.com/gitlab-org/gitlab/-/issues/345769) removed in GitLab 15.1. + +Prerequisites: + +- You must have at least the Developer role for the project. + +To change the status of an incident: + +1. In an incident, on the right sidebar, next to **Status**, select **Edit**. +1. From the dropdown list, select the new severity. + +**Triggered** is the default status for new incidents. + +### As an on-call responder **(PREMIUM)** + +On-call responders can respond to [incident pages](paging.md#escalating-an-incident) +by changing the status. + +Changing the status has the following effects: + +- To **Acknowledged**: limits on-call pages based on the project's [escalation policy](escalation_policies.md). +- To **Resolved**: silences all on-call pages for the incident. +- From **Resolved** to **Triggered**: restarts the incident escalating. + +In GitLab 15.1 and earlier, changing the status of an [incident created from an alert](#from-an-alert) +also changes the alert status. In [GitLab 15.2 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/356057), +the alert status is independent and does not change when the incident status changes. + +## Change escalation policy **(PREMIUM)** + +Prerequisites: + +- You must have at least the Developer role for the project. + +To change the escalation policy of an incident: + +1. In an incident, on the right sidebar, next to **Escalation policy**, select **Edit**. +1. From the dropdown list, select the escalation policy. + +By default, new incidents do not have an escalation policy selected. + +Selecting an escalation policy [changes the incident status](#change-status) to **Triggered** and begins +[escalating the incident to on-call responders](paging.md#escalating-an-incident). + +In GitLab 15.1 and earlier, the escalation policy for [incidents created from alerts](#from-an-alert) +reflects the alert's escalation policy and cannot be changed. In [GitLab 15.2 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/356057), +the incident escalation policy is independent and can be changed. + +## Embed metrics + +You can embed metrics anywhere [GitLab Flavored Markdown](../../user/markdown.md) is +used, like descriptions or comments. Embedding +metrics helps you share them when discussing incidents or performance issues. + +To embed metrics in a Markdown text box in GitLab, +[paste the link to the dashboard](../metrics/embed.md#embedding-gitlab-managed-kubernetes-metrics). + +You can embed both [GitLab-hosted metrics](../metrics/embed.md) (deprecated) and +[Grafana metrics](../metrics/embed_grafana.md) in incidents and issue +templates. + +## Close an incident + +Prerequisites: + +- You must have at least the Reporter role for the project. + +To close an incident, in the top right, select **Close incident**. + +When you close an incident that is linked to an [alert](alerts.md), +the linked alert's status changes to **Resolved**. +You are then credited with the alert's status change. + +### Automatically close incidents via recovery alerts + +> [Introduced for HTTP integrations](https://gitlab.com/gitlab-org/gitlab/-/issues/13402) in GitLab 13.4. + +Turn on closing an incident automatically when GitLab receives a recovery alert +from a HTTP or Prometheus webhook. + +Prerequisites: + +- You must have at least the Maintainer role for the project. + +To configure the setting: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Monitor**. +1. Expand the **Incidents** section. +1. Select the **Automatically close associated incident** checkbox. +1. Select **Save changes**. + +When GitLab receives a recovery alert, it closes the associated incident. +This action is recorded as a system note on the incident indicating that it +was closed automatically by the GitLab Alert bot. + +## Other actions + +Because incidents in GitLab are built on top of [issues](../../user/project/issues/index.md), +they have the following actions in common: + +- [Add a to-do item](../../user/todos.md#create-a-to-do-item) +- [Add labels](../../user/project/labels.md#assign-and-unassign-labels) +- [Assign a milestone](../../user/project/milestones/index.md#assign-a-milestone-to-an-issue-or-merge-request) +- [Make an incident confidential](../../user/project/issues/confidential_issues.md) +- [Set a due date](../../user/project/issues/due_dates.md) +- [Toggle notifications](../../user/profile/notifications.md#edit-notification-settings-for-issues-merge-requests-and-epics) +- [Track time spent](../../user/project/time_tracking.md) diff --git a/doc/operations/incident_management/paging.md b/doc/operations/incident_management/paging.md index dcb17048d35..70c67977b73 100644 --- a/doc/operations/incident_management/paging.md +++ b/doc/operations/incident_management/paging.md @@ -14,12 +14,11 @@ notifications using the methods described on this page. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216326) in GitLab 13.1. -Responders can be paged via Slack using the -[Slack Notifications Service](../../user/project/integrations/slack.md), which you -can configure for new alerts and new incidents. After configuring, responders -receive a **single** page via Slack. To set up Slack notifications on your mobile -device, make sure to enable notifications for the Slack app on your phone so -you never miss a page. +The GitLab for Slack app can be used to receive important incident notifications. + +When [the GitLab for Slack app is configured](slack.md), incident responders are notified in Slack +every time a new incident is declared. To ensure you don't miss any important incident notifications +on your mobile device, enable notifications for Slack on your phone. ## Email notifications for alerts @@ -55,12 +54,14 @@ or stop alert escalations by [updating the alert's status](alerts.md#change-an-a > - [Feature flag `incident_escalations`](https://gitlab.com/gitlab-org/gitlab/-/issues/345769) removed in GitLab 15.1. For incidents, paging on-call responders is optional for each individual incident. -To begin escalating the incident, [set the incident's escalation policy](incidents.md#change-escalation-policy). + +To begin escalating the incident, [set the incident's escalation policy](manage_incidents.md#change-escalation-policy). + For each escalation rule, the designated on-call responders receive one email when -the rule fires. You can respond to a page or stop incident escalations by -[updating the incident's status](incidents.md#change-incident-status) or, if applicable, -[unsetting the incident's escalation policy](incidents.md#change-escalation-policy). +the rule fires. Respond to a page or stop incident escalations by +[changing the incident's status](manage_incidents.md#change-status) or +changing the incident's escalation policy back to **No escalation policy**. -In GitLab 15.1 and earlier, [incidents created from alerts](alerts.md#create-an-incident-from-an-alert) +In GitLab 15.1 and earlier, [incidents created from alerts](manage_incidents.md#from-an-alert) do not support independent escalation. In [GitLab 15.2 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/356057), all incidents can be escalated independently. diff --git a/doc/operations/incident_management/slack.md b/doc/operations/incident_management/slack.md new file mode 100644 index 00000000000..1ab1391ea2a --- /dev/null +++ b/doc/operations/incident_management/slack.md @@ -0,0 +1,117 @@ +--- +stage: Monitor +group: Respond +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Incident management for Slack **(FREE SAAS)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344856) in GitLab 15.7 [with a flag](../../administration/feature_flags.md) named `incident_declare_slash_command`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `incident_declare_slash_command`. +On GitLab.com, this feature is not available. +The feature is not ready for production use. + +Many teams receive alerts and collaborate in real time during incidents in Slack. +Use the GitLab for Slack app to: + +- Create GitLab incidents from Slack. +- Receive incident notifications. +<!-- The below content is commented out until these features are implemented in https://gitlab.com/groups/gitlab-org/-/epics/8545 --> +<!-- - Send important updates between Slack and GitLab incidents. --> + +Incident management for Slack is only available for GitLab.com. Some of the functionality +described might be available for +[the self-managed Slack app](../../user/project/integrations/slack_slash_commands.md). + +To stay up to date, follow [epic 1211](https://gitlab.com/groups/gitlab-org/-/epics/1211). + +## Manage an incident from Slack + +Prerequisites: + +1. Install the [GitLab for Slack app](../../user/project/integrations/gitlab_slack_application.md). + This way, you can use slash commands in Slack to create and update GitLab incidents. +1. Enable [Slack notifications](../../user/project/integrations/slack.md). Be sure to enable + notifications for `Issue` events, and to define a Slack channel to receive the relevant notifications. +1. Authorize GitLab to take actions on behalf of your Slack user. + Each user must do this before they can use any of the incident slash commands. + + To start the authorization flow, try executing a non-incident [Slack slash command](../../integration/slash_commands.md), + like `/gitlab <project-alias> issue show <id>`. + The `<project-alias>` you select must be a project that has the GitLab for Slack app set up. + For more context, visit [issue 377548](https://gitlab.com/gitlab-org/gitlab/-/issues/377548). + +<!-- The below content is commented out until these features are implemented in https://gitlab.com/groups/gitlab-org/-/epics/8545 --> +<!-- +To manage incidents, use the following slash commands in Slack: + +| Command | Description | +| ---------------------------------- | ------------------------------------------- | +| `/gitlab incident declare` | Creates an incident in GitLab. | +| `/gitlab incident comment <text>` | Adds a comment on a GitLab incident. | +| `/gitlab incident timeline <text>` | Adds a timeline event to a GitLab incident. | +| `/gitlab incident close` | Closes an incident in GitLab. | +--> + +After the GitLab for Slack app is configured, you can also use any of the existing [Slack slash commands](../../user/project/integrations/slack_slash_commands.md). + +## Declare an incident + +To declare a GitLab incident from Slack: + +1. In Slack, in any channel or DM, enter the `/gitlab incident declare` slash command. +1. From the modal, select the relevant incident details, including: + + - The incident title and description. + - The project where the incident should be created. + - The severity of the incident. + + If there is an existing [incident template](../metrics/alerts.md#trigger-actions-from-alerts) for your + project, that template is automatically applied to the description text box. The template is applied + only if the description text box is empty. + + You can also include GitLab [quick actions](../../user/project/quick_actions.md) in the description text box. + For example, entering `/link https://example.slack.com/archives/123456789 Dedicated Slack channel` + adds a dedicated Slack channel to the incident you create. For a complete list of + quick actions for incidents, see [Use GitLab quick actions](#use-gitlab-quick-actions). +1. Optional. Add a link to an existing Zoom meeting. +1. Select **Create**. + +If the incident is successfully created, Slack shows a confirmation notification. + +### Use GitLab quick actions + +Use [quick actions](../../user/project/quick_actions.md) in the description text box when creating +a GitLab incident from Slack. The following quick actions might be most relevant to you: + +| Command | Description | +| ------------------------ | ----------------------------------------- | +| `/assign @user1 @user2` | Adds an assignee to the GitLab incident. | +| `/label ~label1 ~label2` | Adds labels to the GitLab incident. | +| `/link <URL> <text>` | Adds a link to a dedicated Slack channel, runbook, or any relevant resource to the `Related resources` section of an incident. | +| `/zoom <URL>` | Adds a Zoom meeting link to the incident. | + +<!-- The below content is commented out until these features are implemented in https://gitlab.com/groups/gitlab-org/-/epics/8545 --> +<!-- ### Comment on a GitLab incident + +To comment on a GitLab incident from Slack, enter the `/gitlab incident comment <text>` slash command. +Slack shows a prompt asking you to confirm which incident you'd like to post your comment to. + +### Add a timeline event + +To add a [timeline event](incident_timeline_events.md) to a GitLab incident from Slack, enter the +`/gitlab incident timeline <text>` slash command. +Slack shows a prompt asking you to confirm which incident you'd like to add your timeline event to. + +### Close an incident + +To close a GitLab incident from Slack when it is resolved, enter the `/gitlab incident close` +slash command. +Slack shows a prompt asking you to confirm which incident you'd like to close. --> + +## Send GitLab incident notifications to Slack + +If you have [enabled notifications](#manage-an-incident-from-slack) for issues, you should receive +notifications to the selected Slack channel every time an incident is opened, closed, or updated. diff --git a/doc/operations/metrics/alerts.md b/doc/operations/metrics/alerts.md index ffd304c8897..4b6bc91cc73 100644 --- a/doc/operations/metrics/alerts.md +++ b/doc/operations/metrics/alerts.md @@ -6,8 +6,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Set up alerts for Prometheus metrics **(FREE)** -> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/42640) to GitLab Free in 12.10. - After [configuring metrics for your CI/CD environment](index.md), you can set up alerting for Prometheus metrics, and [trigger actions from alerts](#trigger-actions-from-alerts) to notify @@ -15,31 +13,48 @@ your team when environment performance falls outside of the boundaries you set. ## Prometheus cluster integrations -Alerts are not currently supported for [Prometheus cluster integrations](../../user/clusters/integrations.md). +Alerts are not supported for [Prometheus cluster integrations](../../user/clusters/integrations.md). ## Trigger actions from alerts **(ULTIMATE)** -Alerts can be used to trigger actions, like opening an issue automatically -(disabled by default since `13.1`). To configure the actions: +> - Introduced in GitLab 13.1: incidents are not created automatically by default . +> - Mapping common severity values from the alert payload ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50871) in GitLab 13.9. + +Turn on creating [incidents](../incident_management/incidents.md) automatically whenever an alert is triggered. + +Prerequisites: + +- You must have at least the Maintainer role for the project. -1. Navigate to your project's **Settings > Monitor > Alerts**. -1. Enable the option to create issues. -1. Choose the [issue template](../../user/project/description_templates.md) to create the issue from. -1. Optionally, select whether to send an email notification to the developers of the project. +To configure the actions: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Monitor**. +1. Expand the **Alerts** section, then select the **Alert settings** tab. +1. Select the **Create an incident** checkbox. +1. Optional. To customize the incident, from the **Incident template**, select a template to be + appended to the [incident summary](../incident_management/incidents.md#summary). + If the dropdown list is empty, + [create an issue template](../../user/project/description_templates.md#create-an-issue-template) first. +1. Optional. To send [an email notification](../incident_management/paging.md#email-notifications-for-alerts), select the + **Send a single email notification to Owners and Maintainers for new alerts** checkbox. 1. Select **Save changes**. -After enabling, GitLab automatically opens an issue when an alert is triggered containing -values extracted from the [`alerts` field in webhook payload](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config): +### Fields in automatically created incidents + +Incidents [created automatically from an alert](#trigger-actions-from-alerts) are filled with +values extracted from the `alerts` field in the +[webhook payload](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config): -- Issue author: `GitLab Alert Bot` -- Issue title: Extracted from the alert payload fields `annotations/title`, `annotations/summary`, or `labels/alertname`. -- Issue description: Extracted from alert payload field `annotations/description`. +- Incident author: `GitLab Alert Bot` +- Incident title: Extracted from the alert payload fields `annotations/title`, `annotations/summary`, or `labels/alertname`. +- Incident description: Extracted from alert payload field `annotations/description`. - Alert `Summary`: A list of properties from the alert's payload. - `starts_at`: Alert start time from the payload's `startsAt` field - `full_query`: Alert query extracted from the payload's `generatorURL` field - Optional list of attached annotations extracted from `annotations/*` - Alert [GLFM](../../user/markdown.md): GitLab Flavored Markdown from the payload's `annotations/gitlab_incident_markdown` field. -- Alert Severity ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50871) in GitLab version 13.9): +- Alert severity: Extracted from the alert payload field `labels/severity`. Maps case-insensitive value to [Alert's severity](../incident_management/alerts.md#alert-severity): - **Critical**: `critical`, `s1`, `p1`, `emergency`, `fatal`, or any value not in this list @@ -48,23 +63,17 @@ values extracted from the [`alerts` field in webhook payload](https://prometheus - **Low**: `low`, `s4`, `p4`, `warn`, `warning` - **Info**: `info`, `s5`, `p5`, `debug`, `information`, `notice` -To further customize the issue, you can add labels, mentions, or any other supported +To further customize the incident, you can add labels, mentions, or any other supported [quick action](../../user/project/quick_actions.md) in the selected issue template, which applies to all incidents. To limit quick actions or other information to only specific types of alerts, use the `annotations/gitlab_incident_markdown` field. -Since [version 12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/63373), GitLab tags each incident issue with the `incident` label automatically. If the label -does not yet exist, it is also created automatically. - -If the metric exceeds the threshold of the alert for over 5 minutes, GitLab sends -an email to all Maintainers and Owners of the project. +does not yet exist, it's created automatically. ### Recovery alerts -> [From GitLab 12.5](https://gitlab.com/gitlab-org/gitlab/-/issues/13401), when GitLab receives a recovery alert, it automatically closes the associated issue. - -The alert in GitLab will be automatically resolved when Prometheus +The alert in GitLab is automatically resolved when Prometheus sends a payload with the field `status` set to `resolved`. -You can also configure the associated [incident to be closed automatically](../incident_management/incidents.md#automatically-close-incidents-via-recovery-alerts) when the alert resolves. +You can also configure the associated [incident to be closed automatically](../incident_management/manage_incidents.md#automatically-close-incidents-via-recovery-alerts) when the alert resolves. diff --git a/doc/operations/metrics/dashboards/panel_types.md b/doc/operations/metrics/dashboards/panel_types.md index 95e8bd82475..177a55fb85b 100644 --- a/doc/operations/metrics/dashboards/panel_types.md +++ b/doc/operations/metrics/dashboards/panel_types.md @@ -40,8 +40,8 @@ Note the following properties: | Property | Type | Required | Description | | ------ | ------ | ------ | ------ | -| type | string | no | Type of panel to be rendered. Optional for area panel types | -| query_range | string | required | For area panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | +| `type` | string | no | Type of panel to be rendered. Optional for area panel types | +| `query_range` | string | required | For area panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) |  @@ -82,8 +82,8 @@ Note the following properties: | Property | Type | Required | Description | | ------ | ------ | ------ | ------ | -| type | string | required | Must be `anomaly-chart` for anomaly panel types | -| query_range | yes | required | For anomaly panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) in every metric. | +| `type` | string | required | Must be `anomaly-chart` for anomaly panel types | +| `query_range` | yes | required | For anomaly panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) in every metric. |  @@ -138,8 +138,8 @@ Note the following properties: | Property | Type | Required | Description | | ------ | ------ | ------ | ------ | -| type | string | yes | Type of panel to be rendered. For column panel types, set to `column` | -| query_range | yes | yes | For column panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | +| `type` | string | yes | Type of panel to be rendered. For column panel types, set to `column` | +| `query_range` | yes | yes | For column panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) |  @@ -201,10 +201,10 @@ panel_groups: Note the following properties: | Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| type | string | yes | Type of panel to be rendered. For single stat panel types, set to `single-stat` | -| field | string | no | Panels display the value of a metric. For a panel to display the value of a label instead, put the name of the label in this key. | -| query | string | yes | For single stat panel types, you must use an [instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries) | +| ------- | ------ | ------ | ------ | +| `type` | string | yes | Type of panel to be rendered. For single stat panel types, set to `single-stat` | +| `field` | string | no | Panels display the value of a metric. For a panel to display the value of a label instead, put the name of the label in this key. | +| `query` | string | yes | For single stat panel types, you must use an [instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries). |  @@ -263,15 +263,15 @@ panel_groups: Note the following properties: -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| type | string | yes | Type of panel to be rendered. For gauge panel types, set to `gauge`. | -| min_value | number | no, defaults to `0` | The minimum value of the gauge chart axis. If either of `min_value` or `max_value` are not set, they both get their default values. | -| max_value | number | no, defaults to `100` | The maximum value of the gauge chart axis. If either of `min_value` or `max_value` are not set, they both get their default values. | -| split | number | no, defaults to `10` | The amount of split segments on the gauge chart axis. | -| thresholds | object | no | Thresholds configuration for the gauge chart axis. | -| format | string | no, defaults to `engineering` | Unit format used. See the [full list of units](yaml_number_format.md). | -| query | string | yes | For gauge panel types, you must use an [instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries). | +| Property | Type | Required | Description | +| ------------ | ------ | ------ | ------ | +| `type` | string | yes | Type of panel to be rendered. For gauge panel types, set to `gauge`. | +| `min_value` | number | no, defaults to `0` | The minimum value of the gauge chart axis. If either of `min_value` or `max_value` are not set, they both get their default values. | +| `max_value` | number | no, defaults to `100` | The maximum value of the gauge chart axis. If either of `min_value` or `max_value` are not set, they both get their default values. | +| `split` | number | no, defaults to `10` | The amount of split segments on the gauge chart axis. | +| `thresholds` | object | no | Thresholds configuration for the gauge chart axis. | +| `format` | string | no, defaults to `engineering` | Unit format used. See the [full list of units](yaml_number_format.md). | +| `query` | string | yes | For gauge panel types, you must use an [instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries). | ### Thresholds properties @@ -306,12 +306,12 @@ Note the following properties: | Property | Type | Required | Description | | ------ | ------ | ------ | ------ | -| type | string | yes | Type of panel to be rendered. For heatmap panel types, set to `heatmap` | -| query_range | yes | yes | For area panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | +| `type` | string | yes | Type of panel to be rendered. For heatmap panel types, set to `heatmap` | +| `query_range` | yes | yes | For area panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) |  WARNING: When a query returns too many data points, the heatmap data bucket dimensions tend downwards to 0, making the chart's data invisible, as shown in the image below. To fix this problem, limit the amount of data returned by changing the time range filter on the metrics dashboard UI, or adding the **step** property to your dashboard's YAML file. - + diff --git a/doc/operations/metrics/dashboards/variables.md b/doc/operations/metrics/dashboards/variables.md index 5cd4dcdfa17..2881c084115 100644 --- a/doc/operations/metrics/dashboards/variables.md +++ b/doc/operations/metrics/dashboards/variables.md @@ -38,7 +38,7 @@ must be lowercase. The supported variables are: - `ci_environment_name` - `__range` -### environment_filter +### `environment_filter` `environment_filter` is automatically expanded to `container_name!="POD",environment="ENVIRONMENT_NAME"` where `ENVIRONMENT_NAME` is the name of the current environment. @@ -46,7 +46,7 @@ where `ENVIRONMENT_NAME` is the name of the current environment. For example, a Prometheus query like `container_memory_usage_bytes{ {{environment_filter}} }` becomes `container_memory_usage_bytes{ container_name!="POD",environment="production" }`. -### __range +### `__range` The `__range` variable is useful in Prometheus [range vector selectors](https://prometheus.io/docs/prometheus/latest/querying/basics/#range-vector-selectors). diff --git a/doc/operations/metrics/dashboards/yaml.md b/doc/operations/metrics/dashboards/yaml.md index 399a8ecb615..68eddc6f087 100644 --- a/doc/operations/metrics/dashboards/yaml.md +++ b/doc/operations/metrics/dashboards/yaml.md @@ -156,7 +156,7 @@ To confirm your dashboard definition contains valid YAML syntax: Files with valid syntax display **Metrics Dashboard YAML definition is valid**, and files with invalid syntax display **Metrics Dashboard YAML definition is invalid**. - + When **Metrics Dashboard YAML definition is invalid** at least one of the following messages is displayed: diff --git a/doc/operations/tracing.md b/doc/operations/tracing.md deleted file mode 100644 index b3c0763bbbc..00000000000 --- a/doc/operations/tracing.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -stage: Monitor -group: Respond -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -remove_date: '2022-11-01' -redirect_to: 'index.md' ---- - -# Tracing (removed) **(FREE SELF)** - -This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346540) in GitLab 14.7 -and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/346540) in GitLab 15.2. -We are working on an alternative to replace tracing. -To learn more, visit the [Observability direction page](https://about.gitlab.com/direction/monitor/observability/). diff --git a/doc/policy/maintenance.md b/doc/policy/maintenance.md index 2bfc7f0243a..fcb6e5c1b20 100644 --- a/doc/policy/maintenance.md +++ b/doc/policy/maintenance.md @@ -36,7 +36,7 @@ The following table describes the version types and their release cadence: | Version type | Description | Cadence | |:-------------|:------------|:--------| -| Major | For significant changes, or when any backward-incompatible changes are introduced to the public API. | Yearly. The next major release is GitLab 15.0 on May 22, 2022. GitLab schedules major releases on May 22 each year, by default. | +| Major | For significant changes, or when any backward-incompatible changes are introduced to the public API. | Yearly. The next major release is GitLab 16.0 on May 22, 2023. GitLab schedules major releases on May 22 each year, by default. | | Minor | For when new backward-compatible functionality is introduced to the public API, a minor feature is introduced, or when a set of smaller features is rolled out. | Monthly on the 22nd. | | Patch | For backward-compatible bug fixes that fix incorrect behavior. See [Patch releases](#patch-releases). | As needed. | diff --git a/doc/raketasks/backup_gitlab.md b/doc/raketasks/backup_gitlab.md index 47696fc1f99..aea61e36037 100644 --- a/doc/raketasks/backup_gitlab.md +++ b/doc/raketasks/backup_gitlab.md @@ -36,7 +36,7 @@ The backup command requires [additional parameters](backup_restore.md#back-up-an your installation is using PgBouncer, for either performance reasons or when using it with a Patroni cluster. WARNING: -The backup command doesn't verify if another backup is already running, as described in +Before GitLab 15.5.0, the backup command doesn't verify if another backup is already running, as described in [issue 362593](https://gitlab.com/gitlab-org/gitlab/-/issues/362593). We strongly recommend you make sure that all backups are complete before starting a new one. diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 2ac79a913f3..796cb71321a 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -85,6 +85,9 @@ In the following cases, consider using file system data transfer or snapshots as - Your GitLab instance has a lot of forked projects and the regular backup task duplicates the Git data for all of them. - Your GitLab instance has a problem and using the regular backup and import Rake tasks isn't possible. +WARNING: +Gitaly Cluster [does not support snapshot backups](../administration/gitaly/index.md#snapshot-backup-and-recovery-limitations). + When considering using file system data transfer or snapshots: - Don't use these methods to migrate from one operating system to another. The operating systems of the source and destination should be as similar as possible. For example, @@ -887,7 +890,7 @@ Truncate the filenames in the `uploads` table: - `current_filename`: a filename that is currently more than 246 characters long. - `new_filename`: a filename that has been truncated to 246 characters maximum. - - `new_path`: new path considering the new_filename (truncated). + - `new_path`: new path considering the `new_filename` (truncated). Once you validate the batch results, you must change the batch size (`row_id`) using the following sequence of numbers (10000 to 20000). Repeat this process until you reach the last record in the `uploads` table. @@ -981,3 +984,98 @@ Truncate the filenames on the filesystem. You must manually rename the files in #### Re-run the backup task After following all the previous steps, re-run the backup task. + +### Restoring database backup fails when `pg_stat_statements` was previously enabled + +The GitLab backup of the PostgreSQL database includes all SQL statements required to enable extensions that were +previously enabled in the database. + +The `pg_stat_statements` extension can only be enabled or disabled by a PostgreSQL user with `superuser` role. +As the restore process uses a database user with limited permissions, it can't execute the following SQL statements: + +```sql +DROP EXTENSION IF EXISTS pg_stat_statements; +CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA public; +``` + +When trying to restore the backup in a PostgreSQL instance that doesn't have the `pg_stats_statements` extension, +the following error message is displayed: + +```plaintext +ERROR: permission denied to create extension "pg_stat_statements" +HINT: Must be superuser to create this extension. +ERROR: extension "pg_stat_statements" does not exist +``` + +When trying to restore in an instance that has the `pg_stats_statements` extension enabled, the cleaning up step +fails with an error message similar to the following: + +```plaintext +rake aborted! +ActiveRecord::StatementInvalid: PG::InsufficientPrivilege: ERROR: must be owner of view pg_stat_statements +/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:42:in `block (4 levels) in <top (required)>' +/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:41:in `each' +/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:41:in `block (3 levels) in <top (required)>' +/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/backup.rake:71:in `block (3 levels) in <top (required)>' +/opt/gitlab/embedded/bin/bundle:23:in `load' +/opt/gitlab/embedded/bin/bundle:23:in `<main>' +Caused by: +PG::InsufficientPrivilege: ERROR: must be owner of view pg_stat_statements +/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:42:in `block (4 levels) in <top (required)>' +/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:41:in `each' +/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:41:in `block (3 levels) in <top (required)>' +/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/backup.rake:71:in `block (3 levels) in <top (required)>' +/opt/gitlab/embedded/bin/bundle:23:in `load' +/opt/gitlab/embedded/bin/bundle:23:in `<main>' +Tasks: TOP => gitlab:db:drop_tables +(See full trace by running task with --trace) +``` + +#### Prevent the dump file to include `pg_stat_statements` + +To prevent the inclusion of the extension in the PostgreSQL dump file that is part of the backup bundle, +enable the extension in any schema except the `public` schema: + +```sql +CREATE SCHEMA adm; +CREATE EXTENSION pg_stat_statements SCHEMA adm; +``` + +If the extension was previously enabled in the `public` schema, move it to a new one: + +```sql +CREATE SCHEMA adm; +ALTER EXTENSION pg_stat_statements SET SCHEMA adm; +``` + +To query the `pg_stat_statements` data after changing the schema, prefix the view name with the new schema: + +```sql +SELECT * FROM adm.pg_stat_statements limit 0; +``` + +To make it compatible with third-party monitoring solutions that expect it to be enabled in the `public` schema, +you need to include it in the `search_path`: + +```sql +set search_path to public,adm; +``` + +#### Fix an existing dump file to remove references to `pg_stat_statements` + +To fix an existing backup file, do the following changes: + +1. Extract from the backup the following file: `db/database.sql.gz`. +1. Decompress the file or use an editor that is capable of handling it compressed. +1. Remove the following lines, or similar ones: + + ```sql + CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA public; + ``` + + ```sql + COMMENT ON EXTENSION pg_stat_statements IS 'track planning and execution statistics of all SQL statements executed'; + ``` + +1. Save the changes and recompress the file. +1. Update the backup file with the modified `db/database.sql.gz`. diff --git a/doc/raketasks/cleanup.md b/doc/raketasks/cleanup.md index a67fec26a9b..60c81b26c05 100644 --- a/doc/raketasks/cleanup.md +++ b/doc/raketasks/cleanup.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Clean up **(FREE SELF)** +# Clean up Rake tasks **(FREE SELF)** GitLab provides Rake tasks for cleaning up GitLab instances. @@ -35,7 +35,8 @@ You can also specify the project with `PROJECT_ID` instead of `PROJECT_PATH`. For example: ```shell -$ sudo gitlab-rake gitlab:cleanup:orphan_lfs_file_references PROJECT_PATH="gitlab-org/gitlab-foss" +$ sudo gitlab-rake gitlab:cleanup:orphan_lfs_file_references PROJECT_ID="13083" + I, [2019-12-13T16:35:31.764962 #82356] INFO -- : Looking for orphan LFS files for project GitLab Org / GitLab Foss I, [2019-12-13T16:35:31.923659 #82356] INFO -- : Removed invalid references: 12 ``` diff --git a/doc/raketasks/generate_sample_prometheus_data.md b/doc/raketasks/generate_sample_prometheus_data.md index ec7f54a41c2..7445065c77e 100644 --- a/doc/raketasks/generate_sample_prometheus_data.md +++ b/doc/raketasks/generate_sample_prometheus_data.md @@ -4,7 +4,7 @@ group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Generate sample Prometheus data **(FREE SELF)** +# Generate sample Prometheus data Rake task **(FREE SELF)** This command runs Prometheus queries for each of the metrics of a specific environment for a series of time intervals to now: diff --git a/doc/raketasks/index.md b/doc/raketasks/index.md index 4a157688a41..0d5a826b0a1 100644 --- a/doc/raketasks/index.md +++ b/doc/raketasks/index.md @@ -7,15 +7,13 @@ comments: false # Rake tasks **(FREE SELF)** -GitLab provides [Rake](https://ruby.github.io/rake/) tasks to assist you with -common administration and operational processes. +GitLab provides [Rake](https://ruby.github.io/rake/) tasks to assist you with common administration and operational +processes. You can perform GitLab Rake tasks by using: -- `gitlab-rake <raketask>` for [Omnibus GitLab](https://docs.gitlab.com/omnibus/index.html) - installations. -- `bundle exec rake <raketask>` for [source](../install/installation.md) - installations. +- `gitlab-rake <raketask>` for [Omnibus GitLab](https://docs.gitlab.com/omnibus/index.html) installations. +- `bundle exec rake <raketask>` for [source](../install/installation.md) installations. ## Available Rake tasks @@ -39,7 +37,7 @@ The following Rake tasks are available for use with GitLab: | [Praefect Rake tasks](../administration/raketasks/praefect.md) | [Praefect](../administration/gitaly/praefect.md)-related tasks. | | [Project import/export](../administration/raketasks/project_import_export.md) | Prepare for [project exports and imports](../user/project/settings/import_export.md). | | [Sample Prometheus data](generate_sample_prometheus_data.md) | Generate sample Prometheus data. | -| [Sidekiq job migration](sidekiq_job_migration.md) | Migrate Sidekiq jobs scheduled for future dates to a new queue. | +| [Sidekiq job migration](../administration/sidekiq/sidekiq_job_migration.md) | Migrate Sidekiq jobs scheduled for future dates to a new queue. | | [SMTP maintenance](../administration/raketasks/smtp.md) | SMTP-related tasks. | | [SPDX license list import](spdx.md) | Import a local copy of the [SPDX license list](https://spdx.org/licenses/) for matching [License Compliance policies](../user/compliance/license_compliance/index.md). | | [Repository storage](../administration/raketasks/storage.md) | List and migrate existing projects and attachments from legacy storage to hashed storage. | diff --git a/doc/raketasks/list_repos.md b/doc/raketasks/list_repos.md index 57d24a2942d..258d41aa190 100644 --- a/doc/raketasks/list_repos.md +++ b/doc/raketasks/list_repos.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Listing repository directories **(FREE SELF)** +# List repository directories Rake task **(FREE SELF)** You can print a list of all Git repositories on disk managed by GitLab. diff --git a/doc/raketasks/migrate_snippets.md b/doc/raketasks/migrate_snippets.md index 62d2866e499..e51edc5c133 100644 --- a/doc/raketasks/migrate_snippets.md +++ b/doc/raketasks/migrate_snippets.md @@ -4,7 +4,7 @@ group: Editor info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Migration to versioned snippets **(FREE SELF)** +# Migrate to versioned snippets Rake tasks **(FREE SELF)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215861) in GitLab 13.0. diff --git a/doc/raketasks/sidekiq_job_migration.md b/doc/raketasks/sidekiq_job_migration.md deleted file mode 100644 index 45a0cbaa267..00000000000 --- a/doc/raketasks/sidekiq_job_migration.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../administration/sidekiq/sidekiq_job_migration.md' -remove_date: '2022-11-11' ---- - -This document was moved to [another location](../administration/sidekiq/sidekiq_job_migration.md). - -<!-- This redirect file can be deleted after <2022-11-11>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/raketasks/spdx.md b/doc/raketasks/spdx.md index 31f860f45de..608139fa404 100644 --- a/doc/raketasks/spdx.md +++ b/doc/raketasks/spdx.md @@ -4,7 +4,7 @@ group: Composition Analysis info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# SPDX license list import **(ULTIMATE SELF)** +# SPDX license list import Rake task **(ULTIMATE SELF)** GitLab provides a Rake task for uploading a fresh copy of the [SPDX license list](https://spdx.org/licenses/) to a GitLab instance. This list is needed for matching the names of [License Compliance policies](../user/compliance/license_compliance/index.md). diff --git a/doc/raketasks/user_management.md b/doc/raketasks/user_management.md index 84d943e2c21..7756774e432 100644 --- a/doc/raketasks/user_management.md +++ b/doc/raketasks/user_management.md @@ -4,9 +4,10 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# User management **(FREE SELF)** +# User management Rake tasks **(FREE SELF)** -GitLab provides Rake tasks for user management. +GitLab provides Rake tasks for managing users. Administrators can also use the Admin Area to +[manage users](../user/admin_area/index.md#administering-users). ## Add user as a developer to all projects diff --git a/doc/raketasks/web_hooks.md b/doc/raketasks/web_hooks.md index b4d01d21dc7..3bd9d7e2d2e 100644 --- a/doc/raketasks/web_hooks.md +++ b/doc/raketasks/web_hooks.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Webhooks administration **(FREE SELF)** +# Webhooks administration Rake tasks **(FREE SELF)** GitLab provides Rake tasks for webhooks management. diff --git a/doc/raketasks/x509_signatures.md b/doc/raketasks/x509_signatures.md index 3404f6ae9cf..364264ae204 100644 --- a/doc/raketasks/x509_signatures.md +++ b/doc/raketasks/x509_signatures.md @@ -4,7 +4,7 @@ group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# X.509 signatures **(FREE SELF)** +# X.509 signatures Rake task **(FREE SELF)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/122159) in GitLab 12.10. diff --git a/doc/security/crime_vulnerability.md b/doc/security/crime_vulnerability.md index 463ccb7b629..e5d8d858df2 100644 --- a/doc/security/crime_vulnerability.md +++ b/doc/security/crime_vulnerability.md @@ -53,7 +53,7 @@ SPDY support earlier than version 4 is advertised. From the report above it is important to note that Nessus is only checking if TLS advertises the SPDY protocol earlier than version 4. It does not perform an attack nor does it check if compression is enabled. The Nessus scanner alone -cannot tell that SPDY's compression is disabled and not subject to the CRIME +cannot tell that SPDY compression is disabled and not subject to the CRIME vulnerability. ## References diff --git a/doc/security/password_storage.md b/doc/security/password_storage.md index bd514de6e2c..67ef161e634 100644 --- a/doc/security/password_storage.md +++ b/doc/security/password_storage.md @@ -21,7 +21,7 @@ GitLab uses the [Devise](https://github.com/heartcombo/devise) authentication library to hash user passwords. Created password hashes have these attributes: - **Hashing**: - - **BCrypt**: By default, the [`bcrypt`](https://en.wikipedia.org/wiki/Bcrypt) hashing + - **bcrypt**: By default, the [`bcrypt`](https://en.wikipedia.org/wiki/Bcrypt) hashing function is used to generate the hash of the provided password. This cryptographic hashing function is strong and industry-standard. - **PBKDF2+SHA512**: PBKDF2+SHA512 is supported: @@ -29,7 +29,7 @@ library to hash user passwords. Created password hashes have these attributes: - In GitLab 15.6 and later when [FIPS mode](../development/fips_compliance.md) is enabled (feature flags are not required). - **Stretching**: Password hashes are [stretched](https://en.wikipedia.org/wiki/Key_stretching) to harden against brute-force attacks. By default, GitLab uses a stretching - factor of 10 for BCrypt and 20,000 for PBKDF2 + SHA512. + factor of 10 for bcrypt and 20,000 for PBKDF2 + SHA512. - **Salting**: A [cryptographic salt](https://en.wikipedia.org/wiki/Salt_(cryptography)) is added to each password to harden against pre-computed hash and dictionary attacks. To increase security, each salt is randomly generated for each diff --git a/doc/security/rate_limits.md b/doc/security/rate_limits.md index 20a81ed0c30..929609cd4a4 100644 --- a/doc/security/rate_limits.md +++ b/doc/security/rate_limits.md @@ -141,6 +141,19 @@ This is to mitigate the risk of misuses, such as mass discovery of usernames in The **rate limit** is 20 calls per minute per IP address. +### Project Jobs API endpoint + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104912) in GitLab 15.7 [with a flag](../administration/feature_flags.md) named `ci_enforce_rate_limits_jobs_api`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, +ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `ci_enforce_rate_limits_jobs_api`. +The feature is not ready for production use. + +There is a rate limit for the endpoint `project/:id/jobs`, which is enforced to reduce timeouts when retrieving jobs. + +The **rate limit** is 600 calls per minute per signed-in user. + ## Troubleshooting ### Rack Attack is denylisting the load balancer diff --git a/doc/security/reset_user_password.md b/doc/security/reset_user_password.md index 248737fc908..38c52912d5c 100644 --- a/doc/security/reset_user_password.md +++ b/doc/security/reset_user_password.md @@ -7,7 +7,7 @@ type: howto # Reset a user's password **(FREE SELF)** -You can reset user passwords by using a Rake task, a Rails console, or the +You can reset user passwords by using the UI, a Rake task, a Rails console, or the [Users API](../api/users.md#user-modification). ## Prerequisites @@ -16,6 +16,18 @@ To reset a user password, you must be an administrator of a self-managed GitLab The user's new password must meet all [password requirements](../user/profile/user_passwords.md#password-requirements). +## Use the UI + +To reset a user's password in the UI: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Overview > Users**. +1. For the user whose password you want to update, select **Edit** (**{pencil-square}**). +1. In the **Password** area, type a password and password confirmation. +1. Select **Save changes**. + +A confirmation is displayed. + ## Use a Rake task > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52347) in GitLab 13.9. diff --git a/doc/security/user_email_confirmation.md b/doc/security/user_email_confirmation.md index ffc537c8f10..c3f19c92f91 100644 --- a/doc/security/user_email_confirmation.md +++ b/doc/security/user_email_confirmation.md @@ -13,7 +13,7 @@ they confirm their email address. 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > General** (`/admin/application_settings/general`). -1. Expand the **Sign-up restrictions** section and look for the **Send confirmation email on sign-up** option. +1. Expand the **Sign-up restrictions** section and look for the **Email confirmation settings** options. ## Confirmation token expiry diff --git a/doc/subscriptions/gitlab_com/index.md b/doc/subscriptions/gitlab_com/index.md index 6f1bf0df044..552e9c46c4a 100644 --- a/doc/subscriptions/gitlab_com/index.md +++ b/doc/subscriptions/gitlab_com/index.md @@ -78,7 +78,7 @@ A top-level group can be [changed](../../user/group/manage.md#change-a-groups-pa Every user is included in seat usage, with the following exceptions: - Users who are pending approval. -- Members with the Guest role on an Ultimate subscription. +- Members with the [Guest role on an Ultimate subscription](#free-guest-users). - GitLab-created service accounts: - [Ghost User](../../user/profile/account/delete_account.md#associated-records). - Bots such as: @@ -102,9 +102,16 @@ To view a list of seats being used: 1. On the left sidebar, select **Settings > Usage Quotas**. 1. On the **Seats** tab, view usage information. -The seat usage listing is updated live, but the usage statistics on the billing page are updated -only once per day. For this reason there can be a minor difference between the seat usage listing -and the billing page. +The data in seat usage listing, **Seats in use**, and **Seats in subscription** are updated live. +The counts for **Max seats used** and **Seats owed** are updated once per day. + +To view your subscription information and a summary of seat counts: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > Billing**. + +The usage statistics are updated once per day, which may cause +a difference between the information in the **Usage Quotas** page and the **Billing page**. ### Search seat usage @@ -153,6 +160,16 @@ For example, if you purchase a subscription for 10 users: Seats owed = 12 - 10 (Maximum users - users in subscription) +### Free Guest users **(ULTIMATE)** + +In the **Ultimate** tier, users who are assigned the Guest role do not consume a seat. +The user must not be assigned any other role, anywhere in the instance. + +- If your project is private or internal, a user with the Guest role has + [a set of permissions](../../user/permissions.md#project-members-permissions). +- If your project is public, all users, including those with the Guest role + can access your project. + ### Add users to your subscription Your subscription cost is based on the maximum number of seats you use during the billing period. @@ -214,6 +231,37 @@ amounts at which the alert displays. | 100-999 | 8% of seats have been used. | | 1000+ | 5% of seats have been used | +## Change the linked account + +To change the GitLab.com account linked to your Customers Portal account: + +1. Log in to the + [Customers Portal](https://customers.gitlab.com/customers/sign_in). +1. In a separate browser tab, go to [GitLab SaaS](https://gitlab.com/users/sign_in) and ensure you + are not logged in. +1. On the Customers Portal page, select **My account > Account details**. +1. Under **Your GitLab.com account**, select **Change linked account**. +1. Log in to the [GitLab SaaS](https://gitlab.com/users/sign_in) account you want to link to the Customers Portal + account. + +## Change the linked namespace + +To change the namespace linked to a subscription: + +1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in) with a + [linked](#change-the-linked-account) GitLab SaaS account. +1. Navigate to the **Manage Purchases** page. +1. Select **Change linked namespace**. +1. Select the desired group from the **This subscription is for** dropdown. For a group to appear here, you must have the Owner role for that group. +1. Select **Proceed to checkout**. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For a demo, see [Linking GitLab Subscription to the Namespace](https://youtu.be/qAq8pyFP-a0). + +Subscription charges are calculated based on the total number of users in a group, including its subgroups and nested projects. If the [total number of users](#view-seat-usage) exceeds the number of seats in your subscription, your account is charged for the additional users and you need to pay for the overage before you can change the linked namespace. + +Only one namespace can be linked to a subscription. + ## Upgrade your GitLab SaaS subscription tier To upgrade your [GitLab tier](https://about.gitlab.com/pricing/): @@ -275,10 +323,17 @@ For details on upgrading your subscription tier, see ### Automatic subscription renewal -When you enable automatic renewal, the subscription automatically renews on the -expiration date without a gap in available service. An invoice is -generated for the renewal and available for viewing or download on the -[View invoices](https://customers.gitlab.com/receipts) page. +When a subscription is set to auto-renew, it renews automatically on the +expiration date without a gap in available service. Subscriptions purchased through Customers Portal or GitLab.com are set to auto-renew by default. The number of seats is adjusted to fit the [number of billable users in your group](#view-seat-usage) at the time of renewal. You can view and download your renewal invoice on the +[View invoices](https://customers.gitlab.com/receipts) page. If your account has a [saved credit card](../index.md#change-your-payment-method), the card is charged for the invoice amount. If we are unable to process a payment or the auto-renewal fails for any other reason, you have 14 days to renew your subscription. After that, your access is downgraded. + +#### Email notifications + +15 days before a subscription automatically renews, an email is sent with information about the renewal. + +- If your credit card is expired, the email tells you how to update it. +- If you have any outstanding overages, the email tells you to contact our Sales team. +- If there are no issues, the email specifies the names and quantity of the products being renewed. The email also includes the total amount you owe. If your usage increases or decreases before renewal, this amount can change. #### Enable or disable automatic subscription renewal @@ -335,7 +390,7 @@ locked. Projects can only be unlocked by purchasing more storage subscription un Prerequisite: -- You must have at least the Owner role. +- You must have the Owner role. You can purchase a storage subscription for your personal or group namespace. @@ -408,7 +463,7 @@ and for communicating directly with the relevant GitLab team members. If your credit card is declined when purchasing a GitLab subscription, possible reasons include: -- The credit card details provided are incorrect. +- The credit card details provided are incorrect. The most common cause for this is an incomplete or dummy address. - The credit card account has insufficient funds. - You are using a virtual credit card and it has insufficient funds, or has expired. - The transaction exceeds the credit limit. @@ -417,7 +472,7 @@ If your credit card is declined when purchasing a GitLab subscription, possible Check with your financial institution to confirm if any of these reasons apply. If they don't apply, contact [GitLab Support](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293). -### Unable to link subcription to namespace +### Unable to link subscription to namespace If you cannot link a subscription to your namespace, ensure that you have the Owner role for that namespace. diff --git a/doc/subscriptions/gitlab_dedicated/index.md b/doc/subscriptions/gitlab_dedicated/index.md index c503c501eeb..b471d1d971f 100644 --- a/doc/subscriptions/gitlab_dedicated/index.md +++ b/doc/subscriptions/gitlab_dedicated/index.md @@ -7,23 +7,25 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Dedicated NOTE: -GitLab Dedicated is currently in limited availability. [Contact us](#contact-us) if you are interested. +GitLab Dedicated is currently in limited availability. You can learn more and join the waitlist [on our website](https://about.gitlab.com/single-tenant-saas). GitLab Dedicated is a fully isolated, single-tenant SaaS service that is: - Hosted and managed by GitLab, Inc. -- Deployed in a region of choice on AWS. +- Deployed on AWS in a cloud region of your choice (see the [regions that are not supported](#aws-regions-not-supported)). -GitLab Dedicated enables you to offload the operational overhead of managing the DevOps Platform. It offers a high level of tenant isolation and deployment customization, ideal for enterprises in highly-regulated industries. By deploying your GitLab instance onto separate Cloud Infrastructure from other tenants, GitLab Dedicated helps you better meet your security and compliance requirements. +GitLab Dedicated removes the overhead of platform management to increase your operational efficiency, reduce risk, and enhance the speed and agility of your organization. Each GitLab Dedicated instance is highly available with disaster recovery and deployed into the cloud region of your choice. GitLab teams fully manage the maintenance and operations of each isolated instance, so customers can access our latest product improvements while meeting the most complex compliance standards. + +It's the offering of choice for enterprises and organizations in highly regulated industries that have complex regulatory, compliance, and data residency requirements. ## Available features -- Authentication: Support for instance-level [SAML OmniAuth](../../integration/saml.md) functionality. GitLab Dedicated acts as the service provider, and you must provide the necessary [configuration](../../integration/saml.md#general-setup) in order for GitLab to communicate with your IdP. This is provided during onboarding. - - SAML [request signing](../../integration/saml.md#request-signing-optional), [group sync](../../user/group/saml_sso/group_sync.md#configure-saml-group-sync), and [SAML groups](../../integration/saml.md#saml-groups) are supported. +- Authentication: Support for instance-level [SAML OmniAuth](../../integration/saml.md) functionality. GitLab Dedicated acts as the service provider, and you must provide the necessary [configuration](../../integration/saml.md#configure-saml-support-in-gitlab) in order for GitLab to communicate with your IdP. This is provided during onboarding. + - SAML [request signing](../../integration/saml.md#sign-saml-authentication-requests-optional), [group sync](../../user/group/saml_sso/group_sync.md#configure-saml-group-sync), and [SAML groups](../../integration/saml.md#configure-users-based-on-saml-group-membership) are supported. - Networking: - - Public connectivity with support for IP Allowlists. During onboarding, you can optionally specify a list of IP addresses that can access your Dedicated instance. Subsequently, when an IP not on the allowlist tries to access your instance the connection will be refused. + - Public connectivity with support for IP Allowlists. During onboarding, you can optionally specify a list of IP addresses that can access your GitLab Dedicated instance. Subsequently, when an IP not on the allowlist tries to access your instance the connection is refused. - Optional. Private connectivity via [AWS PrivateLink](https://aws.amazon.com/privatelink/). - You can specify an AWS IAM Principal and preferred Availability Zones during onboarding to enable this functionality. Both Ingress and Egress Private Links are supported. When connecting to an internal service running in your VPC over https via PrivateLink, Dedicated supports the ability to use a private SSL certificate, which can be provided during onboarding. + You can specify an AWS IAM Principal and preferred Availability Zones during onboarding to enable this functionality. Both Ingress and Egress PrivateLinks are supported. When connecting to an internal service running in your VPC over HTTPS via PrivateLink, GitLab Dedicated supports the ability to use a private SSL certificate, which can be provided during onboarding. - Upgrades: - Monthly upgrades tracking one release behind the latest (n-1), with the latest security release. - Out of band security patches provided for high severity releases. @@ -53,7 +55,7 @@ The following features will not be supported: - Mattermost - Server-side Git hooks -### Dedicated service features +### GitLab Dedicated service features The following operational features are not available: @@ -74,55 +76,14 @@ The following AWS regions are not available: - Cape Town (`af-south-1`) - Milan (`eu-south-1`) - Paris (`eu-west-3`) -- GovCloud +- Zurich (`eu-central-2`) +- GovCloud (US-East) (`us-gov-east-1`) +- GovCloud (US-West) (`us-gov-west-1`) ## Planned features -Learn more about the planned improvements to Dedicated on the public [direction page](https://about.gitlab.com/direction/saas-platforms/dedicated/). - -## Contact us - -Fill in the following form to contact us and learn more about this offering. - -<!-- markdownlint-disable --> - -<!-- NOTE: The following form only shows when the site is served under HTTPS, - so it will not appear when developing locally or in a review app. - See https://gitlab.com/gitlab-com/marketing/marketing-operations/-/issues/6238#note_923358643 ---> - -<script src="https://page.gitlab.com/js/forms2/js/forms2.min.js"></script> -<form id="mktoForm_3226"></form> -<script>MktoForms2.loadForm("https://page.gitlab.com", "194-VVC-221", 3226);</script> -<style> - #mktoForm_3226 { - font-size: .875rem !important; - } - .mktoLabel { - margin-top: 1rem !important; - padding-bottom: .5rem !important; - font-weight: 600; - } - .mktoHtmlText, - #LblPhone, - .mktoTextField, - #commentCapture, - .mktoField, - .mktoButtonRow button { - width: 20rem !important; - } - .mktoHtmlText { - font-size: .875rem; - } - .mktoButtonRow { - margin: 1em 0; - } - .mktoButtonRow span { - margin-left: 0 !important; - } - .mktoButtonRow button { - margin: 1em 0 1.5em !important; - } -</style> - -<!-- markdownlint-enable --> +Learn more about the planned improvements to GitLab Dedicated on the public [direction page](https://about.gitlab.com/direction/saas-platforms/dedicated/). + +## Learn more about GitLab Dedicated and join our waitlist + +As we scale this new offering, we are making GitLab Dedicated available by inviting customers to learn more and join our waitlist [on our website](https://about.gitlab.com/single-tenant-saas). diff --git a/doc/subscriptions/img/add-license.png b/doc/subscriptions/img/add-license.png Binary files differnew file mode 100644 index 00000000000..157f0921796 --- /dev/null +++ b/doc/subscriptions/img/add-license.png diff --git a/doc/subscriptions/index.md b/doc/subscriptions/index.md index 47c1f730746..98ce40c9e9b 100644 --- a/doc/subscriptions/index.md +++ b/doc/subscriptions/index.md @@ -73,11 +73,13 @@ With the [Customers Portal](https://customers.gitlab.com/) you can: - [Change account owner information](#change-account-owner-information) - [Change your company details](#change-your-company-details) - [Change your payment method](#change-your-payment-method) -- [Change the linked account](#change-the-linked-account) -- [Change the namespace the subscription is linked to](#change-the-linked-namespace) +- [Change the linked account](gitlab_com/index.md#change-the-linked-account) +- [Change the namespace the subscription is linked to](gitlab_com/index.md#change-the-linked-namespace) - [Change customers portal account password](#change-customers-portal-account-password) -The Customers Portal is available only to customers who purchased their subscription from GitLab. If you made your purchase through a partner or reseller, you must contact them directly for assistance with your subscription. +The Customers Portal is available only to customers who purchased their +subscription from GitLab. If you made your purchase through a partner or +reseller, you must contact them directly for assistance with your subscription. ### Change account owner information @@ -132,40 +134,6 @@ method as the default: 1. **Edit** the selected payment method and check the **Make default payment method** checkbox. 1. Select **Save Changes**. -### Change the linked account - -To change the GitLab.com account linked to your Customers Portal account: - -1. Log in to the - [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. In a separate browser tab, go to [GitLab SaaS](https://gitlab.com) and ensure you - are not logged in. -1. On the Customers Portal page, select **My account > Account details**. -1. Under **Your GitLab.com account**, select **Change linked account**. If the account is not yet linked, select **Link my GitLab.com account**. -1. Log in to the [GitLab SaaS](https://gitlab.com) account you want to link to the Customers Portal - account. - -### Change the linked namespace - -To change the namespace linked to a subscription: - -1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in) with a - [linked](#change-the-linked-account) GitLab SaaS account. -1. Navigate to the **Manage Purchases** page. -1. Select **Change linked namespace**. -1. Select the desired group from the **Select user or group** dropdown list. For a group to appear - here, you must have the Owner role for that group. -1. Select **Proceed to checkout**. - -If the group you want to link does not appear in the dropdown list, check: - -- You have [linked your Customers Portal account with your GitLab.com account](#change-the-linked-account). -- That the linked account is a member of the group you want to select, and you are assigned the Owner role. - -Subscription charges are calculated based on the total number of users in a group, including its subgroups and nested projects. If the [total number of users](gitlab_com/index.md#view-seat-usage) exceeds the number of seats in your subscription, your account is charged for the additional users and you need to pay for the overage before you can change the linked namespace. - -Only one namespace can be linked to a subscription. - ### Change Customers Portal account password To change the password for this customers portal account: @@ -187,9 +155,6 @@ For qualifying open source projects, the [GitLab for Open Source Program](https: #### Meeting GitLab for Open Source Program requirements -NOTE: -GitLab for Open Source Program benefits apply to an entire GitLab namespace. To qualify for the GitLab for Open Source Program, all projects in an applicant's namespace must meet program requirements. Applicants submit materials related to one project in the applying namespace, and the open source program team uses that project to verify eligibility of the entire namespace. - To meet GitLab for Open Source Program requirements, first add an OSI-approved open source license to all projects in your namespace. To add a license to a project: @@ -197,8 +162,13 @@ To add a license to a project: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the overview page, select **Add LICENSE**. If the license you want is not available as a license template, manually copy the entire, unaltered [text of your chosen license](https://opensource.org/licenses/alphabetical) into the `LICENSE` file. Note that GitLab defaults to **All rights reserved** if users do not perform this action. + + Applicants must add the correct license to each project in their respective groups or namespaces. When you're sure you're using OSI-approved licenses for your projects, you can take your screenshots. +NOTE: +GitLab for Open Source Program benefits apply to an entire GitLab namespace. To qualify for the GitLab for Open Source Program, all projects in an applicant's namespace must meet program requirements. Applicants submit materials related to one project in the applying namespace, and the open source program team uses that project to verify eligibility of the entire namespace. + #### Verification for Open Source Program Next, take screenshots of your project to confirm that project's eligibility. You must upload three screenshots: diff --git a/doc/subscriptions/self_managed/index.md b/doc/subscriptions/self_managed/index.md index c4110fe4638..93cf5afad63 100644 --- a/doc/subscriptions/self_managed/index.md +++ b/doc/subscriptions/self_managed/index.md @@ -63,7 +63,7 @@ billable user, with the following exceptions: [blocked users](../../user/admin_area/moderate_users.md#block-a-user) don't count as billable users in the current subscription. When they are either deactivated or blocked they release a _billable user_ seat. However, they may count toward overages in the subscribed seat count. - Users who are [pending approval](../../user/admin_area/moderate_users.md#users-pending-approval). -- Members with the Guest role on an Ultimate subscription. +- Members with the [Guest role on an Ultimate subscription](#free-guest-users). - Users without project or group memberships on an Ultimate subscription. - GitLab-created service accounts: - [Ghost User](../../user/profile/account/delete_account.md#associated-records). @@ -97,6 +97,21 @@ If you add more users to your GitLab instance than you are licensed for, payment If you do not add these users during the renewal process, your license key will not work. +#### Free Guest users **(ULTIMATE)** + +In the **Ultimate** tier, users who are assigned the Guest role do not consume a seat. +The user must not be assigned any other role, anywhere in the instance. + +- If your project is private or internal, a user with the Guest role has + [a set of permissions](../../user/permissions.md#project-members-permissions). +- If your project is public, all users, including those with the Guest role + can access your project. + +NOTE: +If a user creates a project, they are assigned the Maintainer or Owner role. +To prevent a user from creating projects, as an administrator, you can mark the user +as [external](../../user/admin_area/external_users.md). + ### Tips for managing users and subscription seats Managing the number of users against the number of subscription seats can be a challenge: diff --git a/doc/topics/authentication/index.md b/doc/topics/authentication/index.md index 12ff5cb50b8..763e16bc31d 100644 --- a/doc/topics/authentication/index.md +++ b/doc/topics/authentication/index.md @@ -12,7 +12,7 @@ This page gathers all the resources for the topic **Authentication** within GitL - [SSH](../../user/ssh.md) - [Two-factor authentication](../../user/profile/account/two_factor_authentication.md) -- [Why do I keep getting signed out?](../../user/profile/index.md#why-do-i-keep-getting-signed-out) +- [Why do you keep getting signed out?](../../user/profile/index.md#why-do-you-keep-getting-signed-out) - **Articles:** - [Support for Universal 2nd Factor Authentication - YubiKeys](https://about.gitlab.com/blog/2016/06/22/gitlab-adds-support-for-u2f/) - [Security Webcast with Yubico](https://about.gitlab.com/blog/2016/08/31/gitlab-and-yubico-security-webcast/) diff --git a/doc/topics/autodevops/cicd_variables.md b/doc/topics/autodevops/cicd_variables.md new file mode 100644 index 00000000000..db2b052784d --- /dev/null +++ b/doc/topics/autodevops/cicd_variables.md @@ -0,0 +1,331 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# CI/CD variables + +Use CI/CD variables to set up the Auto DevOps domain, provide a custom +Helm chart, or scale your application. + +## Build and deployment variables + +Use these variables to customize and deploy your build. + +| **CI/CD variable** | **Description** | +|-----------------------------------------|------------------------------------| +| `ADDITIONAL_HOSTS` | Fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. | +| `<ENVIRONMENT>_ADDITIONAL_HOSTS` | For a specific environment, the fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. This takes precedence over `ADDITIONAL_HOSTS`. | +| `AUTO_BUILD_IMAGE_VERSION` | Customize the image version used for the `build` job. See [list of versions](https://gitlab.com/gitlab-org/cluster-integration/auto-build-image/-/releases). | +| `AUTO_DEPLOY_IMAGE_VERSION` | Customize the image version used for Kubernetes deployment jobs. See [list of versions](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/releases). | +| `AUTO_DEVOPS_ATOMIC_RELEASE` | As of GitLab 13.0, Auto DevOps uses [`--atomic`](https://v2.helm.sh/docs/helm/#options-43) for Helm deployments by default. Set this variable to `false` to disable the use of `--atomic` | +| `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` | Set to `false` to use Herokuish instead of Cloud Native Buildpacks with Auto Build. [More details](stages.md#auto-build-using-cloud-native-buildpacks). | +| `AUTO_DEVOPS_BUILD_IMAGE_CNB_BUILDER` | The builder used when building with Cloud Native Buildpacks. The default builder is `heroku/buildpacks:18`. [More details](stages.md#auto-build-using-cloud-native-buildpacks). | +| `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` | Extra arguments to be passed to the `docker build` command. Using quotes doesn't prevent word splitting. [More details](customize.md#passing-arguments-to-docker-build). | +| `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` | A [comma-separated list of CI/CD variable names](customize.md#forward-cicd-variables-to-the-build-environment) to be forwarded to the build environment (the buildpack builder or `docker build`). | +| `AUTO_DEVOPS_BUILD_IMAGE_CNB_PORT` | In GitLab 15.0 and later, port exposed by the generated Docker image. Set to `false` to prevent exposing any ports. Defaults to `5000`. | +| `AUTO_DEVOPS_CHART` | Helm Chart used to deploy your apps. Defaults to the one [provided by GitLab](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/tree/master/assets/auto-deploy-app). | +| `AUTO_DEVOPS_CHART_REPOSITORY` | Helm Chart repository used to search for charts. Defaults to `https://charts.gitlab.io`. | +| `AUTO_DEVOPS_CHART_REPOSITORY_NAME` | Used to set the name of the Helm repository. Defaults to `gitlab`. | +| `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME` | Used to set a username to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD`. | +| `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD` | Used to set a password to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME`. | +| `AUTO_DEVOPS_CHART_REPOSITORY_PASS_CREDENTIALS` | From GitLab 14.2, set to a non-empty value to enable forwarding of the Helm repository credentials to the chart server when the chart artifacts are on a different host than repository. | +| `AUTO_DEVOPS_COMMON_NAME` | From GitLab 15.5, set to a valid domain name to customize the common name used for the TLS certificate. Defaults to `le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN`. Set to `false` to not set this alternative host on the Ingress. | +| `AUTO_DEVOPS_DEPLOY_DEBUG` | From GitLab 13.1, if this variable is present, Helm outputs debug logs. | +| `AUTO_DEVOPS_ALLOW_TO_FORCE_DEPLOY_V<N>` | From [auto-deploy-image](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image) v1.0.0, if this variable is present, a new major version of chart is forcibly deployed. For more information, see [Ignore warnings and continue deploying](upgrading_auto_deploy_dependencies.md#ignore-warnings-and-continue-deploying). | +| `BUILDPACK_URL` | A full Buildpack URL. [Must point to a URL supported by Pack or Herokuish](customize.md#custom-buildpacks). | +| `CANARY_ENABLED` | Used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments). | +| `BUILDPACK_VOLUMES` | Specify one or more [Buildpack volumes to mount](stages.md#mount-volumes-into-the-build-container). Use a pipe `|` as list separator. | +| `CANARY_PRODUCTION_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md) in the production environment. Takes precedence over `CANARY_REPLICAS`. Defaults to 1. | +| `CANARY_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md). Defaults to 1. | +| `CI_APPLICATION_REPOSITORY` | The repository of container image being built or deployed, `$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG`. For more details, read [Custom container image](customize.md#custom-container-image). | +| `CI_APPLICATION_TAG` | The tag of the container image being built or deployed, `$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG`. For more details, read [Custom container image](customize.md#custom-container-image). | +| `DAST_AUTO_DEPLOY_IMAGE_VERSION` | Customize the image version used for DAST deployments on the default branch. Should usually be the same as `AUTO_DEPLOY_IMAGE_VERSION`. See [list of versions](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/releases). | +| `DOCKERFILE_PATH` | From GitLab 13.2, allows overriding the [default Dockerfile path for the build stage](customize.md#custom-dockerfile) | +| `HELM_RELEASE_NAME` | From GitLab 12.1, allows the `helm` release name to be overridden. Can be used to assign unique release names when deploying multiple projects to a single namespace. | +| `HELM_UPGRADE_VALUES_FILE` | From GitLab 12.6, allows the `helm upgrade` values file to be overridden. Defaults to `.gitlab/auto-deploy-values.yaml`. | +| `HELM_UPGRADE_EXTRA_ARGS` | Allows extra options in `helm upgrade` commands when deploying the application. Using quotes doesn't prevent word splitting. | +| `INCREMENTAL_ROLLOUT_MODE` | If present, can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment. Set to `manual` for manual deployment jobs or `timed` for automatic rollout deployments with a 5 minute delay each one. | +| `K8S_SECRET_*` | Any variable prefixed with [`K8S_SECRET_`](#configure-application-secret-variables) is made available by Auto DevOps as environment variables to the deployed application. | +| `KUBE_CONTEXT` | From GitLab 14.5, can be used to select a context to use from `KUBECONFIG`. When `KUBE_CONTEXT` is blank, the default context in `KUBECONFIG` (if any) is used. A context must be selected when used [with the agent for Kubernetes](../../user/clusters/agent/ci_cd_workflow.md). | +| `KUBE_INGRESS_BASE_DOMAIN` | Can be used to set a domain per cluster. See [cluster domains](../../user/project/clusters/gitlab_managed_clusters.md#base-domain) for more information. | +| `KUBE_NAMESPACE` | The namespace used for deployments. When using certificate-based clusters, [this value should not be overwritten directly](../../user/project/clusters/deploy_to_cluster.md#custom-namespace). | +| `KUBECONFIG` | The kubeconfig to use for deployments. User-provided values take priority over GitLab-provided values. | +| `PRODUCTION_REPLICAS` | Number of replicas to deploy in the production environment. Takes precedence over `REPLICAS` and defaults to 1. For zero downtime upgrades, set to 2 or greater. | +| `REPLICAS` | Number of replicas to deploy. Defaults to 1. Change this variable instead of [modifying](customize.md#customize-values-for-helm-chart) `replicaCount`. | +| `ROLLOUT_RESOURCE_TYPE` | Allows specification of the resource type being deployed when using a custom Helm chart. Default value is `deployment`. | +| `ROLLOUT_STATUS_DISABLED` | From GitLab 12.0, used to disable rollout status check because it does not support all resource types, for example, `cronjob`. | +| `STAGING_ENABLED` | Used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). | +| `TRACE` | Set to any value to make Helm commands produce verbose output. You can use this setting to help diagnose Auto DevOps deployment problems. | + +## Database variables + +Use these variables to integrate CI/CD with PostgreSQL databases. + +| **CI/CD variable** | **Description** | +|-----------------------------------------|------------------------------------| +| `DB_INITIALIZE` | Used to specify the command to run to initialize the application's PostgreSQL database. Runs inside the application pod. | +| `DB_MIGRATE` | Used to specify the command to run to migrate the application's PostgreSQL database. Runs inside the application pod. | +| `POSTGRES_ENABLED` | Whether PostgreSQL is enabled. Defaults to `true`. Set to `false` to disable the automatic deployment of PostgreSQL. | +| `POSTGRES_USER` | The PostgreSQL user. Defaults to `user`. Set it to use a custom username. | +| `POSTGRES_PASSWORD` | The PostgreSQL password. Defaults to `testing-password`. Set it to use a custom password. | +| `POSTGRES_DB` | The PostgreSQL database name. Defaults to the value of [`$CI_ENVIRONMENT_SLUG`](../../ci/variables/index.md#predefined-cicd-variables). Set it to use a custom database name. | +| `POSTGRES_VERSION` | Tag for the [`postgres` Docker image](https://hub.docker.com/_/postgres) to use. Defaults to `9.6.16` for tests and deployments as of GitLab 13.0 (previously `9.6.2`). If `AUTO_DEVOPS_POSTGRES_CHANNEL` is set to `1`, deployments uses the default version `9.6.2`. | +| `POSTGRES_HELM_UPGRADE_VALUES_FILE` | In GitLab 13.8 and later, and when using [auto-deploy-image v2](upgrading_auto_deploy_dependencies.md), this variable allows the `helm upgrade` values file for PostgreSQL to be overridden. Defaults to `.gitlab/auto-deploy-postgres-values.yaml`. | +| `POSTGRES_HELM_UPGRADE_EXTRA_ARGS` | In GitLab 13.8 and later, and when using [auto-deploy-image v2](upgrading_auto_deploy_dependencies.md), this variable allows extra PostgreSQL options in `helm upgrade` commands when deploying the application. Note that using quotes doesn't prevent word splitting. | +| `POSTGRES_CHART_REPOSITORY` | Helm Chart repository used to search for PostgreSQL chart. Defaults to `https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami`. | +| `POSTGRES_CHART_VERSION` | Helm Chart version used for PostgreSQL chart. Defaults to `8.2.1`. | + +## Job-disabling variables + +Use these variables to disable CI/CD jobs. + +| **Job name** | **CI/CD variable** | **GitLab version** | **Description** | +|----------------------------------------|---------------------------------|-----------------------|-----------------| +| `.fuzz_base` | `COVFUZZ_DISABLED` | [From GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34984) | [Read more](../../user/application_security/coverage_fuzzing/index.md) about how `.fuzz_base` provide capability for your own jobs. If the variable is present, your jobs aren't created. | +| `apifuzzer_fuzz` | `API_FUZZING_DISABLED` | [From GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39135) | If the variable is present, the job isn't created. | +| `build` | `BUILD_DISABLED` | | If the variable is present, the job isn't created. | +| `build_artifact` | `BUILD_DISABLED` | | If the variable is present, the job isn't created. | +| `bandit-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `brakeman-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `canary` | `CANARY_ENABLED` | | This manual job is created if the variable is present. | +| `cluster_image_scanning` | `CLUSTER_IMAGE_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | +| `code_intelligence` | `CODE_INTELLIGENCE_DISABLED` | From GitLab 13.6 | If the variable is present, the job isn't created. | +| `code_quality` | `CODE_QUALITY_DISABLED` | | If the variable is present, the job isn't created. | +| `container_scanning` | `CONTAINER_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | +| `dast` | `DAST_DISABLED` | | If the variable is present, the job isn't created. | +| `dast_environment_deploy` | `DAST_DISABLED_FOR_DEFAULT_BRANCH` or `DAST_DISABLED` | [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17789) | If either variable is present, the job isn't created. | +| `dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | +| `eslint-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `flawfinder-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `gemnasium-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | +| `gemnasium-maven-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | +| `gemnasium-python-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | +| `gosec-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `kubesec-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `license_management` | `LICENSE_MANAGEMENT_DISABLED` | GitLab 12.7 and earlier | If the variable is present, the job isn't created. Job deprecated [from GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22773) | +| `license_scanning` | `LICENSE_MANAGEMENT_DISABLED` | [From GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22773) | If the variable is present, the job isn't created. | +| `load_performance` | `LOAD_PERFORMANCE_DISABLED` | From GitLab 13.2 | If the variable is present, the job isn't created. | +| `nodejs-scan-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `performance` | `PERFORMANCE_DISABLED` | GitLab 13.12 and earlier | Browser performance. If the variable is present, the job isn't created. Replaced by `browser_performance`. | +| `browser_performance` | `BROWSER_PERFORMANCE_DISABLED` | From GitLab 14.0 | Browser performance. If the variable is present, the job isn't created. Replaces `performance`. | +| `phpcs-security-audit-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `pmd-apex-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `review` | `REVIEW_DISABLED` | | If the variable is present, the job isn't created. | +| `review:stop` | `REVIEW_DISABLED` | | Manual job. If the variable is present, the job isn't created. | +| `sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `sast:container` | `CONTAINER_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | +| `secret_detection` | `SECRET_DETECTION_DISABLED` | From GitLab 13.1 | If the variable is present, the job isn't created. | +| `secret_detection_default_branch` | `SECRET_DETECTION_DISABLED` | [From GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22773) | If the variable is present, the job isn't created. | +| `security-code-scan-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `secrets-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `sobelaw-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `stop_dast_environment` | `DAST_DISABLED_FOR_DEFAULT_BRANCH` or `DAST_DISABLED` | [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17789) | If either variable is present, the job isn't created. | +| `spotbugs-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `test` | `TEST_DISABLED` | | If the variable is present, the job isn't created. | +| `staging` | `STAGING_ENABLED` | | The job is created if the variable is present. | +| `stop_review` | `REVIEW_DISABLED` | | If the variable is present, the job isn't created. | + +## Configure application secret variables + +Some deployed applications require access to secret variables. +Auto DevOps detects CI/CD variables starting with `K8S_SECRET_`, +and makes them available to the deployed application as +environment variables. + +To configure secret variables: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Variables**. +1. Create a CI/CD variable with the prefix `K8S_SECRET_`. For example, you + can create a variable called `K8S_SECRET_RAILS_MASTER_KEY`. +1. Run an Auto DevOps pipeline, either by manually creating a new + pipeline or by pushing a code change to GitLab. + +### Kubernetes secrets + +Auto DevOps pipelines use your application secret variables to +populate a Kubernetes secret. This secret is unique per environment. +When deploying your application, the secret is loaded as environment +variables in the container running the application. For example, if +you create a secret called `K8S_SECRET_RAILS_MASTER_KEY`, your +Kubernetes secret might look like: + +```shell +$ kubectl get secret production-secret -n minimal-ruby-app-54 -o yaml + +apiVersion: v1 +data: + RAILS_MASTER_KEY: MTIzNC10ZXN0 +kind: Secret +metadata: + creationTimestamp: 2018-12-20T01:48:26Z + name: production-secret + namespace: minimal-ruby-app-54 + resourceVersion: "429422" + selfLink: /api/v1/namespaces/minimal-ruby-app-54/secrets/production-secret + uid: 57ac2bfd-03f9-11e9-b812-42010a9400e4 +type: Opaque +``` + +## Update application secrets + +Environment variables are generally immutable in a Kubernetes pod. +If you update an application secret and then manually +create a new pipeline, running applications do not receive the +updated secret. + +To update application secrets, either: + +- Push a code update to GitLab to force the Kubernetes deployment to recreate pods. +- Manually delete running pods to cause Kubernetes to create new pods with updated + secrets. + +Variables with multi-line values are not supported due to +limitations with the Auto DevOps scripting environment. + +## Configure replica variables + +Add replica variables when you want to scale your deployments: + +1. Add a replica variable as a [project CI/CD variable](../../ci/variables/index.md#add-a-cicd-variable-to-a-project). +1. To scale your application, redeploy it. + + WARNING: + Do not scale your application using Kubernetes directly. Helm might not detect the change, + and subsequent deployments with Auto DevOps can undo your changes. + +### Custom replica variables + +You can create custom replica variables with the format `<TRACK>_<ENV>_REPLICAS`: + +- `<TRACK>` is the all-caps value of the `track` + [Kubernetes label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) + set in the Helm Chart app definition. If `track` is not set, omit `<TRACK>` from the custom variable. +- `<ENV>` is the all-caps environment name of the deploy job set in + `.gitlab-ci.yml`. + +For example, if the environment is `qa` and the track is +`foo`, create an environment variable called `FOO_QA_REPLICAS`: + +```yaml +QA testing: + stage: deploy + environment: + name: qa + script: + - deploy foo +``` + +The track `foo` must be defined in the application's Helm chart. +For example: + +```yaml +replicaCount: 1 +image: + repository: gitlab.example.com/group/project + tag: stable + pullPolicy: Always + secrets: + - name: gitlab-registry +application: + track: foo + tier: web +service: + enabled: true + name: web + type: ClusterIP + url: http://my.host.com/ + externalPort: 5000 + internalPort: 5000 +``` + +## Deploy policy for staging and production environments + +Auto DevOps typically uses continuous deployment, and pushes +automatically to the `production` environment whenever a new pipeline +runs on the default branch. To deploy to production manually, you can +use the `STAGING_ENABLED` CI/CD variable. + +If you set `STAGING_ENABLED`, GitLab automatically deploys the +application to a `staging` environment. When you're ready to deploy to +production, GitLab creates a `production_manual` job. + +You can also enable manual deployment in your [project settings](requirements.md#auto-devops-deployment-strategy). + +## Deploy policy for canary environments **(PREMIUM)** + +You can use a [canary environment](../../user/project/canary_deployments.md) before +deploying any changes to production. + +If you set `CANARY_ENABLED`, GitLab creates two [manual jobs](../../ci/pipelines/index.md#add-manual-interaction-to-your-pipeline): + +- `canary` - Deploys the application to the canary environment. +- `production_manual` - Deploys the application to production. + +## Incremental rollout to production **(PREMIUM)** + +Use an incremental rollout to continuously deploy your application, +starting with only a few pods. You can increase the number of pods +manually. + +You can enable manual deployment in your [project settings](requirements.md#auto-devops-deployment-strategy), +or by setting `INCREMENTAL_ROLLOUT_MODE` to `manual`. + +If you set `INCREMENTAL_ROLLOUT_MODE` to `manual`, GitLab creates four +manual jobs: + +1. `rollout 10%` +1. `rollout 25%` +1. `rollout 50%` +1. `rollout 100%` + +The percentage is based on the `REPLICAS` CI/CD variable, and defines the number of +pods used for deployment. For example, if the value is `10` and you run the +`10%` rollout job, your application is deployed to only one pod. + +You can run the rollout jobs in any order. To scale down, rerun a +lower percentage job. + +After you run the `rollout 100%` job, you cannot scale down, and must +[rollback your deployment](../../ci/environments/index.md#retry-or-roll-back-a-deployment). + +### Example incremental rollout configurations + +Without `INCREMENTAL_ROLLOUT_MODE` and without `STAGING_ENABLED`: + + + +Without `INCREMENTAL_ROLLOUT_MODE` and with `STAGING_ENABLED`: + + + +With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and without `STAGING_ENABLED`: + + + +With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and with `STAGING_ENABLED`: + + + +WARNING: +This configuration is deprecated, and is scheduled to be removed in the future. + +## Timed incremental rollout to production **(PREMIUM)** + +Use a timed incremental rollout to continuously deploy your application, starting with +only a few pods. + +You can enable timed incremental deployment in your [project settings](requirements.md#auto-devops-deployment-strategy), +or by setting the `INCREMENTAL_ROLLOUT_MODE` CI/CD variable to `timed`. + +If you set `INCREMENTAL_ROLLOUT_MODE` to `timed`, GitLab creates four jobs: + +1. `timed rollout 10%` +1. `timed rollout 25%` +1. `timed rollout 50%` +1. `timed rollout 100%` + +There is a five-minute delay between jobs. diff --git a/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md b/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md index 8a041b08a4d..78c51572973 100644 --- a/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md +++ b/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md @@ -231,7 +231,7 @@ takes you to the pod's logs page. NOTE: The example shows only one pod hosting the application at the moment, but you can add -more pods by defining the [`REPLICAS` CI/CD variable](../customize.md#cicd-variables) +more pods by defining the [`REPLICAS` CI/CD variable](../cicd_variables.md) in **Settings > CI/CD > Variables**. ### Work with branches @@ -300,7 +300,7 @@ and customized to fit your workflow. Here are some helpful resources for further 1. [Auto DevOps](../index.md) 1. [Multiple Kubernetes clusters](../multiple_clusters_auto_devops.md) -1. [Incremental rollout to production](../customize.md#incremental-rollout-to-production) -1. [Disable jobs you don't need with CI/CD variables](../customize.md#cicd-variables) +1. [Incremental rollout to production](../cicd_variables.md#incremental-rollout-to-production) +1. [Disable jobs you don't need with CI/CD variables](../cicd_variables.md) 1. [Use your own buildpacks to build your application](../customize.md#custom-buildpacks) 1. [Prometheus monitoring](../../../user/project/integrations/prometheus.md) diff --git a/doc/topics/autodevops/customize.md b/doc/topics/autodevops/customize.md index 264ac790453..c42f5825b6a 100644 --- a/doc/topics/autodevops/customize.md +++ b/doc/topics/autodevops/customize.md @@ -105,7 +105,7 @@ You can override this behavior by defining specific variables: These variables also affect Auto Build and Auto Container Scanning. If you don't want to build and push an image to `$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG`, consider -including only `Jobs/Deploy.gitlab-ci.yml`, or [disabling the `build` jobs](#disable-jobs). +including only `Jobs/Deploy.gitlab-ci.yml`, or [disabling the `build` jobs](cicd_variables.md#job-disabling-variables). If you use Auto Container Scanning and set a value for `$CI_APPLICATION_REPOSITORY`, then you should also update `$CS_DEFAULT_BRANCH_IMAGE`. See [Setting the default branch image](../../user/application_security/container_scanning/index.md#setting-the-default-branch-image) @@ -187,11 +187,11 @@ You can override the default values in the `values.yaml` file in the - Adding a file named `.gitlab/auto-deploy-values.yaml` to your repository, which is automatically used, if found. - Adding a file with a different name or path to the repository, and setting the - `HELM_UPGRADE_VALUES_FILE` [CI/CD variable](#cicd-variables) with + `HELM_UPGRADE_VALUES_FILE` [CI/CD variable](cicd_variables.md) with the path and name. Some values cannot be overridden with the options above. Settings like `replicaCount` should instead be overridden with the `REPLICAS` -[build and deployment](#build-and-deployment) CI/CD variable. Follow [this issue](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/issues/31) for more information. +[build and deployment](cicd_variables.md#build-and-deployment-variables) CI/CD variable. Follow [this issue](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/issues/31) for more information. NOTE: For GitLab 12.5 and earlier, use the `HELM_UPGRADE_EXTRA_ARGS` variable @@ -308,13 +308,13 @@ You can configure many Auto DevOps jobs to run in an [offline environment](../.. To support applications requiring a database, [PostgreSQL](https://www.postgresql.org/) is provisioned by default. The credentials to access the database are preconfigured, but can be customized by setting the associated -[CI/CD variables](#cicd-variables). You can use these credentials to define a `DATABASE_URL`: +[CI/CD variables](cicd_variables.md). You can use these credentials to define a `DATABASE_URL`: ```yaml postgres://user:password@postgres-host:postgres-port/postgres-database ``` -### Upgrading PostgresSQL +### Upgrading PostgreSQL WARNING: The CI/CD variable `AUTO_DEVOPS_POSTGRES_CHANNEL` that controls default provisioned @@ -340,9 +340,9 @@ To set custom values, do one of the following: - Add a file named `.gitlab/auto-deploy-postgres-values.yaml` to your repository. If found, this file is used automatically. This file is used by default for PostgreSQL Helm upgrades. - Add a file with a different name or path to the repository, and set the - `POSTGRES_HELM_UPGRADE_VALUES_FILE` [environment variable](#database) with the path + `POSTGRES_HELM_UPGRADE_VALUES_FILE` [environment variable](cicd_variables.md#database-variables) with the path and name. -- Set the `POSTGRES_HELM_UPGRADE_EXTRA_ARGS` [environment variable](#database). +- Set the `POSTGRES_HELM_UPGRADE_EXTRA_ARGS` [environment variable](cicd_variables.md#database-variables). ### Using external PostgreSQL database providers @@ -371,342 +371,6 @@ You must define environment-scoped CI/CD variables for `POSTGRES_ENABLED` and You must ensure that your Kubernetes cluster has network access to wherever PostgreSQL is hosted. -## CI/CD variables - -The following variables can be used for setting up the Auto DevOps domain, -providing a custom Helm chart, or scaling your application. PostgreSQL can -also be customized, and you can use a [custom buildpack](#custom-buildpacks). - -### Build and deployment - -The following table lists CI/CD variables related to building and deploying -applications. - -| **CI/CD Variable** | **Description** | -|-----------------------------------------|------------------------------------| -| `ADDITIONAL_HOSTS` | Fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. | -| `<ENVIRONMENT>_ADDITIONAL_HOSTS` | For a specific environment, the fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. This takes precedence over `ADDITIONAL_HOSTS`. | -| `AUTO_BUILD_IMAGE_VERSION` | Customize the image version used for the `build` job. See [list of versions](https://gitlab.com/gitlab-org/cluster-integration/auto-build-image/-/releases). | -| `AUTO_DEPLOY_IMAGE_VERSION` | Customize the image version used for Kubernetes deployment jobs. See [list of versions](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/releases). | -| `AUTO_DEVOPS_ATOMIC_RELEASE` | As of GitLab 13.0, Auto DevOps uses [`--atomic`](https://v2.helm.sh/docs/helm/#options-43) for Helm deployments by default. Set this variable to `false` to disable the use of `--atomic` | -| `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` | Set to `false` to use Herokuish instead of Cloud Native Buildpacks with Auto Build. [More details](stages.md#auto-build-using-cloud-native-buildpacks). | -| `AUTO_DEVOPS_BUILD_IMAGE_CNB_BUILDER` | The builder used when building with Cloud Native Buildpacks. The default builder is `heroku/buildpacks:18`. [More details](stages.md#auto-build-using-cloud-native-buildpacks). | -| `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` | Extra arguments to be passed to the `docker build` command. Note that using quotes doesn't prevent word splitting. [More details](#passing-arguments-to-docker-build). | -| `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` | A [comma-separated list of CI/CD variable names](#forward-cicd-variables-to-the-build-environment) to be forwarded to the build environment (the buildpack builder or `docker build`). | -| `AUTO_DEVOPS_BUILD_IMAGE_CNB_PORT` | In GitLab 15.0 and later, port exposed by the generated Docker image. Set to `false` to prevent exposing any ports. Defaults to `5000`. | -| `AUTO_DEVOPS_CHART` | Helm Chart used to deploy your apps. Defaults to the one [provided by GitLab](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/tree/master/assets/auto-deploy-app). | -| `AUTO_DEVOPS_CHART_REPOSITORY` | Helm Chart repository used to search for charts. Defaults to `https://charts.gitlab.io`. | -| `AUTO_DEVOPS_CHART_REPOSITORY_NAME` | Used to set the name of the Helm repository. Defaults to `gitlab`. | -| `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME` | Used to set a username to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD`. | -| `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD` | Used to set a password to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME`. | -| `AUTO_DEVOPS_CHART_REPOSITORY_PASS_CREDENTIALS` | From GitLab 14.2, set to a non-empty value to enable forwarding of the Helm repository credentials to the chart server when the chart artifacts are on a different host than repository. | -| `AUTO_DEVOPS_COMMON_NAME` | From GitLab 15.5, set to a valid domain name to customize the common name used for the TLS certificate. Defaults to `le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN`. Set to `false` to not set this alternative host on the Ingress. | -| `AUTO_DEVOPS_DEPLOY_DEBUG` | From GitLab 13.1, if this variable is present, Helm outputs debug logs. | -| `AUTO_DEVOPS_ALLOW_TO_FORCE_DEPLOY_V<N>` | From [auto-deploy-image](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image) v1.0.0, if this variable is present, a new major version of chart is forcibly deployed. For more information, see [Ignore warnings and continue deploying](upgrading_auto_deploy_dependencies.md#ignore-warnings-and-continue-deploying). | -| `BUILDPACK_URL` | Buildpack's full URL. [Must point to a URL supported by Pack or Herokuish](#custom-buildpacks). | -| `CANARY_ENABLED` | Used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments). | -| `BUILDPACK_VOLUMES` | Specify one or more [Buildpack volumes to mount](stages.md#mount-volumes-into-the-build-container). Use a pipe `|` as list separator. | -| `CANARY_PRODUCTION_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md) in the production environment. Takes precedence over `CANARY_REPLICAS`. Defaults to 1. | -| `CANARY_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md). Defaults to 1. | -| `CI_APPLICATION_REPOSITORY` | The repository of container image being built or deployed, `$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG`. For more details, read [Custom container image](#custom-container-image). | -| `CI_APPLICATION_TAG` | The tag of the container image being built or deployed, `$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG`. For more details, read [Custom container image](#custom-container-image). | -| `DAST_AUTO_DEPLOY_IMAGE_VERSION` | Customize the image version used for DAST deployments on the default branch. Should usually be the same as `AUTO_DEPLOY_IMAGE_VERSION`. See [list of versions](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/releases). | -| `DOCKERFILE_PATH` | From GitLab 13.2, allows overriding the [default Dockerfile path for the build stage](#custom-dockerfile) | -| `HELM_RELEASE_NAME` | From GitLab 12.1, allows the `helm` release name to be overridden. Can be used to assign unique release names when deploying multiple projects to a single namespace. | -| `HELM_UPGRADE_VALUES_FILE` | From GitLab 12.6, allows the `helm upgrade` values file to be overridden. Defaults to `.gitlab/auto-deploy-values.yaml`. | -| `HELM_UPGRADE_EXTRA_ARGS` | Allows extra options in `helm upgrade` commands when deploying the application. Note that using quotes doesn't prevent word splitting. | -| `INCREMENTAL_ROLLOUT_MODE` | If present, can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment. Set to `manual` for manual deployment jobs or `timed` for automatic rollout deployments with a 5 minute delay each one. | -| `K8S_SECRET_*` | Any variable prefixed with [`K8S_SECRET_`](#application-secret-variables) is made available by Auto DevOps as environment variables to the deployed application. | -| `KUBE_CONTEXT` | From GitLab 14.5, can be used to select a context to use from `KUBECONFIG`. When `KUBE_CONTEXT` is blank, the default context in `KUBECONFIG` (if any) is used. A context must be selected when used [with the agent for Kubernetes](../../user/clusters/agent/ci_cd_workflow.md). | -| `KUBE_INGRESS_BASE_DOMAIN` | Can be used to set a domain per cluster. See [cluster domains](../../user/project/clusters/gitlab_managed_clusters.md#base-domain) for more information. | -| `KUBE_NAMESPACE` | The namespace used for deployments. When using certificate-based clusters, [this value should not be overwritten directly](../../user/project/clusters/deploy_to_cluster.md#custom-namespace). | -| `KUBECONFIG` | The kubeconfig to use for deployments. User-provided values take priority over GitLab-provided values. | -| `PRODUCTION_REPLICAS` | Number of replicas to deploy in the production environment. Takes precedence over `REPLICAS` and defaults to 1. For zero downtime upgrades, set to 2 or greater. | -| `REPLICAS` | Number of replicas to deploy. Defaults to 1. Change this variable instead of [modifying](#customize-values-for-helm-chart) `replicaCount`. | -| `ROLLOUT_RESOURCE_TYPE` | Allows specification of the resource type being deployed when using a custom Helm chart. Default value is `deployment`. | -| `ROLLOUT_STATUS_DISABLED` | From GitLab 12.0, used to disable rollout status check because it does not support all resource types, for example, `cronjob`. | -| `STAGING_ENABLED` | Used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). | -| `TRACE` | Set to any value to make Helm commands produce verbose output. You can use this setting to help diagnose Auto DevOps deployment problems. | - -NOTE: -After you set up your replica variables using a -[project CI/CD variable](../../ci/variables/index.md), -you can scale your application by redeploying it. - -WARNING: -You should *not* scale your application using Kubernetes directly. This can -cause confusion with Helm not detecting the change, and subsequent deploys with -Auto DevOps can undo your changes. - -### Database - -The following table lists CI/CD variables related to the database. - -| **CI/CD Variable** | **Description** | -|-----------------------------------------|------------------------------------| -| `DB_INITIALIZE` | Used to specify the command to run to initialize the application's PostgreSQL database. Runs inside the application pod. | -| `DB_MIGRATE` | Used to specify the command to run to migrate the application's PostgreSQL database. Runs inside the application pod. | -| `POSTGRES_ENABLED` | Whether PostgreSQL is enabled. Defaults to `true`. Set to `false` to disable the automatic deployment of PostgreSQL. | -| `POSTGRES_USER` | The PostgreSQL user. Defaults to `user`. Set it to use a custom username. | -| `POSTGRES_PASSWORD` | The PostgreSQL password. Defaults to `testing-password`. Set it to use a custom password. | -| `POSTGRES_DB` | The PostgreSQL database name. Defaults to the value of [`$CI_ENVIRONMENT_SLUG`](../../ci/variables/index.md#predefined-cicd-variables). Set it to use a custom database name. | -| `POSTGRES_VERSION` | Tag for the [`postgres` Docker image](https://hub.docker.com/_/postgres) to use. Defaults to `9.6.16` for tests and deployments as of GitLab 13.0 (previously `9.6.2`). If `AUTO_DEVOPS_POSTGRES_CHANNEL` is set to `1`, deployments uses the default version `9.6.2`. | -| `POSTGRES_HELM_UPGRADE_VALUES_FILE` | In GitLab 13.8 and later, and when using [auto-deploy-image v2](upgrading_auto_deploy_dependencies.md), this variable allows the `helm upgrade` values file for PostgreSQL to be overridden. Defaults to `.gitlab/auto-deploy-postgres-values.yaml`. | -| `POSTGRES_HELM_UPGRADE_EXTRA_ARGS` | In GitLab 13.8 and later, and when using [auto-deploy-image v2](upgrading_auto_deploy_dependencies.md), this variable allows extra PostgreSQL options in `helm upgrade` commands when deploying the application. Note that using quotes doesn't prevent word splitting. | -| `POSTGRES_CHART_REPOSITORY` | Helm Chart repository used to search for PostgreSQL chart. Defaults to `https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami`. | -| `POSTGRES_CHART_VERSION` | Helm Chart version used for PostgreSQL chart. Defaults to `8.2.1`. | - -### Disable jobs - -The following table lists variables used to disable jobs. - -| **Job Name** | **CI/CDVariable** | **GitLab version** | **Description** | -|----------------------------------------|---------------------------------|-----------------------|-----------------| -| `.fuzz_base` | `COVFUZZ_DISABLED` | [From GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34984) | [Read more](../../user/application_security/coverage_fuzzing/index.md) about how `.fuzz_base` provide capability for your own jobs. If the variable is present, your jobs aren't created. | -| `apifuzzer_fuzz` | `API_FUZZING_DISABLED` | [From GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39135) | If the variable is present, the job isn't created. | -| `build` | `BUILD_DISABLED` | | If the variable is present, the job isn't created. | -| `build_artifact` | `BUILD_DISABLED` | | If the variable is present, the job isn't created. | -| `bandit-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `brakeman-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `canary` | `CANARY_ENABLED` | | This manual job is created if the variable is present. | -| `cluster_image_scanning` | `CLUSTER_IMAGE_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | -| `code_intelligence` | `CODE_INTELLIGENCE_DISABLED` | From GitLab 13.6 | If the variable is present, the job isn't created. | -| `code_quality` | `CODE_QUALITY_DISABLED` | | If the variable is present, the job isn't created. | -| `container_scanning` | `CONTAINER_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | -| `dast` | `DAST_DISABLED` | | If the variable is present, the job isn't created. | -| `dast_environment_deploy` | `DAST_DISABLED_FOR_DEFAULT_BRANCH` or `DAST_DISABLED` | [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17789) | If either variable is present, the job isn't created. | -| `dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | -| `eslint-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `flawfinder-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `gemnasium-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | -| `gemnasium-maven-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | -| `gemnasium-python-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | -| `gosec-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `kubesec-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `license_management` | `LICENSE_MANAGEMENT_DISABLED` | GitLab 12.7 and earlier | If the variable is present, the job isn't created. Job deprecated [from GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22773) | -| `license_scanning` | `LICENSE_MANAGEMENT_DISABLED` | [From GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22773) | If the variable is present, the job isn't created. | -| `load_performance` | `LOAD_PERFORMANCE_DISABLED` | From GitLab 13.2 | If the variable is present, the job isn't created. | -| `nodejs-scan-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `performance` | `PERFORMANCE_DISABLED` | GitLab 13.12 and earlier | Browser performance. If the variable is present, the job isn't created. Replaced by `browser_performance`. | -| `browser_performance` | `BROWSER_PERFORMANCE_DISABLED` | From GitLab 14.0 | Browser performance. If the variable is present, the job isn't created. Replaces `performance`. | -| `phpcs-security-audit-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `pmd-apex-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `review` | `REVIEW_DISABLED` | | If the variable is present, the job isn't created. | -| `review:stop` | `REVIEW_DISABLED` | | Manual job. If the variable is present, the job isn't created. | -| `sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `sast:container` | `CONTAINER_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | -| `secret_detection` | `SECRET_DETECTION_DISABLED` | From GitLab 13.1 | If the variable is present, the job isn't created. | -| `secret_detection_default_branch` | `SECRET_DETECTION_DISABLED` | [From GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22773) | If the variable is present, the job isn't created. | -| `security-code-scan-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `secrets-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `sobelaw-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `stop_dast_environment` | `DAST_DISABLED_FOR_DEFAULT_BRANCH` or `DAST_DISABLED` | [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17789) | If either variable is present, the job isn't created. | -| `spotbugs-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `test` | `TEST_DISABLED` | | If the variable is present, the job isn't created. | -| `staging` | `STAGING_ENABLED` | | The job is created if the variable is present. | -| `stop_review` | `REVIEW_DISABLED` | | If the variable is present, the job isn't created. | - -### Application secret variables - -Some applications need to define secret variables that are accessible by the deployed -application. Auto DevOps detects CI/CD variables starting with `K8S_SECRET_`, and makes -these prefixed variables available to the deployed application as environment variables. - -To configure your application variables: - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Settings > CI/CD**. -1. Expand **Variables**. -1. Create a CI/CD variable, ensuring the key is prefixed with - `K8S_SECRET_`. For example, you can create a variable with key - `K8S_SECRET_RAILS_MASTER_KEY`. - -1. Run an Auto DevOps pipeline, either by manually creating a new - pipeline or by pushing a code change to GitLab. - -Auto DevOps pipelines take your application secret variables to -populate a Kubernetes secret. This secret is unique per environment. -When deploying your application, the secret is loaded as environment -variables in the container running the application. Following the -example above, you can see the secret below containing the -`RAILS_MASTER_KEY` variable. - -```shell -$ kubectl get secret production-secret -n minimal-ruby-app-54 -o yaml - -apiVersion: v1 -data: - RAILS_MASTER_KEY: MTIzNC10ZXN0 -kind: Secret -metadata: - creationTimestamp: 2018-12-20T01:48:26Z - name: production-secret - namespace: minimal-ruby-app-54 - resourceVersion: "429422" - selfLink: /api/v1/namespaces/minimal-ruby-app-54/secrets/production-secret - uid: 57ac2bfd-03f9-11e9-b812-42010a9400e4 -type: Opaque -``` - -Environment variables are generally considered immutable in a Kubernetes pod. -If you update an application secret without changing any code, then manually -create a new pipeline, any running application pods don't receive -the updated secrets. To update the secrets, either: - -- Push a code update to GitLab to force the Kubernetes deployment to recreate pods. -- Manually delete running pods to cause Kubernetes to create new pods with updated - secrets. - -Variables with multi-line values are not currently supported due to -limitations with the current Auto DevOps scripting environment. - -### Advanced replica variables setup - -Apart from the two replica-related variables for production mentioned above, -you can also use other variables for different environments. - -The Kubernetes' label named `track`, GitLab CI/CD environment names, and the -replicas environment variable are combined into the format `TRACK_ENV_REPLICAS`, -enabling you to define your own variables for scaling the pod's replicas: - -- `TRACK`: The capitalized value of the `track` - [Kubernetes label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) - in the Helm Chart app definition. If not set, it isn't taken into account - to the variable name. -- `ENV`: The capitalized environment name of the deploy job, set in - `.gitlab-ci.yml`. - -In the example below, the environment's name is `qa`, and it deploys the track -`foo`, which results in an environment variable named `FOO_QA_REPLICAS`: - -```yaml -QA testing: - stage: deploy - environment: - name: qa - script: - - deploy foo -``` - -The track `foo` being referenced must also be defined in the application's Helm chart, like: - -```yaml -replicaCount: 1 -image: - repository: gitlab.example.com/group/project - tag: stable - pullPolicy: Always - secrets: - - name: gitlab-registry -application: - track: foo - tier: web -service: - enabled: true - name: web - type: ClusterIP - url: http://my.host.com/ - externalPort: 5000 - internalPort: 5000 -``` - -### Deploy policy for staging and production environments - -NOTE: -You can also set this inside your [project's settings](requirements.md#auto-devops-deployment-strategy). - -The normal behavior of Auto DevOps is to use continuous deployment, pushing -automatically to the `production` environment every time a new pipeline is run -on the default branch. However, there are cases where you might want to use a -staging environment, and deploy to production manually. For this scenario, the -`STAGING_ENABLED` CI/CD variable was introduced. - -If you define `STAGING_ENABLED` with a non-empty value, then GitLab automatically deploys the application -to a `staging` environment, and creates a `production_manual` job for -you when you're ready to manually deploy to production. - -### Deploy policy for canary environments **(PREMIUM)** - -You can use a [canary environment](../../user/project/canary_deployments.md) before -deploying any changes to production. - -If you define `CANARY_ENABLED` with a non-empty value, then two manual jobs are created: - -- `canary` - Deploys the application to the canary environment. -- `production_manual` - Manually deploys the application to production. - -### Incremental rollout to production **(PREMIUM)** - -NOTE: -You can also set this inside your [project's settings](requirements.md#auto-devops-deployment-strategy). - -When you're ready to deploy a new version of your app to production, you may want -to use an incremental rollout to replace just a few pods with the latest code to -check how the application is behaving before manually increasing the rollout up to 100%. - -If `INCREMENTAL_ROLLOUT_MODE` is set to `manual` in your project, then instead -of the standard `production` job, 4 different -[manual jobs](../../ci/pipelines/index.md#add-manual-interaction-to-your-pipeline) -are created: - -1. `rollout 10%` -1. `rollout 25%` -1. `rollout 50%` -1. `rollout 100%` - -The percentage is based on the `REPLICAS` CI/CD variable, and defines the number of -pods you want to have for your deployment. If the value is `10`, and you run the -`10%` rollout job, there is `1` new pod and `9` old ones. - -To start a job, select the play icon (**{play}**) next to the job's name. You're not -required to go from `10%` to `100%`, you can jump to whatever job you want. -You can also scale down by running a lower percentage job, just before hitting -`100%`. Once you get to `100%`, you can't scale down, and you'd have to roll -back by redeploying the old version using the -[rollback button](../../ci/environments/index.md#retry-or-roll-back-a-deployment) in the -environment page. - -Below, you can see how the pipeline appears if the rollout or staging -variables are defined. - -Without `INCREMENTAL_ROLLOUT_MODE` and without `STAGING_ENABLED`: - - - -Without `INCREMENTAL_ROLLOUT_MODE` and with `STAGING_ENABLED`: - - - -With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and without `STAGING_ENABLED`: - - - -With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and with `STAGING_ENABLED` - - - -WARNING: -This configuration is deprecated, and is scheduled to be removed in the future. - -### Timed incremental rollout to production **(PREMIUM)** - -NOTE: -You can also set this inside your [project's settings](requirements.md#auto-devops-deployment-strategy). - -This configuration is based on -[incremental rollout to production](#incremental-rollout-to-production). - -Everything behaves the same way, except: - -- To enable it, set the `INCREMENTAL_ROLLOUT_MODE` CI/CD variable to `timed`. -- Instead of the standard `production` job, the following jobs are created with - a 5 minute delay between each: - - 1. `timed rollout 10%` - 1. `timed rollout 25%` - 1. `timed rollout 50%` - 1. `timed rollout 100%` - ## Auto DevOps banner The following Auto DevOps banner displays for users with Maintainer or greater diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 73e87aa190f..d16229b525f 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -129,7 +129,7 @@ for the subgroups and projects where you don't want to use it. Prerequisites: -- You must have at least the Owner role for the group. +- You must have the Owner role for the group. To enable Auto DevOps for a group: diff --git a/doc/topics/autodevops/multiple_clusters_auto_devops.md b/doc/topics/autodevops/multiple_clusters_auto_devops.md index af8c54a8edd..cf775a35eb7 100644 --- a/doc/topics/autodevops/multiple_clusters_auto_devops.md +++ b/doc/topics/autodevops/multiple_clusters_auto_devops.md @@ -25,7 +25,7 @@ To deploy your environments to different Kubernetes clusters: 1. [Install a GitLab Agent on each cluster](../../user/clusters/agent/index.md). 1. [Configure each agent to access your project](../../user/clusters/agent/install/index.md#configure-your-agent). 1. [Install NGINX Ingress Controller](cloud_deployments/auto_devops_with_gke.md#install-ingress) in each cluster. Save the IP address and Kubernetes namespace for the next step. -1. [Configure the Auto DevOps CI/CD Pipeline variables](customize.md#build-and-deployment) +1. [Configure the Auto DevOps CI/CD Pipeline variables](cicd_variables.md#build-and-deployment-variables) - Set up a `KUBE_CONTEXT` variable [for each environment](../../ci/variables/index.md#limit-the-environment-scope-of-a-cicd-variable). The value must point to the agent of the relevant cluster. - Set up a `KUBE_INGRESS_BASE_DOMAIN`. You must [configure the base domain](requirements.md#auto-devops-base-domain) for each environment to point to the Ingress of the relevant cluster. - Add a `KUBE_NAMESPACE` variable with a value of the Kubernetes namespace you want your deployments to target. You can scope the variable to multiple environments. @@ -44,8 +44,8 @@ NOTE: | Cluster name | Cluster environment scope | `KUBE_INGRESS_BASE_DOMAIN` value | `KUBE CONTEXT` value | Variable environment scope | Notes | | :------------| :-------------------------| :------------------------------- | :--------------------------------- | :--------------------------|:--| | review | `review/*` | `review.example.com` | `path/to/project:review-agent` | `review/*` | A review cluster that runs all [Review Apps](../../ci/review_apps/index.md).| -| staging | `staging` | `staging.example.com` | `path/to/project:staging-agent` | `staging` | Optional. A staging cluster that runs the deployments of the staging environments. You must [enable it first](customize.md#deploy-policy-for-staging-and-production-environments). | -| production | `production` | `example.com` | `path/to/project:production-agent` | `production` | A production cluster that runs the production environment deployments. You can use [incremental rollouts](customize.md#incremental-rollout-to-production). | +| staging | `staging` | `staging.example.com` | `path/to/project:staging-agent` | `staging` | Optional. A staging cluster that runs the deployments of the staging environments. You must [enable it first](cicd_variables.md#deploy-policy-for-staging-and-production-environments). | +| production | `production` | `example.com` | `path/to/project:production-agent` | `production` | A production cluster that runs the production environment deployments. You can use [incremental rollouts](cicd_variables.md#incremental-rollout-to-production). | ## Test your configuration diff --git a/doc/topics/autodevops/prepare_deployment.md b/doc/topics/autodevops/prepare_deployment.md index 7805f9cd9c6..ebba20ca821 100644 --- a/doc/topics/autodevops/prepare_deployment.md +++ b/doc/topics/autodevops/prepare_deployment.md @@ -21,8 +21,8 @@ that works best for your needs: | Deployment strategy | Setup | Methodology | |--|--|--| | **Continuous deployment to production** | Enables [Auto Deploy](stages.md#auto-deploy) with the default branch continuously deployed to production. | Continuous deployment to production.| -| **Continuous deployment to production using timed incremental rollout** | Sets the [`INCREMENTAL_ROLLOUT_MODE`](customize.md#timed-incremental-rollout-to-production) variable to `timed`. | Continuously deploy to production with a 5 minutes delay between rollouts. | -| **Automatic deployment to staging, manual deployment to production** | Sets [`STAGING_ENABLED`](customize.md#deploy-policy-for-staging-and-production-environments) to `1` and [`INCREMENTAL_ROLLOUT_MODE`](customize.md#incremental-rollout-to-production) to `manual`. | The default branch is continuously deployed to staging and continuously delivered to production. | +| **Continuous deployment to production using timed incremental rollout** | Sets the [`INCREMENTAL_ROLLOUT_MODE`](cicd_variables.md#timed-incremental-rollout-to-production) variable to `timed`. | Continuously deploy to production with a 5 minutes delay between rollouts. | +| **Automatic deployment to staging, manual deployment to production** | Sets [`STAGING_ENABLED`](cicd_variables.md#deploy-policy-for-staging-and-production-environments) to `1` and [`INCREMENTAL_ROLLOUT_MODE`](cicd_variables.md#incremental-rollout-to-production) to `manual`. | The default branch is continuously deployed to staging and continuously delivered to production. | You can choose the deployment method when enabling Auto DevOps or later: diff --git a/doc/topics/autodevops/requirements.md b/doc/topics/autodevops/requirements.md index 7c69e0327c8..f5e06843e60 100644 --- a/doc/topics/autodevops/requirements.md +++ b/doc/topics/autodevops/requirements.md @@ -36,8 +36,8 @@ that works best for your needs: | Deployment strategy | Setup | Methodology | |--|--|--| | **Continuous deployment to production** | Enables [Auto Deploy](stages.md#auto-deploy) with the default branch continuously deployed to production. | Continuous deployment to production.| -| **Continuous deployment to production using timed incremental rollout** | Sets the [`INCREMENTAL_ROLLOUT_MODE`](customize.md#timed-incremental-rollout-to-production) variable to `timed`. | Continuously deploy to production with a 5 minutes delay between rollouts. | -| **Automatic deployment to staging, manual deployment to production** | Sets [`STAGING_ENABLED`](customize.md#deploy-policy-for-staging-and-production-environments) to `1` and [`INCREMENTAL_ROLLOUT_MODE`](customize.md#incremental-rollout-to-production) to `manual`. | The default branch is continuously deployed to staging and continuously delivered to production. | +| **Continuous deployment to production using timed incremental rollout** | Sets the [`INCREMENTAL_ROLLOUT_MODE`](cicd_variables.md#timed-incremental-rollout-to-production) variable to `timed`. | Continuously deploy to production with a 5 minutes delay between rollouts. | +| **Automatic deployment to staging, manual deployment to production** | Sets [`STAGING_ENABLED`](cicd_variables.md#deploy-policy-for-staging-and-production-environments) to `1` and [`INCREMENTAL_ROLLOUT_MODE`](cicd_variables.md#incremental-rollout-to-production) to `manual`. | The default branch is continuously deployed to staging and continuously delivered to production. | You can choose the deployment method when enabling Auto DevOps or later: diff --git a/doc/topics/autodevops/stages.md b/doc/topics/autodevops/stages.md index 022ee131e40..0ca9c6fa3de 100644 --- a/doc/topics/autodevops/stages.md +++ b/doc/topics/autodevops/stages.md @@ -385,7 +385,7 @@ default, but the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) contains job definitions for these tasks if you want to enable them. -You can use [CI/CD variables](customize.md#cicd-variables) to automatically +You can use [CI/CD variables](cicd_variables.md) to automatically scale your pod replicas, and to apply custom arguments to the Auto DevOps `helm upgrade` commands. This is an easy way to [customize the Auto Deploy Helm chart](customize.md#custom-helm-chart). @@ -620,4 +620,4 @@ for updates. This stage is enabled by default. You can disable it by adding the `CODE_INTELLIGENCE_DISABLED` CI/CD variable. Read more about -[disabling Auto DevOps jobs](../../topics/autodevops/customize.md#disable-jobs). +[disabling Auto DevOps jobs](../../topics/autodevops/cicd_variables.md#job-disabling-variables). diff --git a/doc/topics/autodevops/troubleshooting.md b/doc/topics/autodevops/troubleshooting.md index ae3cc42223f..ef420323b32 100644 --- a/doc/topics/autodevops/troubleshooting.md +++ b/doc/topics/autodevops/troubleshooting.md @@ -13,7 +13,7 @@ Auto DevOps, and any available workarounds. Set the CI/CD variable `TRACE` to any value to make Helm commands produce verbose output. You can use this output to diagnose Auto DevOps deployment problems. -You can resolve some problems with Auto DevOps deployment by changing advanced Auto DevOps configuration variables. Read more about [customizing Auto DevOps CI/CD variables](customize.md#cicd-variables). +You can resolve some problems with Auto DevOps deployment by changing advanced Auto DevOps configuration variables. Read more about [customizing Auto DevOps CI/CD variables](cicd_variables.md). ## Unable to select a buildpack @@ -40,7 +40,7 @@ The following are possible reasons: If your pipeline fails with the following message: ```plaintext -Found errors in your .gitlab-ci.yml: +Unable to create pipeline jobs:test config key may not be used with `rules`: only ``` diff --git a/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md b/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md index 602bdec6140..4fafc89cac1 100644 --- a/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md +++ b/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md @@ -128,7 +128,7 @@ with the [v1 auto-deploy-image](#use-a-specific-version-of-auto-deploy-dependenc > [Introduced](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/merge_requests/109) in GitLab 13.4. -Auto Deploy supports advanced deployment strategies such as [canary deployments](customize.md#deploy-policy-for-canary-environments) +Auto Deploy supports advanced deployment strategies such as [canary deployments](cicd_variables.md#deploy-policy-for-canary-environments) and [incremental rollouts](../../ci/environments/incremental_rollouts.md). Previously, `auto-deploy-image` created one service to balance the traffic between @@ -171,7 +171,7 @@ Alternatively, you can use the [v13.12 Auto DevOps templates archive](https://gi ### Ignore warnings and continue deploying If you are certain that the new chart version is safe to be deployed, you can add -the `AUTO_DEVOPS_FORCE_DEPLOY_V<major-version-number>` [CI/CD variable](customize.md#build-and-deployment) +the `AUTO_DEVOPS_FORCE_DEPLOY_V<major-version-number>` [CI/CD variable](cicd_variables.md#build-and-deployment-variables) to force the deployment to continue. For example, if you want to deploy the `v2.0.0` chart on a deployment that previously diff --git a/doc/topics/awesome_co.md b/doc/topics/awesome_co.md index 49e39542b2b..fc5f79b4d18 100644 --- a/doc/topics/awesome_co.md +++ b/doc/topics/awesome_co.md @@ -71,7 +71,7 @@ AwesomeCo seeding uses FactoryBot definitions from `spec/factories` which ... 1. Are always up-to-date 1. Execute on the lowest-level (`ActiveRecord`) possible to create data as quickly as possible -> _from the [FactoryBot README](https://github.com/thoughtbot/factory_bot#readme_) : factory_bot is a fixtures replacement with a straightforward definition syntax, support for multiple build +> From the [FactoryBot README](https://github.com/thoughtbot/factory_bot#readme_) : `factory_bot` is a fixtures replacement with a straightforward definition syntax, support for multiple build > strategies (saved instances, unsaved instances, attribute hashes, and stubbed objects), and support for multiple factories for the same class, including factory > inheritance diff --git a/doc/topics/git/feature_branch_development.md b/doc/topics/git/feature_branch_development.md index de9f9980811..d53c8eae835 100644 --- a/doc/topics/git/feature_branch_development.md +++ b/doc/topics/git/feature_branch_development.md @@ -77,7 +77,7 @@ In this case, the feature branch would be `release-X-Y`. Assuming the `release-X 1. After you select **Create merge request**, an option to **Change branches** displays. Select that option. -1. In the **New Merge Request** screen, you can now select the **Source** and **Target** branches. +1. In the **New merge request** screen, you can now select the **Source** and **Target** branches. In the screenshot shown, we have selected `test-branch` as the source, and `release-13-0` as the target. @@ -87,14 +87,14 @@ we have selected `test-branch` as the source, and `release-13-0` as the target. You should see an entry similar to: ```plaintext - New Merge Request + New merge request From test-branch into release-13-0 ``` An entry like this confirms your merge request's destination. -1. Make any additional changes in the **New Merge Request** screen, and select **Submit merge request**. +1. Make any additional changes in the **New merge request** screen, and select **Create merge request**. 1. In the new merge request, look for **Request to merge**. An entry similar to this displays: ```plaintext diff --git a/doc/topics/git/lfs/index.md b/doc/topics/git/lfs/index.md index 98710315652..731705fd72e 100644 --- a/doc/topics/git/lfs/index.md +++ b/doc/topics/git/lfs/index.md @@ -148,6 +148,12 @@ LFS object. Technical details about how this works can be found in the [development documentation for LFS](../../../development/lfs.md#including-lfs-blobs-in-project-archives). +## Related topics + +- Blog post: [Getting started with Git LFS](https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/) +- [Git LFS developer information](../../../development/lfs.md) +- [GitLab Git Large File Storage (LFS) Administration](../../../administration/lfs/index.md) for self-managed instances + ## Troubleshooting ### Encountered `n` files that should have been pointers, but weren't diff --git a/doc/topics/git/subtree.md b/doc/topics/git/subtree.md index 1d624b45348..a8a665d4e13 100644 --- a/doc/topics/git/subtree.md +++ b/doc/topics/git/subtree.md @@ -16,9 +16,7 @@ comments: false - Add: `git subtree add --prefix <target-folder> <url> <branch> --squash` - Pull: `git subtree pull --prefix <target-folder> <url> <branch> --squash` -- Push: `git subtree add --prefix <target-folder> <url> <branch>` -- Ex: `git config alias.sbp 'subtree pull --prefix st / - git@gitlab.com:balameb/subtree-nested-example.git master --squash'` +- Push: `git subtree push --prefix <target-folder> <url> <branch>` ```shell # Add an alias diff --git a/doc/topics/offline/quick_start_guide.md b/doc/topics/offline/quick_start_guide.md index bb2f651786c..015fd9fc720 100644 --- a/doc/topics/offline/quick_start_guide.md +++ b/doc/topics/offline/quick_start_guide.md @@ -205,3 +205,13 @@ users are on the most up-to-date instances of GitLab. These two services can be environments so that they do not attempt and fail to reach out to GitLab services. Learn more about [disabling usage statistics](../../user/admin_area/settings/usage_statistics.md#enable-or-disable-usage-statistics). + +### Configure NTP + +In GitLab 15.4 and 15.5, Gitaly Cluster doesn't function if `pool.ntp.org` is unreachable. +[Customize the time server setting](../../administration/gitaly/praefect.md#customize-time-server-setting) on the Gitaly +and Praefect servers so they can use an accessible NTP server. + +On offline instances, the [GitLab Geo check Rake task](../../administration/geo/replication/troubleshooting.md#can-geo-detect-the-current-site-correctly) +always fails because it uses `pool.ntp.org`. This error can be ignored but you can +[read more about how to work around it](../../administration/geo/replication/troubleshooting.md#message-machine-clock-is-synchronized--exception). diff --git a/doc/topics/plan_and_track.md b/doc/topics/plan_and_track.md index cf5dfe488db..862c41aa4d9 100644 --- a/doc/topics/plan_and_track.md +++ b/doc/topics/plan_and_track.md @@ -11,12 +11,16 @@ with milestones and track your team's time. Learn how to save time with quick actions, see how GitLab renders Markdown text, and learn how to use Git to interact with GitLab. +<!-- vale gitlab.Spelling = NO --> + <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For a thorough demo of Plan features, see [Multi-team planning with GitLab Ultimate](https://www.youtube.com/watch?v=KmASFwSap7c). In this video, Gabe describes a use case of a multi-team organization that uses GitLab with [Scaled Agile Framework (SAFe)](https://about.gitlab.com/solutions/agile-delivery/scaled-agile/). +<!-- vale gitlab.Spelling = YES --> + ## Basic workflow features Planning features everyone needs to use day-to-day. diff --git a/doc/tutorials/index.md b/doc/tutorials/index.md index 9b264a8c636..c1b538bafbe 100644 --- a/doc/tutorials/index.md +++ b/doc/tutorials/index.md @@ -19,10 +19,8 @@ and running quickly. | [GitLab 101](https://levelup.gitlab.com/learn/course/gitlab101) | Learn the basics of GitLab in this certification course. | **{star}** | | <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [Use GitLab for DevOps](https://www.youtube.com/watch?v=7q9Y1Cv-ib0) (12m 34s) | Use GitLab through the entire DevOps lifecycle, from planning to monitoring. | **{star}** | | [Use Markdown at GitLab](../user/markdown.md) | GitLab Flavored Markdown (GLFM) is used in many areas of GitLab, for example, in merge requests. | **{star}** | -| [GitLab 201](https://levelup.gitlab.com/learn/course/gitlab-201-certification) | Go beyond the basics to learn more about using GitLab for your work. | | | <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [Learn GitLab project walkthrough](https://www.youtube.com/watch?v=-oaI2WEKdI4&list=PL05JrBw4t0KofkHq4GZJ05FnNGa11PQ4d) (59m 2s) | Step through the tutorial-style issues in the **Learn GitLab** project. If you don't have this project, download [the export file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/vendor/project_templates/learn_gitlab_ultimate.tar.gz) and [import it to a new project](../user/project/settings/import_export.md#import-a-project-and-its-data). | | | [Productivity tips](https://about.gitlab.com/blog/2021/02/18/improve-your-gitlab-productivity-with-these-10-tips/) | Get tips to help make you a productive GitLab user. | | -| <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [Structure a multi-team organization](https://www.youtube.com/watch?v=KmASFwSap7c) (37m 37s) | Learn to use issues, milestones, epics, labels, and more to plan and manage your work. | | ## Use Git @@ -33,6 +31,7 @@ the most out of GitLab. |-------|-------------|--------------------| | [Make your first Git commit](make_your_first_git_commit.md) | Create a project, edit a file, and commit changes to a Git repository from the command line. | **{star}** | | [Start using Git on the command line](../gitlab-basics/start-using-git.md) | Learn how to set up Git, clone repositories, and work with branches. | **{star}** | +| [Take advantage of Git rebase](https://about.gitlab.com/blog/2022/10/06/take-advantage-of-git-rebase/)| Learn how to use the `rebase` command in your workflow. | | | [Git cheat sheet](https://about.gitlab.com/images/press/git-cheat-sheet.pdf) | Download a PDF of common Git commands. | | ## Plan your work in projects @@ -42,10 +41,9 @@ collaborating, and more. | Topic | Description | Good for beginners | |-------|-------------|--------------------| -| [Create a project from a template](https://gitlab.com/projects/new#create_from_template) | For hands-on learning, select **Sample GitLab Project** and create a project with example issues and merge requests. | **{star}** | +| [Create a project from a template](https://gitlab.com/projects/new#create_from_template) | Choose a project template and create a project with files to get you started. | | | [Migrate to GitLab](../user/project/import/index.md) | If you are coming to GitLab from another platform, you can import or convert your projects. | | | [Run an agile iteration](agile_sprint.md) | Use group, projects, and iterations to run an agile development iteration. | -| <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [Use GitLab for multi-team planning (SAFe)](https://www.youtube.com/watch?v=KmASFwSap7c) (37m 37s) | A use case of a multi-team organization that uses GitLab with [Scaled Agile Framework (SAFe)](https://about.gitlab.com/solutions/agile-delivery/scaled-agile/). | ## Use CI/CD pipelines @@ -87,6 +85,7 @@ GitLab can check your application for security vulnerabilities. | Topic | Description | Good for beginners | |-------|-------------|--------------------| | [Set up dependency scanning](https://about.gitlab.com/blog/2021/01/14/try-dependency-scanning/) | Try out dependency scanning, which checks for known vulnerabilities in dependencies. | **{star}** | +| [Get started with GitLab application security](../user/application_security/get-started-security.md) | Follow recommended steps to set up security tools. | | ## Work with a self-managed instance @@ -95,13 +94,12 @@ can help you manage and configure your instance. | Topic | Description | Good for beginners | |-------|-------------|--------------------| -| [Install GitLab](../install/index.md) | Install GitLab according to your requirements.| | +| [Install GitLab](../install/install_methods.md) | Install GitLab according to your requirements.| | | [Get started administering GitLab](../administration/get_started.md) | Configure your organization and its authentication, then secure, monitor, and back up GitLab. | | -| [Secure your instance](https://about.gitlab.com/blog/2020/05/20/gitlab-instance-security-best-practices/) | Implement security features for your instance. | | ## Integrate with GitLab -GitLab [integrates](../integration/index.md) with a number of third-party services, +GitLab [integrates](../user/project/integrations/index.md) with a number of third-party services, enabling you to work with those services directly from GitLab. | Topic | Description | Good for beginners | diff --git a/doc/tutorials/move_personal_project_to_a_group.md b/doc/tutorials/move_personal_project_to_a_group.md index ad86851393a..1431dc48d99 100644 --- a/doc/tutorials/move_personal_project_to_a_group.md +++ b/doc/tutorials/move_personal_project_to_a_group.md @@ -4,35 +4,29 @@ group: Tutorials info: For assistance with this tutorial, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. --- -# Move your personal project to a group **(FREE SAAS)** +# Tutorial: Move your personal project to a group **(FREE SAAS)** -This tutorial will show you how to move a personal project to a group. +If you created a project under a [personal namespace](../user/namespace/index.md), +you can perform common tasks, like managing issue and merge requests, +and using source control and CI/CD. -## Why is a group important? +However, at some point you might outgrow your personal project and +want to move your project to a group namespace instead. With a group namespace, you can: -In GitLab, you use [groups](../user/group/index.md) -to manage one or more related projects at the same time. -A group gives you some great benefits. For example, you can: - -- Manage permissions for your projects. -- View all of the issues and merge requests for the projects in the group. -- View all unique users in your namespace, across all projects. +- Give a group of users access to your project, rather than adding users one-by-one. +- View all issues and merge requests for all projects in the group. +- View all unique users in the group namespace, across all projects. - Manage usage quotas. -- Start a trial or upgrade to a paid tier. This option is important if you're +- Start a trial or upgrade to a paid subscription tier. This option is important if you're impacted by the [changes to user limits](https://about.gitlab.com/blog/2022/03/24/efficient-free-tier/), and need more users. -However, if you're working in a [personal project](../user/project/working_with_projects.md#view-personal-projects), -you can't use these features. Personal projects are created under your -[personal namespace](../user/namespace/index.md). They're not part of a group, -so you can't get any of the benefits and features of a group. - -But don't worry! You can move your existing personal project to a group. -The next steps show you how. +This tutorial shows you how to move your project from a personal namespace +to a group namespace. ## Steps -Here's an overview of what we're going to do: +Here's an overview of the steps: 1. [Create a group](#create-a-group). 1. [Move your project to a group](#move-your-project-to-a-group). @@ -59,8 +53,8 @@ If you don't have a group, create one: Before you move your project to a group: - You must have the Owner role for the project. -- Remove any [container images](../user/packages/container_registry/index.md#limitations) - and [NPM packages](../user/packages/npm_registry/index.md#limitations). +- Remove any [container images](../user/packages/container_registry/index.md#known-issues) +- Remove any npm packages. If you transfer a project to a different root namespace, the project must not contain any npm packages. When you update the path of a user or group, or transfer a subgroup or project, you must remove any npm packages first. You cannot update the root namespace of a project with npm packages. Make sure you update your .npmrc files to follow the naming convention and run npm publish if necessary. Now you're ready to move your project: diff --git a/doc/update/background_migrations.md b/doc/update/background_migrations.md new file mode 100644 index 00000000000..2e0bd1bb348 --- /dev/null +++ b/doc/update/background_migrations.md @@ -0,0 +1,461 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Check for background migrations before upgrading + +Certain releases may require different migrations to be +finished before you update to the newer version. + +There are two kinds of migrations: + +- [Background migrations](#background-migrations) +- [Batched background migrations](#batched-background-migrations) (available in GitLab 14.0 and later) + +Background migrations and batched migrations are not the same, so you should check that both are +complete before updating. + +Decrease the time required to complete these migrations by increasing the number of +[Sidekiq workers](../administration/sidekiq/extra_sidekiq_processes.md) +that can process jobs in the `background_migration` queue. + +## Background migrations + +### Pending migrations + +**For Omnibus installations:** + +```shell +sudo gitlab-rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' +sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.queued.count' +``` + +**For installations from source:** + +```shell +cd /home/git/gitlab +sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' +sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.queued.count' +``` + +### Failed migrations + +**For Omnibus installations:** + +For GitLab 14.0-14.9: + +```shell +sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.failed.count' +``` + +For GitLab 14.10 and later: + +```shell +sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:failed).count' +``` + +**For installations from source:** + +For GitLab 14.0-14.9: + +```shell +cd /home/git/gitlab +sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.failed.count' +``` + +For GitLab 14.10 and later: + +```shell +cd /home/git/gitlab +sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:failed).count' +``` + +## Batched background migrations **(FREE SELF)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/51332) in GitLab 13.11, [behind a feature flag](../user/feature_flags.md), disabled by default. +> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/329511) in GitLab 13.12. +> - Enabled on GitLab.com. +> - Recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-batched-background-migrations). + +There can be [risks when disabling released features](../administration/feature_flags.md#risks-when-disabling-released-features). +Refer to this feature's version history for more details. + +To update database tables in batches, GitLab can use batched background migrations. These migrations +are created by GitLab developers and run automatically on upgrade. However, such migrations are +limited in scope to help with migrating some `integer` database columns to `bigint`. This is needed to +prevent integer overflow for some tables. + +Some installations [may need to run GitLab 14.0 for at least a day](index.md#1400) to complete the database changes introduced by that upgrade. + +Batched background migrations are handled by Sidekiq and [run in isolation](../development/database/batched_background_migrations.md#isolation), so an instance can remain operational while the migrations are processed. However, there may be performance degradation on larger instances that are heavily used while batched background migrations are run, so it's a good idea to [actively monitor the Sidekiq status](../user/admin_area/index.md#background-jobs) until all migrations are completed. + +### Check the status of batched background migrations + +To check the status of batched background migrations: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Monitoring > Background Migrations**. + +  + +All migrations must have a `Finished` status before you upgrade GitLab. + +The status of batched background migrations can also be queried directly in the database. + +1. Log into a `psql` prompt according to the directions for your instance's installation method +(for example, `sudo gitlab-psql` for Omnibus installations). +1. Run the following query in the `psql` session to see details on incomplete batched background migrations: + + ```sql + select job_class_name, table_name, column_name, job_arguments from batched_background_migrations where status <> 3; + ``` + +If the migrations are not finished and you try to update to a later version, +GitLab prompts you with an error: + +```plaintext +Expected batched background migration for the given configuration to be marked as 'finished', but it is 'active': +``` + +If you get this error, [check the batched background migration options](#database-migrations-failing-because-of-batched-background-migration-not-finished) to complete the upgrade. + +### Enable or disable batched background migrations + +WARNING: +If you disable this feature flag, GitLab upgrades may fail. + +Batched background migrations are under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) +can opt to disable it. + +To enable it: + +```ruby +Feature.enable(:execute_batched_migrations_on_schedule) +``` + +To disable it: + +```ruby +Feature.disable(:execute_batched_migrations_on_schedule) +``` + +### Pause batched background migrations in GitLab 14.x + +To pause an ongoing batched background migration, use the `disable` command above. +This command causes the migration to complete the current batch, and then wait to start the next batch. + +Use the following database queries to see the state of the current batched background migration: + +1. Obtain the ID of the running migration: + + ```sql + SELECT + id, + job_class_name, + table_name, + column_name, + job_arguments + FROM batched_background_migrations + WHERE status <> 3; + ``` + +1. Run this query, replacing `XX` with the ID you obtained in the previous step, + to see the status of the migration: + + ```sql + SELECT + started_at, + finished_at, + finished_at - started_at AS duration, + min_value, + max_value, + batch_size, + sub_batch_size + FROM batched_background_migration_jobs + WHERE batched_background_migration_id = XX + ORDER BY id DESC + limit 10; + ``` + +1. Run the query multiple times within a few minutes to ensure no new row has been added. + If no new row has been added, the migration has been paused. + +1. After confirming the migration has paused, restart the migration (using the `enable` + command above) to proceed with the batch when ready. On larger instances, + background migrations can take as long as 48 hours to complete each batch. + +### Automatic batch size optimization + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60133) +> in GitLab 13.12, [behind a feature flag](../user/feature_flags.md), +> [enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/329511). +> - Enabled on GitLab.com. +> - Recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to +> [disable it](#enable-or-disable-automatic-batch-size-optimization). + +There can be [risks when disabling released features](../administration/feature_flags.md#risks-when-disabling-released-features). +Refer to this feature's version history for more details. + +To maximize throughput of batched background migrations (in terms of the number of tuples updated per time unit), batch sizes are automatically adjusted based on how long the previous batches took to complete. + +### Enable or disable automatic batch size optimization + +Automatic batch size optimization for batched background migrations is under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) +can opt to disable it. + +To enable it: + +```ruby +Feature.enable(:optimize_batched_migrations) +``` + +To disable it: + +```ruby +Feature.disable(:optimize_batched_migrations) +``` + +## Troubleshooting + +### Database migrations failing because of batched background migration not finished + +When updating to GitLab 14.2 or later there might be a database migration failing with a message like: + +```plaintext +StandardError: An error has occurred, all later migrations canceled: + +Expected batched background migration for the given configuration to be marked as 'finished', but it is 'active': + {:job_class_name=>"CopyColumnUsingBackgroundMigrationJob", :table_name=>"push_event_payloads", :column_name=>"event_id", :job_arguments=>[["event_id"], ["event_id_convert_to_bigint"]]} +``` + +First, check if you have followed the [version-specific upgrade instructions for 14.2](../update/index.md#1420). +If you have, you can [manually finish the batched background migration](#manually-finishing-a-batched-background-migration). +If you haven't, choose one of the following methods: + +1. [Rollback and upgrade](#roll-back-and-follow-the-required-upgrade-path) through one of the required +versions before updating to 14.2+. +1. [Roll forward](#roll-forward-and-finish-the-migrations-on-the-upgraded-version), staying on the current +version and manually ensuring that the batched migrations complete successfully. + +#### Roll back and follow the required upgrade path + +1. [Rollback and restore the previously installed version](../raketasks/backup_restore.md) +1. Update to either 14.0.5 or 14.1 **before** updating to 14.2+ +1. [Check the status](#check-the-status-of-batched-background-migrations) of the batched background migrations and +make sure they are all marked as finished before attempting to upgrade again. If any remain marked as active, +you can [manually finish them](#manually-finishing-a-batched-background-migration). + +#### Roll forward and finish the migrations on the upgraded version + +##### For a deployment with downtime + +To run all the batched background migrations, it can take a significant amount of time +depending on the size of your GitLab installation. + +1. [Check the status](#check-the-status-of-batched-background-migrations) of the batched background migrations in the +database, and [manually run them](#manually-finishing-a-batched-background-migration) with the appropriate +arguments until the status query returns no rows. +1. When the status of all of all them is marked as complete, re-run migrations for your installation. +1. [Complete the database migrations](../administration/raketasks/maintenance.md#run-incomplete-database-migrations) from your GitLab upgrade: + + ```plaintext + sudo gitlab-rake db:migrate + ``` + +1. Run a reconfigure: + + ```plaintext + sudo gitlab-ctl reconfigure + ``` + +1. Finish the upgrade for your installation. + +##### For a no-downtime deployment + +As the failing migrations are post-deployment migrations, you can remain on a running instance of the upgraded +version and wait for the batched background migrations to finish normally. + +1. [Check the status](#check-the-status-of-batched-background-migrations) of the batched background migration from +the error message, and make sure it is listed as finished. If it is still active, either wait until it is done, +or [manually finish it](#manually-finishing-a-batched-background-migration). +1. Re-run migrations for your installation, so the remaining post-deployment migrations finish. + +### Manually finishing a batched background migration + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62634) in GitLab 14.1 + +If you need to manually finish a batched background migration due to an +error, you can run: + +```shell +sudo gitlab-rake gitlab:background_migrations:finalize[<job_class_name>,<table_name>,<column_name>,'<job_arguments>'] +``` + +Replace the values in angle brackets with the correct +arguments. For example, if you receive an error similar to this: + +```plaintext +StandardError: An error has occurred, all later migrations canceled: + +Expected batched background migration for the given configuration to be marked as 'finished', but it is 'active': + {:job_class_name=>"CopyColumnUsingBackgroundMigrationJob", :table_name=>"push_event_payloads", :column_name=>"event_id", :job_arguments=>[["event_id"], ["event_id_convert_to_bigint"]]} +``` + +Plug the arguments from the error message into the command: + +```shell +sudo gitlab-rake gitlab:background_migrations:finalize[CopyColumnUsingBackgroundMigrationJob,push_event_payloads,event_id,'[["event_id"]\, ["event_id_convert_to_bigint"]]'] +``` + +If you need to manually run a batched background migration to continue an upgrade, you can +[check the status](#check-the-status-of-batched-background-migrations) in the database and get the +arguments from the query results. For example, if the query returns this: + +```plaintext + job_class_name | table_name | column_name | job_arguments +---------------------------------------+------------+-------------+------------------------------------ + CopyColumnUsingBackgroundMigrationJob | events | id | [["id"], ["id_convert_to_bigint"]] + ``` + +The results from the query can be plugged into the command: + +```shell +sudo gitlab-rake gitlab:background_migrations:finalize[CopyColumnUsingBackgroundMigrationJob,events,id,'[["id"]\, ["id_convert_to_bigint"]]'] +``` + +### The `BackfillNamespaceIdForNamespaceRoute` batched migration job fails + +In GitLab 14.8, the `BackfillNamespaceIdForNamespaceRoute` batched background migration job +may fail to complete. When retried, a `500 Server Error` is returned. This issue was +[resolved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82387) in GitLab 14.9. + +To resolve this issue, [upgrade GitLab](../update/index.md) from 14.8 to 14.9. +You can ignore the failed batch migration until after you update to GitLab 14.9. + +### Background migrations remain in the Sidekiq queue + +WARNING: +The following operations can disrupt your GitLab performance. They run a number of Sidekiq jobs that perform various database or file updates. + +Run the following check. If it returns non-zero and the count does not decrease over time, follow the rest of the steps in this section. + +```shell +# For Omnibus installations: +sudo gitlab-rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' + +# For installations from source: +cd /home/git/gitlab +sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' +``` + +It is safe to re-execute the following commands, especially if you have 1000+ pending jobs which would likely overflow your runtime memory. + +**For Omnibus installations** + +```shell +# Start the rails console +sudo gitlab-rails c + +# Execute the following in the rails console +scheduled_queue = Sidekiq::ScheduledSet.new +pending_job_classes = scheduled_queue.select { |job| job["class"] == "BackgroundMigrationWorker" }.map { |job| job["args"].first }.uniq +pending_job_classes.each { |job_class| Gitlab::BackgroundMigration.steal(job_class) } +``` + +**For installations from source** + +```shell +# Start the rails console +sudo -u git -H bundle exec rails RAILS_ENV=production + +# Execute the following in the rails console +scheduled_queue = Sidekiq::ScheduledSet.new +pending_job_classes = scheduled_queue.select { |job| job["class"] == "BackgroundMigrationWorker" }.map { |job| job["args"].first }.uniq +pending_job_classes.each { |job_class| Gitlab::BackgroundMigration.steal(job_class) } +``` + +### Background migrations stuck in 'pending' state + +WARNING: +The following operations can disrupt your GitLab performance. They run a number +of Sidekiq jobs that perform various database or file updates. + +- GitLab 13.6 introduced an issue where a background migration named + `BackfillJiraTrackerDeploymentType2` can be permanently stuck in a + **pending** state across upgrades. To clean up this stuck migration, see the + [13.6.0 version-specific instructions](index.md#1360). +- GitLab 14.2 introduced an issue where a background migration named + `BackfillDraftStatusOnMergeRequests` can be permanently stuck in a + **pending** state across upgrades when the instance lacks records that match + the migration's target. To clean up this stuck migration, see the + [14.2.0 version-specific instructions](index.md#1420). +- GitLab 14.4 introduced an issue where a background migration named + `PopulateTopicsTotalProjectsCountCache` can be permanently stuck in a + **pending** state across upgrades when the instance lacks records that match + the migration's target. To clean up this stuck migration, see the + [14.4.0 version-specific instructions](index.md#1440). +- GitLab 14.5 introduced an issue where a background migration named + `UpdateVulnerabilityOccurrencesLocation` can be permanently stuck in a + **pending** state across upgrades when the instance lacks records that match + the migration's target. To clean up this stuck migration, see the + [14.5.0 version-specific instructions](index.md#1450). +- GitLab 14.8 introduced an issue where a background migration named + `PopulateTopicsNonPrivateProjectsCount` can be permanently stuck in a + **pending** state across upgrades. To clean up this stuck migration, see the + [14.8.0 version-specific instructions](index.md#1480). +- GitLab 14.9 introduced an issue where a background migration named + `ResetDuplicateCiRunnersTokenValuesOnProjects` can be permanently stuck in a + **pending** state across upgrades when the instance lacks records that match + the migration's target. To clean up this stuck migration, see the + [14.9.0 version-specific instructions](index.md#1490). + +For other background migrations stuck in pending, run the following check. If +it returns non-zero and the count does not decrease over time, follow the rest +of the steps in this section. + +```shell +# For Omnibus installations: +sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigrationJob.pending.count' + +# For installations from source: +cd /home/git/gitlab +sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::BackgroundMigrationJob.pending.count' +``` + +It is safe to re-attempt these migrations to clear them out from a pending status: + +**For Omnibus installations** + +```shell +# Start the rails console +sudo gitlab-rails c + +# Execute the following in the rails console +Gitlab::Database::BackgroundMigrationJob.pending.find_each do |job| + puts "Running pending job '#{job.class_name}' with arguments #{job.arguments}" + result = Gitlab::BackgroundMigration.perform(job.class_name, job.arguments) + puts "Result: #{result}" +end +``` + +**For installations from source** + +```shell +# Start the rails console +sudo -u git -H bundle exec rails RAILS_ENV=production + +# Execute the following in the rails console +Gitlab::Database::BackgroundMigrationJob.pending.find_each do |job| + puts "Running pending job '#{job.class_name}' with arguments #{job.arguments}" + result = Gitlab::BackgroundMigration.perform(job.class_name, job.arguments) + puts "Result: #{result}" +end +``` diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index 353fbf191b5..ed49a4ade09 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -34,6 +34,9 @@ In each release, GitLab announces features that are deprecated and no longer rec Each deprecated feature will be removed in a future release. Some features cause breaking changes when they are removed. +**{rss}** **To be notified of upcoming breaking changes**, +add this URL to your RSS feed reader: `https://about.gitlab.com/breaking-changes.xml` + DISCLAIMER: This page contains information related to upcoming products, features, and functionality. It is important to note that the information presented is for informational purposes only. @@ -45,11 +48,77 @@ sole discretion of GitLab Inc. <div class="announcement-milestone"> -## Announced in 15.6 +## Announced in 15.7 <div class="deprecation removal-160 breaking-change"> -### Configuration fields in GitLab Runner Helm Chart +### DAST API scans using DAST template is deprecated + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +With the move to the new DAST API analyzer and the `DAST-API.gitlab-ci.yml` template for DAST API scans, we will be removing the ability to scan APIs with the DAST analyzer. Use of the `DAST.gitlab-ci.yml` or `DAST-latest.gitlab-ci.yml` templates for API scans is deprecated as of GitLab 15.7 and will no longer work in GitLab 16.0. Please use `DAST-API.gitlab-ci.yml` template and refer to the [DAST API analyzer](https://docs.gitlab.com/ee/user/application_security/dast_api/#configure-dast-api-with-an-openapi-specification) documentation for configuration details. + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### DAST API variables + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +With the switch to the new DAST API analyzer in GitLab 15.6, two legacy DAST API variables are being deprecated. The variables `DAST_API_HOST_OVERRIDE` and `DAST_API_SPECIFICATION` will no longer be used for DAST API scans. + +`DAST_API_HOST_OVERRIDE` has been deprecated in favor of using the `DAST_API_TARGET_URL` to automatically override the host in the OpenAPI specification. + +`DAST_API_SPECIFICATION` has been deprecated in favor of `DAST_API_OPENAPI`. To continue using an OpenAPI specification to guide the test, users must replace the `DAST_API_SPECIFICATION` variable with the `DAST_API_OPENAPI` variable. The value can remain the same, but the variable name must be replaced. + +These two variables will be removed in GitLab 16.0. + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### DAST ZAP advanced configuration variables deprecation + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +With the new browser-based DAST analyzer GA in GitLab 15.7, we are working towards making it the default DAST analyzer at some point in the future. In preparation for this, the following legacy DAST variables are being deprecated and scheduled for removal in GitLab 16.0: `DAST_ZAP_CLI_OPTIONS` and `DAST_ZAP_LOG_CONFIGURATION`. These variables allowed for advanced configuration of the legacy DAST analyzer, which was based on OWASP ZAP. The new browser-based analyzer will not include the same functionality, as these were specific to how ZAP worked. + +These three variables will be removed in GitLab 16.0. + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### DAST report variables deprecation + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +With the new browser-based DAST analyzer GA in GitLab 15.7, we are working towards making it the default DAST analyzer at some point in the future. In preparation for this, the following legacy DAST variables are being deprecated and scheduled for removal in GitLab 16.0: `DAST_HTML_REPORT`, `DAST_XML_REPORT`, and `DAST_MARKDOWN_REPORT`. These reports relied on the legacy DAST analyzer and we do not plan to implement them in the new browser-based analyzer. As of GitLab 16.0, these report artifacts will no longer be generated. + +These three variables will be removed in GitLab 16.0. + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### KAS Metrics Port in GitLab Helm Chart End of Support: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)<br /> Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) @@ -58,13 +127,14 @@ WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. -From GitLab 13.6, users can [specify any runner configuration in the GitLab Runner Helm chart](https://docs.gitlab.com/runner/install/kubernetes.html). When we implemented this feature, we deprecated values in the GitLab Helm Chart configuration that were specific to GitLab Runner. These fields are deprecated and we plan to remove them in v1.0 of the GitLab Runner Helm chart. +The `gitlab.kas.metrics.port` has been deprecated in favor of the new `gitlab.kas.observability.port` configuration field for the [GitLab Helm Chart](https://gitlab.com/gitlab-org/charts/gitlab/-/merge_requests/2839). +This port is used for much more than just metrics, which warranted this change to avoid confusion in configuration. </div> <div class="deprecation removal-160 breaking-change"> -### GitLab Runner registration token in Runner Operator +### Shimo integration End of Support: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)<br /> Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) @@ -73,13 +143,28 @@ WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. -The [`runner-registration-token`](https://docs.gitlab.com/runner/install/operator.html#install-the-kubernetes-operator) parameter that uses the OpenShift and k8s Vanilla Operator to install a runner on Kubernetes is deprecated. GitLab plans to introduce a new [GitLab Runner token architecture](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/) in GitLab 15.8, which introduces a new method for registering runners and eliminates the legacy runner registration token. +The [Shimo Workspace integration](https://docs.gitlab.com/ee/user/project/integrations/shimo.html) has been deprecated +and will be moved to the JiHu GitLab codebase. </div> <div class="deprecation removal-160 breaking-change"> -### `POST /api/v4/runners` method to register runners +### Single merge request changes API endpoint + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The endpoint to get [changes from a single merge request](https://docs.gitlab.com/ee/api/merge_requests.html#get-single-merge-request-changes) has been deprecated in favor the [list merge request diffs](https://docs.gitlab.com/ee/api/merge_requests.html#list-merge-request-diffs) endpoint. API users are encouraged to switch to the new diffs endpoint instead. The `changes from a single merge request` endpoint will be removed in v5 of the GitLab REST API. + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### Support for REST API endpoints that reset runner registration tokens End of Support: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)<br /> Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) @@ -88,10 +173,13 @@ WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. -The `POST` method operation on the `/api/v4/runners` endpoint is deprecated. -This endpoint and method [registers](https://docs.gitlab.com/ee/api/runners.html#register-a-new-runner) a runner -with a GitLab instance at the instance, group, or project level through the API. We plan to remove this endpoint -and method in GitLab 16.0. +The support for runner registration tokens is deprecated. As a consequence, the REST API endpoints to reset a registration token are also deprecated and will +be removed in GitLab 16.0. +The deprecated endpoints are: + +- `POST /runners/reset_registration_token` +- `POST /projects/:id/runners/reset_registration_token` +- `POST /groups/:id/runners/reset_registration_token` In GitLab 15.8, we plan to implement a new method to bind runners to a GitLab instance, as part of the new [GitLab Runner token architecture](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/). @@ -103,7 +191,46 @@ From GitLab 16.0 and later, the runner registration methods implemented by the n <div class="deprecation removal-160 breaking-change"> -### `gitlab-runner register` command +### Support for periods (`.`) in Terraform state names might break existing states + +End of Support: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)<br /> +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +Previously, Terraform state names containing periods were not supported. However, you could still use state names with periods via a workaround. + +GitLab 15.7 [adds full support](https://docs.gitlab.com/ee/user/infrastructure/iac/troubleshooting.html#state-not-found-if-the-state-name-contains-a-period) for state names that contain periods. If you used a workaround to handle these state names, your jobs might fail, or it might look like you've run Terraform for the first time. + +To resolve the issue: + + 1. Change any references to the state file by excluding the period and any characters that follow. + - For example, if your state name is `state.name`, change all references to `state`. + 1. Run your Terraform commands. + +To use the full state name, including the period, [migrate to the full state file](https://docs.gitlab.com/ee/user/infrastructure/iac/terraform_state.html#migrate-to-a-gitlab-managed-terraform-state). + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### The Phabricator task importer is deprecated + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The [Phabricator task importer](https://docs.gitlab.com/ee/user/project/import/phabricator.html) is being deprecated. Phabricator itself as a project is no longer actively maintained since June 1, 2021. We haven't observed imports using this tool. There has been no activity on the open related issues on GitLab. + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### The `gitlab-runner exec` command is deprecated End of Support: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)<br /> Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) @@ -112,21 +239,126 @@ WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. -The command to [register](https://docs.gitlab.com/runner/register/) a runner, `gitlab-runner register` is deprecated. +The [`gitlab-runner exec`](https://docs.gitlab.com/runner/commands/#gitlab-runner-exec) command is deprecated and will be fully removed from GitLab Runner in 16.0. The `gitlab-runner exec` feature was initially developed to provide the ability to validate a GitLab CI pipeline on a local system without needing to commit the updates to a GitLab instance. However, with the continued evolution of GitLab CI, replicating all GitLab CI features into `gitlab-runner exec` was no longer viable. Pipeline syntax and validation [simulation](https://docs.gitlab.com/ee/ci/pipeline_editor/#simulate-a-cicd-pipeline) are available in the GitLab pipeline editor. + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### ZenTao integration + +End of Support: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)<br /> +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The [ZenTao product integration](https://docs.gitlab.com/ee/user/project/integrations/zentao.html) has been deprecated +and will be moved to the JiHu GitLab codebase. + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### `POST ci/lint` API endpoint deprecated + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The `POST ci/lint` API endpoint is deprecated in 15.7, and will be removed in 16.0. This endpoint does not validate the full range of CI/CD configuration options. Instead, use [`POST /projects/:id/ci/lint`](https://docs.gitlab.com/15.5/ee/api/lint.html#validate-a-ci-yaml-configuration-with-a-namespace), which properly validates CI/CD configuration. + +</div> +</div> + +<div class="announcement-milestone"> + +## Announced in 15.6 + +<div class="deprecation removal-160 breaking-change"> + +### Configuration fields in GitLab Runner Helm Chart + +End of Support: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)<br /> +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +From GitLab 13.6, users can [specify any runner configuration in the GitLab Runner Helm chart](https://docs.gitlab.com/runner/install/kubernetes.html). When we implemented this feature, we deprecated values in the GitLab Helm Chart configuration that were specific to GitLab Runner. These fields are deprecated and we plan to remove them in v1.0 of the GitLab Runner Helm chart. + +</div> + +<div class="deprecation removal-170 breaking-change"> + +### GitLab Runner registration token in Runner Operator + +End of Support: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)<br /> +Planned removal: GitLab <span class="removal-milestone">17.0</span> (2024-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The [`runner-registration-token`](https://docs.gitlab.com/runner/install/operator.html#install-the-kubernetes-operator) parameter that uses the OpenShift and k8s Vanilla Operator to install a runner on Kubernetes is deprecated. GitLab plans to introduce a new [GitLab Runner token architecture](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/) in GitLab 15.8, which introduces a new method for registering runners and eliminates the legacy runner registration token. + +</div> + +<div class="deprecation removal-170 breaking-change"> + +### Registration tokens and server-side runner arguments in `POST /api/v4/runners` endpoint + +End of Support: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)<br /> +Planned removal: GitLab <span class="removal-milestone">17.0</span> (2024-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The support for registration tokens and certain runner configuration arguments in the `POST` method operation on the `/api/v4/runners` endpoint is deprecated. +This endpoint [registers](https://docs.gitlab.com/ee/api/runners.html#register-a-new-runner) a runner +with a GitLab instance at the instance, group, or project level through the API. We plan to remove the support for +registration tokens and certain configuration arguments in this endpoint in GitLab 17.0. + +In GitLab 15.8, we plan to implement a new method to bind runners to a GitLab instance, +as part of the new [GitLab Runner token architecture](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/). +This new architecture introduces a new method for registering runners and will eliminate the legacy +[runner registration token](https://docs.gitlab.com/ee/security/token_overview.html#runner-registration-tokens). +From GitLab 17.0 and later, the runner registration methods implemented by the new GitLab Runner token architecture will be the only supported methods. + +</div> + +<div class="deprecation removal-170 breaking-change"> + +### Registration tokens and server-side runner arguments in `gitlab-runner register` command + +End of Support: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)<br /> +Planned removal: GitLab <span class="removal-milestone">17.0</span> (2024-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The support for registration tokens and certain configuration arguments in the command to [register](https://docs.gitlab.com/runner/register/) a runner, `gitlab-runner register` is deprecated. GitLab plans to introduce a new [GitLab Runner token architecture](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/) in GitLab 15.8, which introduces a new method for registering runners and eliminates the legacy [runner registration token](https://docs.gitlab.com/ee/security/token_overview.html#runner-registration-tokens). -The new method will involve passing a [runner authentication token](https://docs.gitlab.com/ee/security/token_overview.html#runner-authentication-tokens-also-called-runner-tokens) -to a new `gitlab-runner deploy` command. +The new method will involve creating the runner in the GitLab UI and passing the +[runner authentication token](https://docs.gitlab.com/ee/security/token_overview.html#runner-authentication-tokens-also-called-runner-tokens) +to the `gitlab-runner register` command. </div> -<div class="deprecation removal-160 breaking-change"> +<div class="deprecation removal-170 breaking-change"> ### `runnerRegistrationToken` parameter for GitLab Runner Helm Chart End of Support: GitLab <span class="removal-milestone">16.0</span> (2023-05-22)<br /> -Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) +Planned removal: GitLab <span class="removal-milestone">17.0</span> (2024-05-22) WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). @@ -136,9 +368,9 @@ The [`runnerRegistrationToken`](https://docs.gitlab.com/runner/install/kubernete As part of the new [GitLab Runner token architecture](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/), in GitLab 15.8 we plan to introduce: -- A new method to bind runners to a GitLab instance. +- A new method to bind runners to a GitLab instance leveraging `runnerToken`. - A unique system ID saved to the `config.toml`, which will ensure traceability between jobs and runners. -From GitLab 16.0 and later, the methods to register runners introduced by the new GitLab Runner token architecture will be the only supported methods. +From GitLab 17.0 and later, the methods to register runners introduced by the new GitLab Runner token architecture will be the only supported methods. </div> @@ -165,7 +397,7 @@ The `merge_status` field in the [merge request API](https://docs.gitlab.com/ee/a ### File Type variable expansion in `.gitlab-ci.yml` -Planned removal: GitLab <span class="removal-milestone">15.7</span> () +Planned removal: GitLab <span class="removal-milestone">15.7</span> (2022-12-22) WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). @@ -269,7 +501,7 @@ GitLab's operational container scanning capabilities no longer require starboard ### Toggle behavior of `/draft` quick action in merge requests -Planned removal: GitLab <span class="removal-milestone">16.0</span> (2022-05-22) +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). @@ -904,7 +1136,7 @@ WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. -The feature flag `PUSH_RULES_SUPERSEDE_CODE_OWNERS` is being removed in GitLab 15.0. Upon its removal, push rules will supersede CODEOWNERS. The CODEOWNERS feature will no longer be available for access control. +The feature flag `PUSH_RULES_SUPERSEDE_CODE_OWNERS` is being removed in GitLab 15.0. Upon its removal, push rules will supersede Code Owners. Even if Code Owner approval is required, a push rule that explicitly allows a specific user to push code supersedes the Code Owners setting. </div> @@ -976,7 +1208,7 @@ To align with this change, API calls to list external status checks will also re ### GraphQL API Runner will not accept `status` filter values of `active` or `paused` -Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). @@ -999,7 +1231,7 @@ status value can be used in place of `active` since GitLab 14.8. ### GraphQL ID and GlobalID compatibility -Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-04-22) +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). @@ -1156,7 +1388,7 @@ The `instanceStatisticsMeasurements` GraphQL node has been renamed to `usageTren ### REST and GraphQL API Runner usage of `active` replaced by `paused` -Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). @@ -1481,7 +1713,7 @@ The new security approvals feature is similar to vulnerability check. For exampl ### `CI_BUILD_*` predefined variables -Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). @@ -1945,7 +2177,7 @@ Administrators who need to add runners for multiple projects can register a runn ### GraphQL API Runner status will not return `paused` -Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). diff --git a/doc/update/index.md b/doc/update/index.md index dbac4304897..d838f8dda34 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -73,210 +73,12 @@ from the chart version to GitLab version to determine the [upgrade path](#upgrad See the guide to [plan your GitLab upgrade](plan_your_upgrade.md). -## Checking for background migrations before upgrading +## Check for background migrations before upgrading Certain releases may require different migrations to be finished before you update to the newer version. -[Batched migrations](#batched-background-migrations) are a migration type available in GitLab 14.0 and later. -Background migrations and batched migrations are not the same, so you should check that both are -complete before updating. - -Decrease the time required to complete these migrations by increasing the number of -[Sidekiq workers](../administration/sidekiq/extra_sidekiq_processes.md) -that can process jobs in the `background_migration` queue. - -### Background migrations - -#### Pending migrations - -**For Omnibus installations:** - -```shell -sudo gitlab-rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' -sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.queued.count' -``` - -**For installations from source:** - -```shell -cd /home/git/gitlab -sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' -sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.queued.count' -``` - -#### Failed migrations - -**For Omnibus installations:** - -For GitLab 14.0-14.9: - -```shell -sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.failed.count' -``` - -For GitLab 14.10 and later: - -```shell -sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:failed).count' -``` - -**For installations from source:** - -For GitLab 14.0-14.9: - -```shell -cd /home/git/gitlab -sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.failed.count' -``` - -For GitLab 14.10 and later: - -```shell -cd /home/git/gitlab -sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:failed).count' -``` - -### Batched background migrations - -GitLab 14.0 introduced [batched background migrations](../user/admin_area/monitoring/background_migrations.md). - -Some installations [may need to run GitLab 14.0 for at least a day](#1400) to complete the database changes introduced by that upgrade. - -Batched background migrations are handled by Sidekiq and [run in isolation](../development/database/batched_background_migrations.md#isolation), so an instance can remain operational while the migrations are processed. However, there may be performance degradation on larger instances that are heavily used while batched background migrations are run, so it's a good idea to [actively monitor the Sidekiq status](../user/admin_area/index.md#background-jobs) until all migrations are completed. - -#### Check the status of batched background migrations - -To check the status of batched background migrations: - -1. On the top bar, select **Main menu > Admin**. -1. On the left sidebar, select **Monitoring > Background Migrations**. - -  - -All migrations must have a `Finished` status before you upgrade GitLab. - -The status of batched background migrations can also be queried directly in the database. - -1. Log into a `psql` prompt according to the directions for your instance's installation method -(for example, `sudo gitlab-psql` for Omnibus installations). -1. Run the following query in the `psql` session to see details on incomplete batched background migrations: - - ```sql - select job_class_name, table_name, column_name, job_arguments from batched_background_migrations where status <> 3; - ``` - -If the migrations are not finished and you try to update to a later version, -GitLab prompts you with an error: - -```plaintext -Expected batched background migration for the given configuration to be marked as 'finished', but it is 'active': -``` - -If you get this error, [check the batched background migration options](../user/admin_area/monitoring/background_migrations.md#database-migrations-failing-because-of-batched-background-migration-not-finished) to complete the upgrade. - -### What do you do if your background migrations are stuck? - -WARNING: -The following operations can disrupt your GitLab performance. They run a number of Sidekiq jobs that perform various database or file updates. - -#### Background migrations remain in the Sidekiq queue - -Run the following check. If it returns non-zero and the count does not decrease over time, follow the rest of the steps in this section. - -```shell -# For Omnibus installations: -sudo gitlab-rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' - -# For installations from source: -cd /home/git/gitlab -sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' -``` - -It is safe to re-execute the following commands, especially if you have 1000+ pending jobs which would likely overflow your runtime memory. - -**For Omnibus installations** - -```shell -# Start the rails console -sudo gitlab-rails c - -# Execute the following in the rails console -scheduled_queue = Sidekiq::ScheduledSet.new -pending_job_classes = scheduled_queue.select { |job| job["class"] == "BackgroundMigrationWorker" }.map { |job| job["args"].first }.uniq -pending_job_classes.each { |job_class| Gitlab::BackgroundMigration.steal(job_class) } -``` - -**For installations from source** - -```shell -# Start the rails console -sudo -u git -H bundle exec rails RAILS_ENV=production - -# Execute the following in the rails console -scheduled_queue = Sidekiq::ScheduledSet.new -pending_job_classes = scheduled_queue.select { |job| job["class"] == "BackgroundMigrationWorker" }.map { |job| job["args"].first }.uniq -pending_job_classes.each { |job_class| Gitlab::BackgroundMigration.steal(job_class) } -``` - -#### Background migrations stuck in 'pending' state - -GitLab 13.6 introduced an issue where a background migration named `BackfillJiraTrackerDeploymentType2` can be permanently stuck in a **pending** state across upgrades. To clean up this stuck migration, see the [13.6.0 version-specific instructions](#1360). - -GitLab 14.2 introduced an issue where a background migration named `BackfillDraftStatusOnMergeRequests` can be permanently stuck in a **pending** state across upgrades when the instance lacks records that match the migration's target. To clean up this stuck migration, see the [14.2.0 version-specific instructions](#1420). - -GitLab 14.4 introduced an issue where a background migration named `PopulateTopicsTotalProjectsCountCache` can be permanently stuck in a **pending** state across upgrades when the instance lacks records that match the migration's target. To clean up this stuck migration, see the [14.4.0 version-specific instructions](#1440). - -GitLab 14.5 introduced an issue where a background migration named `UpdateVulnerabilityOccurrencesLocation` can be permanently stuck in a **pending** state across upgrades when the instance lacks records that match the migration's target. To clean up this stuck migration, see the [14.5.0 version-specific instructions](#1450). - -GitLab 14.8 introduced an issue where a background migration named `PopulateTopicsNonPrivateProjectsCount` can be permanently stuck in a **pending** state across upgrades. To clean up this stuck migration, see the [14.8.0 version-specific instructions](#1480). - -GitLab 14.9 introduced an issue where a background migration named `ResetDuplicateCiRunnersTokenValuesOnProjects` can be permanently stuck in a **pending** state across upgrades when the instance lacks records that match the migration's target. To clean up this stuck migration, see the [14.9.0 version-specific instructions](#1490). - -For other background migrations stuck in pending, run the following check. If it returns non-zero and the count does not decrease over time, follow the rest of the steps in this section. - -```shell -# For Omnibus installations: -sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigrationJob.pending.count' - -# For installations from source: -cd /home/git/gitlab -sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::BackgroundMigrationJob.pending.count' -``` - -It is safe to re-attempt these migrations to clear them out from a pending status: - -**For Omnibus installations** - -```shell -# Start the rails console -sudo gitlab-rails c - -# Execute the following in the rails console -Gitlab::Database::BackgroundMigrationJob.pending.find_each do |job| - puts "Running pending job '#{job.class_name}' with arguments #{job.arguments}" - result = Gitlab::BackgroundMigration.perform(job.class_name, job.arguments) - puts "Result: #{result}" -end -``` - -**For installations from source** - -```shell -# Start the rails console -sudo -u git -H bundle exec rails RAILS_ENV=production - -# Execute the following in the rails console -Gitlab::Database::BackgroundMigrationJob.pending.find_each do |job| - puts "Running pending job '#{job.class_name}' with arguments #{job.arguments}" - result = Gitlab::BackgroundMigration.perform(job.class_name, job.arguments) - puts "Result: #{result}" -end -``` - -#### Batched migrations (GitLab 14.0 and later) - -See [troubleshooting batched background migrations](../user/admin_area/monitoring/background_migrations.md#troubleshooting). +For more information, see [background migrations](background_migrations.md). ## Dealing with running CI/CD pipelines and jobs @@ -360,7 +162,7 @@ A *major* upgrade requires the following steps: 1. Upgrade to the "dot zero" release of the next major version (`X.0.Z`). 1. Optional. Follow the [upgrade path](#upgrade-paths), and proceed with upgrading to newer releases of that major version. -It's also important to ensure that any [background migrations have been fully completed](#checking-for-background-migrations-before-upgrading) +It's also important to ensure that any [background migrations have been fully completed](background_migrations.md) before upgrading to a new major version. If you have enabled the [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) **(PREMIUM SELF)**, then @@ -384,7 +186,7 @@ Find where your version sits in the upgrade path below, and upgrade GitLab accordingly, while also consulting the [version-specific upgrade instructions](#version-specific-upgrading-instructions): -`8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> [`11.11.8`](#1200) -> `12.0.12` -> [`12.1.17`](#1210) -> [`12.10.14`](#12100) -> `13.0.14` -> [`13.1.11`](#1310) -> [`13.8.8`](#1388) -> [`13.12.15`](#13120) -> [`14.0.12`](#1400) -> [`14.3.6`](#1430) -> [`14.9.5`](#1490) -> [`14.10.Z`](#14100) -> [`15.0.Z`](#1500) -> [`15.1.Z`](#1510) (for GitLab instances with multiple web nodes) -> [`15.4.0`](#1540) -> [latest `15.Y.Z`](https://gitlab.com/gitlab-org/gitlab/-/releases) +`8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.0.13` -> `9.5.10` -> `10.0.7` -> `10.8.7` -> `11.0.6` -> [`11.11.8`](#1200) -> `12.0.12` -> [`12.1.17`](#1210) -> [`12.10.14`](#12100) -> `13.0.14` -> [`13.1.11`](#1310) -> [`13.8.8`](#1388) -> [`13.12.15`](#13120) -> [`14.0.12`](#1400) -> [`14.3.6`](#1430) -> [`14.9.5`](#1490) -> [`14.10.Z`](#14100) -> [`15.0.Z`](#1500) -> [`15.1.Z`](#1510) (for GitLab instances with multiple web nodes) -> [`15.4.0`](#1540) -> [latest `15.Y.Z`](https://gitlab.com/gitlab-org/gitlab/-/releases) NOTE: When not explicitly specified, upgrade GitLab to the latest available patch @@ -398,18 +200,18 @@ The following table, while not exhaustive, shows some examples of the supported upgrade paths. Additional steps between the mentioned versions are possible. We list the minimally necessary steps only. -| Target version | Your version | Supported upgrade path | Note | -| -------------- | ------------ | ---------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| `15.1.0` | `14.6.2` | `14.6.2` -> `14.9.5` -> `14.10.5` -> `15.0.2` -> `15.1.0` | Three intermediate versions are required: `14.9` and `14.10`, `15.0`, then `15.1.0`. | -| `15.0.0` | `14.6.2` | `14.6.2` -> `14.9.5` -> `14.10.5` -> `15.0.2` | Two intermediate versions are required: `14.9` and `14.10`, then `15.0.0`. | -| `14.6.2` | `13.10.2` | `13.10.2` -> `13.12.15` -> `14.0.12` -> `14.3.6` => `14.6.2` | Three intermediate versions are required: `13.12` and `14.0`, `14.3`, then `14.6.2`. | -| `14.1.8` | `13.9.2` | `13.9.2` -> `13.12.15` -> `14.0.12` -> `14.1.8` | Two intermediate versions are required: `13.12` and `14.0`, then `14.1.8`. | -| `13.12.15` | `12.9.2` | `12.9.2` -> `12.10.14` -> `13.0.14` -> `13.1.11` -> `13.8.8` -> `13.12.15` | Four intermediate versions are required: `12.10`, `13.0`, `13.1` and `13.8.8`, then `13.12.15`. | -| `13.2.10` | `11.5.0` | `11.5.0` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.10.14` -> `13.0.14` -> `13.1.11` -> `13.2.10` | Six intermediate versions are required: `11.11`, `12.0`, `12.1`, `12.10`, `13.0` and `13.1`, then `13.2.10`. | -| `12.10.14` | `11.3.4` | `11.3.4` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.10.14` | Three intermediate versions are required: `11.11`, `12.0` and `12.1`, then `12.10.14`. | -| `12.9.5` | `10.4.5` | `10.4.5` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.9.5` | Four intermediate versions are required: `10.8`, `11.11`, `12.0` and `12.1`, then `12.9.5`. | -| `12.2.5` | `9.2.6` | `9.2.6` -> `9.5.10` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.2.5` | Five intermediate versions are required: `9.5`, `10.8`, `11.11`, `12.0`, and `12.1`, then `12.2.5`. | -| `11.3.4` | `8.13.4` | `8.13.4` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> `11.3.4` | `8.17.7` is the last version in version 8, `9.5.10` is the last version in version 9, `10.8.7` is the last version in version 10. | +| Target version | Your version | Supported upgrade path | Note | +| -------------- | ------------ | ---------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| `15.1.0` | `14.6.2` | `14.6.2` -> `14.9.5` -> `14.10.5` -> `15.0.2` -> `15.1.0` | Three intermediate versions are required: `14.9`, `14.10`, and `15.0`. | +| `15.0.0` | `14.6.2` | `14.6.2` -> `14.9.5` -> `14.10.5` -> `15.0.2` | Two intermediate versions are required: `14.9` and `14.10`. | +| `14.6.2` | `13.10.2` | `13.10.2` -> `13.12.15` -> `14.0.12` -> `14.3.6` => `14.6.2` | Three intermediate versions are required: `13.12`, `14.0`, and `14.3`. | +| `14.1.8` | `13.9.2` | `13.9.2` -> `13.12.15` -> `14.0.12` -> `14.1.8` | Two intermediate versions are required: `13.12` and `14.0`. | +| `13.12.15` | `12.9.2` | `12.9.2` -> `12.10.14` -> `13.0.14` -> `13.1.11` -> `13.8.8` -> `13.12.15` | Four intermediate versions are required: `12.10`, `13.0`, `13.1`, and `13.8`. | +| `13.2.10` | `11.5.0` | `11.5.0` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.10.14` -> `13.0.14` -> `13.1.11` -> `13.2.10` | Six intermediate versions are required: `11.11`, `12.0`, `12.1`, `12.10`, `13.0`, and `13.1`. | +| `12.10.14` | `11.3.4` | `11.3.4` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.10.14` | Three intermediate versions are required: `11.11`, `12.0`, and `12.1`. | +| `12.9.5` | `10.4.5` | `10.4.5` -> `10.8.7` -> `11.0.6` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.9.5` | Five intermediate versions are required: `10.8`, `11.0`, `11.11`, `12.0`, and `12.1`. | +| `12.2.5` | `9.2.6` | `9.2.6` -> `9.5.10` -> `10.0.7` -> `10.8.7` -> `11.0.6` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.2.5` | Seven intermediate versions are required: `9.5`, `10.0`, `10.8`, `11.0`, `11.11`, `12.0`, and `12.1`. | +| `11.3.4` | `8.13.4` | `8.13.4` -> `8.17.7` -> `9.0.13` -> `9.5.10` -> `10.0.7` -> `10.8.7` -> `11.0.6` -> `11.3.4` | Six intermediate versions are required: `8.17`, `9.0`, `9.5`, `10.0`, `10.8`, and `11.0`. | ## Upgrading between editions @@ -471,13 +273,68 @@ NOTE: Specific information that follow related to Ruby and Git versions do not apply to [Omnibus installations](https://docs.gitlab.com/omnibus/) and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with appropriate Ruby and Git versions and are not using system binaries for Ruby and Git. There is no need to install Ruby or Git when utilizing these two approaches. +### 15.7.0 + +- This version validates a `NOT NULL DB` constraint on the `issues.work_item_type_id` column. + To upgrade to this version, no records with a `NULL` `work_item_type_id` should exist on the `issues` table. + There are multiple `BackfillWorkItemTypeIdForIssues` background migrations that will be finalized with + the `EnsureWorkItemTypeBackfillMigrationFinished` post-deploy migration. +- GitLab 15.4.0 introduced a [batched background migration](background_migrations.md#batched-background-migrations) to + [backfill `namespace_id` values on issues table](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91921). This + migration might take multiple hours or days to complete on larger GitLab instances. Please make sure the migration + has completed successfully before upgrading to 15.7.0. +- A database constraint is added, specifying that the `namespace_id` column on the issues + table has no `NULL` values. + + - If the `namespace_id` batched background migration from 15.4 failed (see above) then the 15.7 upgrade + fails with a database migration error. + + - On GitLab instances with large issues tables, validating this constraint causes the upgrade to take + longer than usual. All database changes need to complete within a one-hour limit: + + ```plaintext + FATAL: Mixlib::ShellOut::CommandTimeout: rails_migration[gitlab-rails] + [..] + Mixlib::ShellOut::CommandTimeout: Command timed out after 3600s: + ``` + + A workaround exists to [complete the data change and the upgrade manually](package/index.md#mixlibshelloutcommandtimeout-rails_migrationgitlab-rails--command-timed-out-after-3600s). +- The default Sidekiq `max_concurrency` has been changed to 20. This is now + consistent in our documentation and product defaults. + + For example, previously: + - Omnibus GitLab default (`sidekiq['max_concurrency']`): 50 + - From source installation default: 50 + - Helm chart default (`gitlab.sidekiq.concurrency`): 25 + + Reference architectures still use a default of 10 as this is set specifically + for those configurations. + + Sites that have configured `max_concurrency` will not be affected by this change. + [Read more about the Sidekiq concurrency setting](../administration/sidekiq/extra_sidekiq_processes.md#concurrency). + ### 15.6.0 +- You should use one of the [officially supported PostgreSQL versions](../administration/package_information/postgresql_versions.md). Some database migrations can cause stability and performance issues with older PostgreSQL versions. - Git 2.37.0 and later is required by Gitaly. For installations from source, we recommend you use the [Git version provided by Gitaly](../install/installation.md#git). +- A database change to modify the behavior of four indexes fails on instances + where these indexes do not exist: + + ```plaintext + Caused by: + PG::UndefinedTable: ERROR: relation "index_issues_on_title_trigram" does not exist + ``` + + The other three indexes are: `index_merge_requests_on_title_trigram`, `index_merge_requests_on_description_trigram`, + and `index_issues_on_description_trigram`. + + This issue was [fixed in GitLab 15.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105375) and backported + to GitLab 15.6.2. The issue can also be worked around: + [read about how to create these indexes](https://gitlab.com/gitlab-org/gitlab/-/issues/378343#note_1199863087). ### 15.5.0 -- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/extra_sidekiq_routing.md) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/extra_sidekiq_processes.md#queue-selector), this will cause [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. +- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/extra_sidekiq_routing.md) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/processing_specific_job_classes.md#queue-selectors), this will cause [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. - The default routing rule has been reverted in 15.5.4, so upgrading to that version or later will return to the previous behavior. - If a GitLab instance now listens only to the `default` queue (which is not currently recommended), it will be required to add this routing rule back in `/etc/gitlab/gitlab.rb`: @@ -485,12 +342,22 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap sidekiq['routing_rules'] = [['*', 'default']] ``` +### 15.4.1 + +A [license caching issue](https://gitlab.com/gitlab-org/gitlab/-/issues/376706) prevents some premium features of GitLab from working correctly if you add a new license. Workarounds for this issue: + +- Restart all Rails, Sidekiq and Gitaly nodes after applying a new license. This clears the relevant license caches and allows all premium features to operate correctly. +- Upgrade to a version that is not impacted by this issue. The following upgrade paths are available for impacted versions: + - 15.2.5 --> 15.3.5 + - 15.3.0 - 15.3.4 --> 15.3.5 + - 15.4.1 --> 15.4.3 + ### 15.4.0 -- GitLab 15.4.0 includes a [batched background migration](#batched-background-migrations) to [remove incorrect values from `expire_at` in `ci_job_artifacts` table](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89318). +- GitLab 15.4.0 includes a [batched background migration](background_migrations.md#batched-background-migrations) to [remove incorrect values from `expire_at` in `ci_job_artifacts` table](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89318). This migration might take hours or days to complete on larger GitLab instances. - By default, Gitaly and Praefect nodes use the time server at `pool.ntp.org`. If your instance can not connect to `pool.ntp.org`, [configure the `NTP_HOST` variable](../administration/gitaly/praefect.md#customize-time-server-setting). -- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/extra_sidekiq_routing.md) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/extra_sidekiq_processes.md#queue-selector), this will cause [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. +- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/extra_sidekiq_routing.md) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/processing_specific_job_classes.md#queue-selectors), this will cause [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. - The default routing rule has been reverted in 15.4.5, so upgrading to that version or later will return to the previous behavior. - If a GitLab instance now listens only to the `default` queue (which is not currently recommended), it will be required to add this routing rule back in `/etc/gitlab/gitlab.rb`: @@ -498,15 +365,83 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap sidekiq['routing_rules'] = [['*', 'default']] ``` +- New Git repositories created in Gitaly cluster [no longer use the `@hashed` storage path](#change-to-praefect-generated-replica-paths-in-gitlab-153). Server + hooks for new repositories must be copied into a different location. +- The structure of `/etc/gitlab/gitlab-secrets.json` was modified in [GitLab 15.4](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/6310), + and new configuration was added to `gitlab_pages`, `grafana`, and `mattermost` sections. + In a highly available or GitLab Geo environment, secrets need to be the same on all nodes. + If you're manually syncing the secrets file across nodes, or manually specifying secrets in + `/etc/gitlab/gitlab.rb`, make sure `/etc/gitlab/gitlab-secrets.json` is the same on all nodes. +- GitLab 15.4.0 introduced a [batched background migration](background_migrations.md#batched-background-migrations) to + [backfill `namespace_id` values on issues table](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91921). This + migration might take multiple hours or days to complete on larger GitLab instances. Please make sure the migration + has completed successfully before upgrading to 15.7.0 or later. + +### 15.3.4 + +A [license caching issue](https://gitlab.com/gitlab-org/gitlab/-/issues/376706) prevents some premium features of GitLab from working correctly if you add a new license. Workarounds for this issue: + +- Restart all Rails, Sidekiq and Gitaly nodes after applying a new license. This clears the relevant license caches and allows all premium features to operate correctly. +- Upgrade to a version that is not impacted by this issue. The following upgrade paths are available for impacted versions: + - 15.2.5 --> 15.3.5 + - 15.3.0 - 15.3.4 --> 15.3.5 + - 15.4.1 --> 15.4.3 + ### 15.3.3 - In GitLab 15.3.3, [SAML Group Links](../api/groups.md#saml-group-links) API `access_level` attribute type changed to `integer`. See -[valid access levels](../api/members.md#valid-access-levels) documentation. +[the API documentation](../api/members.md). +- A [license caching issue](https://gitlab.com/gitlab-org/gitlab/-/issues/376706) prevents some premium features of GitLab from working correctly if you add a new license. Workarounds for this issue: + + - Restart all Rails, Sidekiq and Gitaly nodes after applying a new license. This clears the relevant license caches and allows all premium features to operate correctly. + - Upgrade to a version that is not impacted by this issue. The following upgrade paths are available for impacted versions: + - 15.2.5 --> 15.3.5 + - 15.3.0 - 15.3.4 --> 15.3.5 + - 15.4.1 --> 15.4.3 + +### 15.3.2 + +A [license caching issue](https://gitlab.com/gitlab-org/gitlab/-/issues/376706) prevents some premium features of GitLab from working correctly if you add a new license. Workarounds for this issue: + +- Restart all Rails, Sidekiq and Gitaly nodes after applying a new license. This clears the relevant license caches and allows all premium features to operate correctly. +- Upgrade to a version that is not impacted by this issue. The following upgrade paths are available for impacted versions: + - 15.2.5 --> 15.3.5 + - 15.3.0 - 15.3.4 --> 15.3.5 + - 15.4.1 --> 15.4.3 + +### 15.3.1 + +A [license caching issue](https://gitlab.com/gitlab-org/gitlab/-/issues/376706) prevents some premium features of GitLab from working correctly if you add a new license. Workarounds for this issue: + +- Restart all Rails, Sidekiq and Gitaly nodes after applying a new license. This clears the relevant license caches and allows all premium features to operate correctly. +- Upgrade to a version that is not impacted by this issue. The following upgrade paths are available for impacted versions: + - 15.2.5 --> 15.3.5 + - 15.3.0 - 15.3.4 --> 15.3.5 + - 15.4.1 --> 15.4.3 ### 15.3.0 - [Incorrect deletion of object storage files on Geo secondary sites](https://gitlab.com/gitlab-org/gitlab/-/issues/371397) can occur in certain situations. See [Geo: Incorrect object storage LFS file deletion on secondary site issue in GitLab 15.0.0 to 15.3.2](#geo-incorrect-object-storage-lfs-file-deletion-on-secondary-sites-in-gitlab-1500-to-1532). - LFS transfers can [redirect to the primary from secondary site mid-session](https://gitlab.com/gitlab-org/gitlab/-/issues/371571) causing failed pull and clone requests when [Geo proxying](../administration/geo/secondary_proxy/index.md) is enabled. Geo proxying is enabled by default in GitLab 15.1 and later. See [Geo: LFS transfer redirect to primary from secondary site mid-session issue in GitLab 15.1.0 to 15.3.2](#geo-lfs-transfers-redirect-to-primary-from-secondary-site-mid-session-in-gitlab-1510-to-1532) for more details. +- New Git repositories created in Gitaly cluster [no longer use the `@hashed` storage path](#change-to-praefect-generated-replica-paths-in-gitlab-153). Server + hooks for new repositories must be copied into a different location. +- A [license caching issue](https://gitlab.com/gitlab-org/gitlab/-/issues/376706) prevents some premium features of GitLab from working correctly if you add a new license. Workarounds for this issue: + + - Restart all Rails, Sidekiq and Gitaly nodes after applying a new license. This clears the relevant license caches and allows all premium features to operate correctly. + - Upgrade to a version that is not impacted by this issue. The following upgrade paths are available for impacted versions: + - 15.2.5 --> 15.3.5 + - 15.3.0 - 15.3.4 --> 15.3.5 + - 15.4.1 --> 15.4.3 + +### 15.2.5 + +A [license caching issue](https://gitlab.com/gitlab-org/gitlab/-/issues/376706) prevents some premium features of GitLab from working correctly if you add a new license. Workarounds for this issue: + +- Restart all Rails, Sidekiq and Gitaly nodes after applying a new license. This clears the relevant license caches and allows all premium features to operate correctly. +- Upgrade to a version that is not impacted by this issue. The following upgrade paths are available for impacted versions: + - 15.2.5 --> 15.3.5 + - 15.3.0 - 15.3.4 --> 15.3.5 + - 15.4.1 --> 15.4.3 ### 15.2.0 @@ -514,7 +449,7 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap [upgraded to 15.1](#1510) before upgrading to 15.2 (and later) due to a configuration change in Rails that can result in inconsistent ETag key generation. -- Some Sidekiq workers were renamed in this release. To avoid any disruption, [run the Rake tasks to migrate any pending jobs](../administration/sidekiq/sidekiq_job_migration.md#future-jobs) before starting the upgrade to GitLab 15.2.0. +- Some Sidekiq workers were renamed in this release. To avoid any disruption, [run the Rake tasks to migrate any pending jobs](../administration/sidekiq/sidekiq_job_migration.md#migrate-queued-and-future-jobs) before starting the upgrade to GitLab 15.2.0. - Gitaly now executes its binaries in a [runtime location](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4670). By default on Omnibus GitLab, this path is `/var/opt/gitlab/gitaly/run/`. If this location is mounted with `noexec`, merge requests generate the following error: @@ -564,6 +499,8 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap than `<custom_hooks_dir>/<hook_name>`. - [Incorrect deletion of object storage files on Geo secondary sites](https://gitlab.com/gitlab-org/gitlab/-/issues/371397) can occur in certain situations. See [Geo: Incorrect object storage LFS file deletion on secondary site issue in GitLab 15.0.0 to 15.3.2](#geo-incorrect-object-storage-lfs-file-deletion-on-secondary-sites-in-gitlab-1500-to-1532). - The `FF_GITLAB_REGISTRY_HELPER_IMAGE` [feature flag](../administration/feature_flags.md#enable-or-disable-the-feature) is removed and helper images are always pulled from GitLab Registry. +- The `AES256-GCM-SHA384` SSL cipher is no longer allowed by NGINX. + See how you can [add the cipher back](https://docs.gitlab.com/omnibus/update/gitlab_15_changes.html#aes256-gcm-sha384-ssl-cipher-no-longer-allowed-by-default-by-nginx) to the allow list. ### 14.10.0 @@ -591,11 +528,11 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap ### 14.9.0 - Database changes made by the upgrade to GitLab 14.9 can take hours or days to complete on larger GitLab instances. - These [batched background migrations](#batched-background-migrations) update whole database tables to ensure corresponding + These [batched background migrations](background_migrations.md#batched-background-migrations) update whole database tables to ensure corresponding records in `namespaces` table for each record in `projects` table. After you update to 14.9.0 or a later 14.9 patch version, - [batched background migrations must finish](#batched-background-migrations) + [batched background migrations must finish](background_migrations.md#batched-background-migrations) before you update to a later version. If the migrations are not finished and you try to update to a later version, @@ -671,7 +608,7 @@ that may remain stuck permanently in a **pending** state. [an issue with job retries](https://gitlab.com/gitlab-org/gitlab/-/issues/357822), first upgrade to GitLab 14.7.x and make sure all batched migrations have finished. - If upgrading from version 14.3.0 or later, you might notice a failed - [batched migration](../user/admin_area/monitoring/background_migrations.md) named + [batched migration](background_migrations.md#batched-background-migrations) named `BackfillNamespaceIdForNamespaceRoute`. You can [ignore](https://gitlab.com/gitlab-org/gitlab/-/issues/357822) this. Retry it after you upgrade to version 14.9.x. - If you run external PostgreSQL, particularly AWS RDS, @@ -794,7 +731,7 @@ that may remain stuck permanently in a **pending** state when the instance lacks ### 14.3.0 - [Instances running 14.0.0 - 14.0.4 should not upgrade directly to GitLab 14.2 or later](#upgrading-to-later-14y-releases). -- Ensure [batched background migrations finish](#batched-background-migrations) before upgrading +- Ensure [batched background migrations finish](background_migrations.md#batched-background-migrations) before upgrading to 14.3.Z from earlier GitLab 14 releases. - Ruby 2.7.4 is required. Refer to [the Ruby installation instructions](../install/installation.md#2-ruby) for how to proceed. @@ -827,15 +764,109 @@ for how to proceed. gitlab-psql`): ```sql - select count(*) from background_migration_jobs where class_name = 'MigrateMergeRequestDiffCommitUsers' and status = 0; + select status, count(*) from background_migration_jobs + where class_name = 'MigrateMergeRequestDiffCommitUsers' group by status; + ``` + + As jobs are completed, the database records change from `0` (pending) to `1`. If the number of + pending jobs doesn't decrease after a while, it's possible that the + `MigrateMergeRequestDiffCommitUsers` background migration jobs have failed. You + can check for errors in the Sidekiq logs: + + ```shell + sudo grep MigrateMergeRequestDiffCommitUsers /var/log/gitlab/sidekiq/current | grep -i error ``` + If needed, you can attempt to run the `MigrateMergeRequestDiffCommitUsers` background + migration jobs manually in the [GitLab Rails Console](../administration/operations/rails_console.md). + This can be done using Sidekiq asynchronously, or by using a Rails process directly: + + - Using Sidekiq to schedule jobs asynchronously: + + ```ruby + # For the first run, only attempt to execute 1 migration. If successful, increase + # the limit for subsequent runs + limit = 1 + + jobs = Gitlab::Database::BackgroundMigrationJob.for_migration_class('MigrateMergeRequestDiffCommitUsers').pending.to_a + + pp "#{jobs.length} jobs remaining" + + jobs.first(limit).each do |job| + BackgroundMigrationWorker.perform_in(5.minutes, 'MigrateMergeRequestDiffCommitUsers', job.arguments) + end + ``` + + NOTE: + The queued jobs can be monitored using Sidekiq's admin panel, which can be accessed at the `/admin/sidekiq` endpoint URI. + + - Using a Rails process to run jobs synchronously: + + ```ruby + def process(concurrency: 1) + queue = Queue.new + + Gitlab::Database::BackgroundMigrationJob + .where(class_name: 'MigrateMergeRequestDiffCommitUsers', status: 0) + .each { |job| queue << job } + + concurrency + .times + .map do + Thread.new do + Thread.abort_on_exception = true + + loop do + job = queue.pop(true) + time = Benchmark.measure do + Gitlab::BackgroundMigration::MigrateMergeRequestDiffCommitUsers + .new + .perform(*job.arguments) + end + + puts "#{job.id} finished in #{time.real.round(2)} seconds" + rescue ThreadError + break + end + end + end + .each(&:join) + end + + ActiveRecord::Base.logger.level = Logger::ERROR + process + ``` + + NOTE: + When using Rails to execute these background migrations synchronously, make sure that the machine running the process has sufficient resources to handle the task. If the process gets terminated, it's likely due to insufficient memory available. If your SSH session times out after a while, it might be necessary to run the previous code by using a terminal multiplexer like `screen` or `tmux`. + - See [Maintenance mode issue in GitLab 13.9 to 14.4](#maintenance-mode-issue-in-gitlab-139-to-144). +- You may see the following error when setting up two factor authentication (2FA) for accounts + that authenticate using an LDAP password: + + ```plaintext + You must provide a valid current password + ``` + + - The error occurs because verification is incorrectly performed against accounts' + randomly generated internal GitLab passwords, not the LDAP passwords. + - This is [fixed in GitLab 14.5.0 and backported to 14.4.3](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73538). + - Workarounds: + - Instead of upgrading to GitLab 14.3.x to comply with the supported upgrade path: + 1. Upgrade to 14.4.5. + 1. Make sure the [`MigrateMergeRequestDiffCommitUsers` background migration](#1430) has finished. + 1. Upgrade to GitLab 14.5 or later. + - Reset the random password for affected accounts, using [the Rake task](../security/reset_user_password.md#use-a-rake-task): + + ```plaintext + sudo gitlab-rake "gitlab:password:reset[user_handle]" + ``` + ### 14.2.0 - [Instances running 14.0.0 - 14.0.4 should not upgrade directly to GitLab 14.2 or later](#upgrading-to-later-14y-releases). -- Ensure [batched background migrations finish](#batched-background-migrations) before upgrading +- Ensure [batched background migrations finish](background_migrations.md#batched-background-migrations) before upgrading to 14.2.Z from earlier GitLab 14 releases. - GitLab 14.2.0 contains background migrations to [address Primary Key overflow risk for tables with an integer PK](https://gitlab.com/groups/gitlab-org/-/epics/4785) for the tables listed below: - [`ci_build_needs`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65216) @@ -879,7 +910,7 @@ for how to proceed. It is not required for instances already running 14.0.5 (or later) to stop at 14.1.Z. 14.1 is included on the upgrade path for the broadest compatibility with self-managed installations, and ensure 14.0.0-14.0.4 installations do not - encounter issues with [batched background migrations](#batched-background-migrations). + encounter issues with [batched background migrations](background_migrations.md#batched-background-migrations). - Upgrading to GitLab [14.5](#1450) (or later) may take a lot longer if you do not upgrade to at least 14.1 first. The 14.1 merge request diff commits database migration can take hours to run, but runs in the @@ -901,14 +932,14 @@ Prerequisites: Long running batched background database migrations: - Database changes made by the upgrade to GitLab 14.0 can take hours or days to complete on larger GitLab instances. - These [batched background migrations](#batched-background-migrations) update whole database tables to mitigate primary key overflow and must be finished before upgrading to GitLab 14.2 or later. + These [batched background migrations](background_migrations.md#batched-background-migrations) update whole database tables to mitigate primary key overflow and must be finished before upgrading to GitLab 14.2 or later. - Due to an issue where `BatchedBackgroundMigrationWorkers` were [not working](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/2785#note_614738345) for self-managed instances, a [fix was created](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65106) that requires an update to at least 14.0.5. The fix was also released in [14.1.0](#1410). After you update to 14.0.5 or a later 14.0 patch version, - [batched background migrations must finish](#batched-background-migrations) + [batched background migrations must finish](background_migrations.md#batched-background-migrations) before you update to a later version. If the migrations are not finished and you try to update to a later version, @@ -918,7 +949,7 @@ Long running batched background database migrations: Expected batched background migration for the given configuration to be marked as 'finished', but it is 'active': ``` - See how to [resolve this error](../user/admin_area/monitoring/background_migrations.md#database-migrations-failing-because-of-batched-background-migration-not-finished). + See how to [resolve this error](background_migrations.md#database-migrations-failing-because-of-batched-background-migration-not-finished). Other issues: @@ -933,11 +964,11 @@ Other issues: #### Upgrading to later 14.Y releases - Instances running 14.0.0 - 14.0.4 should not upgrade directly to GitLab 14.2 or later, - because of [batched background migrations](#batched-background-migrations). + because of [batched background migrations](background_migrations.md#batched-background-migrations). 1. Upgrade first to either: - 14.0.5 or a later 14.0.Z patch release. - 14.1.0 or a later 14.1.Z patch release. - 1. [Batched background migrations must finish](#batched-background-migrations) + 1. [Batched background migrations must finish](background_migrations.md#batched-background-migrations) before you update to a later version [and may take longer than usual](#1400). ### 13.12.0 @@ -1040,7 +1071,7 @@ See [Maintenance mode issue in GitLab 13.9 to 14.4](#maintenance-mode-issue-in-g ### 13.8.8 -GitLab 13.8 includes a background migration to address [an issue with duplicate service records](https://gitlab.com/gitlab-org/gitlab/-/issues/290008). If duplicate services are present, this background migration must complete before a unique index is applied to the services table, which was [introduced in GitLab 13.9](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52563). Upgrades from GitLab 13.8 and earlier to later versions must include an intermediate upgrade to GitLab 13.8.8 and [must wait until the background migrations complete](#checking-for-background-migrations-before-upgrading) before proceeding. +GitLab 13.8 includes a background migration to address [an issue with duplicate service records](https://gitlab.com/gitlab-org/gitlab/-/issues/290008). If duplicate services are present, this background migration must complete before a unique index is applied to the services table, which was [introduced in GitLab 13.9](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52563). Upgrades from GitLab 13.8 and earlier to later versions must include an intermediate upgrade to GitLab 13.8.8 and [must wait until the background migrations complete](background_migrations.md) before proceeding. If duplicate services are still present, an upgrade to 13.9.x or later results in a failed upgrade with the following error: @@ -1157,9 +1188,14 @@ any downgrades would result to all sessions being invalidated and users are logg ### 12.1.0 -If you are planning to upgrade from `12.0.Z` to `12.10.Z`, it is necessary to -perform an intermediary upgrade to `12.1.Z` before upgrading to `12.10.Z` to -avoid issues like [#215141](https://gitlab.com/gitlab-org/gitlab/-/issues/215141). +- If you are planning to upgrade from `12.0.Z` to `12.10.Z`, it is necessary to + perform an intermediary upgrade to `12.1.Z` before upgrading to `12.10.Z` to + avoid issues like [#215141](https://gitlab.com/gitlab-org/gitlab/-/issues/215141). + +- Support for MySQL was removed in GitLab 12.1. Existing users using GitLab with + MySQL/MariaDB should + [migrate to PostgreSQL](https://gitlab.com/gitlab-org/gitlab/-/blob/v15.6.0-ee/doc/update/mysql_to_postgresql.md) + before upgrading. ### 12.0.0 @@ -1181,6 +1217,19 @@ After upgraded to 11.11.8 you can safely upgrade to 12.0.Z. See our [documentation on upgrade paths](../policy/maintenance.md#upgrade-recommendations) for more information. +### Change to Praefect-generated replica paths in GitLab 15.3 + +New Git repositories created in Gitaly cluster no longer use the `@hashed` storage path. + +Praefect now generates replica paths for use by Gitaly cluster. +This change is a pre-requisite for Gitaly cluster atomically creating, deleting, and +renaming Git repositories. + +To identify the replica path, [query the Praefect repository metadata](../administration/gitaly/troubleshooting.md#view-repository-metadata) +and pass the `@hashed` storage path to `-relative-path`. + +With this information, you can correctly install [server hooks](../administration/server_hooks.md). + ### Maintenance mode issue in GitLab 13.9 to 14.4 When [Maintenance mode](../administration/maintenance_mode/index.md) is enabled, users cannot sign in with SSO, SAML, or LDAP. @@ -1232,9 +1281,4 @@ This issue is resolved in GitLab 15.3.3, so customers with the following configu ## Miscellaneous -- [MySQL to PostgreSQL](mysql_to_postgresql.md) guides you through migrating - your database from MySQL to PostgreSQL. -- [Restoring from backup after a failed upgrade](restore_after_failure.md) -- [Upgrading PostgreSQL Using Slony](upgrading_postgresql_using_slony.md), for - upgrading a PostgreSQL database with minimal downtime. - [Managing PostgreSQL extensions](../install/postgresql_extensions.md) diff --git a/doc/update/mysql_to_postgresql.md b/doc/update/mysql_to_postgresql.md index 14914eee27c..ad36a9ff534 100644 --- a/doc/update/mysql_to_postgresql.md +++ b/doc/update/mysql_to_postgresql.md @@ -2,307 +2,10 @@ stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-02-28' +redirect_to: 'index.md' --- -# Migrating from MySQL to PostgreSQL **(FREE SELF)** +# Migrating from MySQL to PostgreSQL (removed) **(FREE SELF)** -This guide documents how to take a working GitLab instance that uses MySQL and -migrate it to a PostgreSQL database. - -## Requirements - -NOTE: -Support for MySQL was removed in GitLab 12.1. This procedure should be performed -**before** installing GitLab 12.1. - -[pgLoader](https://pgloader.io/) 3.4.1+ is required, confirm with `pgloader -V`. - -You can install it directly from your distribution, for example in -Debian/Ubuntu: - -1. Search for the version: - - ```shell - apt-cache madison pgloader - ``` - -1. If the version is 3.4.1+, install it with: - - ```shell - sudo apt-get install pgloader - ``` - - If your distribution's version is too old, use PostgreSQL's repository: - - ```shell - # Add repository - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' - - # Add key - sudo apt-get install wget ca-certificates - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - - - # Install package - sudo apt-get update - sudo apt-get install pgloader - ``` - -For other distributions, follow the instructions in PostgreSQL's -[download page](https://www.postgresql.org/download/) to add their repository -and then install `pgloader`. - -If you are migrating to a Docker based installation, you must install -pgLoader in the container as it is not included in the container image. - -1. Start a shell session in the context of the running container: - - ```shell - docker exec -it gitlab bash - ``` - -1. Install pgLoader: - - ```shell - apt-get update - apt-get -y install pgloader - ``` - -## Omnibus GitLab installations - -For [Omnibus GitLab packages](https://about.gitlab.com/install/), you first -enable the bundled PostgreSQL: - -1. Stop GitLab: - - ```shell - sudo gitlab-ctl stop - ``` - -1. Edit `/etc/gitlab/gitlab.rb` to enable bundled PostgreSQL: - - ```ruby - postgresql['enable'] = true - ``` - -1. Edit `/etc/gitlab/gitlab.rb` to use the bundled PostgreSQL. Review all of the - settings beginning with `db_` (such as `gitlab_rails['db_adapter']`). To use - the default values, you can comment all of them out. - -1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) - for the changes to take effect. - -1. Start Puma and PostgreSQL so that we can prepare the schema: - - ```shell - sudo gitlab-ctl start puma - sudo gitlab-ctl start postgresql - ``` - -1. Run the following commands to prepare the schema: - - ```shell - sudo gitlab-rake db:create db:migrate - ``` - -1. Stop Puma to prevent other database access from interfering with the loading of data: - - ```shell - sudo gitlab-ctl stop puma - ``` - -After these steps, you have a fresh PostgreSQL database with up-to-date schema. - -Next, use `pgloader` to migrate the data from the old MySQL database to the -new PostgreSQL one: - -1. Save the following snippet in a `commands.load` file, and edit with your - MySQL database `username`, `password` and `host`: - - ```sql - LOAD DATABASE - FROM mysql://username:password@host/gitlabhq_production - INTO postgresql://gitlab-psql@unix://var/opt/gitlab/postgresql:/gitlabhq_production - - WITH include no drop, truncate, disable triggers, create no tables, - create no indexes, preserve index names, no foreign keys, - data only - - SET MySQL PARAMETERS - net_read_timeout = '90', - net_write_timeout = '180' - - ALTER SCHEMA 'gitlabhq_production' RENAME TO 'public' - - ; - ``` - -1. Start the migration: - - ```shell - sudo -u gitlab-psql pgloader commands.load - ``` - -1. After the migration finishes, you should see a summary table that looks like - the following: - - ```plaintext - table name read imported errors total time - ----------------------------------------------- --------- --------- --------- -------------- - fetch meta data 119 119 0 0.388s - Truncate 119 119 0 1.134s - ----------------------------------------------- --------- --------- --------- -------------- - public.abuse_reports 0 0 0 0.490s - public.appearances 0 0 0 0.488s - . - . - . - public.web_hook_logs 0 0 0 1.080s - ----------------------------------------------- --------- --------- --------- -------------- - COPY Threads Completion 4 4 0 2.008s - Reset Sequences 113 113 0 0.304s - Install Comments 0 0 0 0.000s - ----------------------------------------------- --------- --------- --------- -------------- - Total import time 1894 1894 0 12.497s - ``` - - If there is no output for more than 30 minutes, it's possible `pgloader` encountered an error. See - the [troubleshooting guide](#troubleshooting) for more details. - -1. Start GitLab: - - ```shell - sudo gitlab-ctl start - ``` - -You can now verify that everything works as expected by visiting GitLab. - -## Source installations - -For installations from source that use MySQL, you must first -[install PostgreSQL and create a database](../install/installation.md#6-database). - -After the database is created, go on with the following steps: - -1. Stop GitLab: - - ```shell - sudo service gitlab stop - ``` - -1. Switch database from MySQL to PostgreSQL - - ```shell - cd /home/git/gitlab - sudo -u git mv config/database.yml config/database.yml.bak - sudo -u git cp config/database.yml.postgresql config/database.yml - sudo -u git -H chmod o-rwx config/database.yml - ``` - -1. Install Gems related to PostgreSQL - - ```shell - sudo -u git -H rm .bundle/config - sudo -u git -H bundle install --deployment --without development test mysql aws kerberos - ``` - -1. Run the following commands to prepare the schema: - - ```shell - sudo -u git -H bundle exec rake db:create db:migrate RAILS_ENV=production - ``` - -After these steps, you have a fresh PostgreSQL database with up-to-date schema. - -Next, use `pgloader` to migrate the data from the old MySQL database to the -new PostgreSQL one: - -1. Save the following snippet in a `commands.load` file, and edit with your - MySQL `username`, `password` and `host`: - - ```sql - LOAD DATABASE - FROM mysql://username:password@host/gitlabhq_production - INTO postgresql://postgres@unix://var/run/postgresql:/gitlabhq_production - - WITH include no drop, truncate, disable triggers, create no tables, - create no indexes, preserve index names, no foreign keys, - data only - - SET MySQL PARAMETERS - net_read_timeout = '90', - net_write_timeout = '180' - - ALTER SCHEMA 'gitlabhq_production' RENAME TO 'public' - - ; - ``` - -1. Start the migration: - - ```shell - sudo -u postgres pgloader commands.load - ``` - -1. After the migration finishes, you should see a summary table that looks like - the following: - - ```plaintext - table name read imported errors total time - ----------------------------------------------- --------- --------- --------- -------------- - fetch meta data 119 119 0 0.388s - Truncate 119 119 0 1.134s - ----------------------------------------------- --------- --------- --------- -------------- - public.abuse_reports 0 0 0 0.490s - public.appearances 0 0 0 0.488s - . - . - . - public.web_hook_logs 0 0 0 1.080s - ----------------------------------------------- --------- --------- --------- -------------- - COPY Threads Completion 4 4 0 2.008s - Reset Sequences 113 113 0 0.304s - Install Comments 0 0 0 0.000s - ----------------------------------------------- --------- --------- --------- -------------- - Total import time 1894 1894 0 12.497s - ``` - - If there is no output for more than 30 minutes, it's possible `pgloader` encountered an error. See - the [troubleshooting guide](#troubleshooting) for more details. - -1. Start GitLab: - - ```shell - sudo service gitlab start - ``` - -You can now verify that everything works as expected by visiting GitLab. - -## Troubleshooting - -Sometimes, you might encounter some errors during or after the migration. - -### Database error permission denied - -The PostgreSQL user that you use for the migration **must** have **superuser** privileges. -Otherwise, you may see a similar message to the following: - -```plaintext -debugger invoked on a CL-POSTGRES-ERROR:INSUFFICIENT-PRIVILEGE in thread - #<THREAD "lparallel" RUNNING {10078A3513}>: - Database error 42501: permission denied: "RI_ConstraintTrigger_a_20937" is a system trigger - QUERY: ALTER TABLE ci_builds DISABLE TRIGGER ALL; - 2017-08-23T00:36:56.782000Z ERROR Database error 42501: permission denied: "RI_ConstraintTrigger_c_20864" is a system trigger - QUERY: ALTER TABLE approver_groups DISABLE TRIGGER ALL; -``` - -### 500 errors after the migration - -If you experience 500 errors after the migration, try to clear the cache: - -```shell -# Omnibus GitLab -sudo gitlab-rake cache:clear - -# Installations from source -sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production -``` +Support for MySQL was removed in GitLab 12.1. diff --git a/doc/update/package/index.md b/doc/update/package/index.md index e0a4a388f56..575194793c2 100644 --- a/doc/update/package/index.md +++ b/doc/update/package/index.md @@ -16,7 +16,7 @@ GitLab package. - If you are upgrading from a non-package installation to a GitLab package installation, see [Upgrading from a non-package installation to a GitLab package installation](https://docs.gitlab.com/omnibus/update/convert_to_omnibus.html). - Ensure that any - [background migrations](../index.md#checking-for-background-migrations-before-upgrading) + [background migrations](../background_migrations.md) are fully completed. Upgrading before background migrations have finished can lead to data corruption. We recommend performing upgrades between major and minor releases no more than once per @@ -178,9 +178,12 @@ To download and install GitLab: # Debian/Ubuntu dpkg -i <package_name> - # CentOS/RHEL + # RHEL/CentOS 6 and 7 rpm -Uvh <package_name> + # RHEL/CentOS 8 + dnf install <package_name> + # SUSE zypper install <package_name> ``` diff --git a/doc/update/patch_versions.md b/doc/update/patch_versions.md index e0c0cdf31f9..efbe1bc7fcd 100644 --- a/doc/update/patch_versions.md +++ b/doc/update/patch_versions.md @@ -136,5 +136,5 @@ If all items are green, then congratulations upgrade complete! ### 11. Make sure background migrations are finished -[Check the status of background migrations](../user/admin_area/monitoring/background_migrations.md#check-the-status-of-background-migrations) +[Check the status of background migrations](../update/background_migrations.md) and make sure they are finished. diff --git a/doc/update/plan_your_upgrade.md b/doc/update/plan_your_upgrade.md index 8169645e278..667ed37af02 100644 --- a/doc/update/plan_your_upgrade.md +++ b/doc/update/plan_your_upgrade.md @@ -126,7 +126,7 @@ to your instance and then upgrade it for any relevant features you're using. - Account for any [version-specific changes](package/index.md#version-specific-changes). - Check the [OS compatibility with the target GitLab version](../administration/package_information/supported_os.md). - Due to background migrations, plan to pause before any further upgrades. - [All migrations must finish running](index.md#checking-for-background-migrations-before-upgrading) + [All migrations must finish running](background_migrations.md) before the next upgrade. - If available in your starting version, consider [turning on maintenance mode](../administration/maintenance_mode/index.md) during the @@ -173,7 +173,7 @@ If you have Kubernetes clusters connected with GitLab, [upgrade your GitLab agen #### Elasticsearch Before updating GitLab, confirm Advanced Search migrations are complete by -[checking for pending advanced search migrations](index.md#checking-for-pending-advanced-search-migrations). +[checking for pending advanced search migrations](background_migrations.md). After updating GitLab, you may have to upgrade [Elasticsearch if the new version breaks compatibility](../integration/advanced_search/elasticsearch.md#version-requirements). diff --git a/doc/update/removals.md b/doc/update/removals.md index 0bc82403f60..10d937f0f16 100644 --- a/doc/update/removals.md +++ b/doc/update/removals.md @@ -9,6 +9,9 @@ info: "See the Technical Writers assigned to Development Guidelines: https://abo In each release, GitLab removes features that were deprecated in an earlier release. Some features cause breaking changes when they are removed. +**{rss}** **To be notified of upcoming breaking changes**, +add this URL to your RSS feed reader: `https://about.gitlab.com/breaking-changes.xml` + <!-- vale off --> <!-- @@ -43,6 +46,43 @@ To change the approvals required for a merge request, you should no longer use t Instead, use the [`/approval_rules` endpoint](https://docs.gitlab.com/ee/api/merge_request_approvals.html#merge-request-level-mr-approvals) to [create](https://docs.gitlab.com/ee/api/merge_request_approvals.html#create-merge-request-level-rule) or [update](https://docs.gitlab.com/ee/api/merge_request_approvals.html#update-merge-request-level-rule) the approval rules for a merge request. +## Removed in 15.8 + +### CiliumNetworkPolicy within the auto deploy Helm chart is removed + +All functionality related to the GitLab Container Network Security and Container Host Security categories was deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. The [CiliumNetworkPolicy definition](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/blob/master/assets/auto-deploy-app/values.yaml#L175) that exists as part of the [GitLab Auto Deploy Helm chart](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/tree/master/assets/auto-deploy-app) was not removed as scheduled in GitLab 15.0. This policy is planned to be removed in the GitLab 15.8 release. + +If you want to preserve this functionality, you can follow one of these two paths: + +1. Fork the [GitLab Auto Deploy Helm chart](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/tree/master/assets/auto-deploy-app) into the `chart/` path within your project +1. Set `AUTO_DEPLOY_IMAGE_VERSION` and `DAST_AUTO_DEPLOY_IMAGE_VERSION` to the most recent version of the image that included the CiliumNetworkPolicy + +## Removed in 15.7 + +### Flowdock integration + +As of December 22, 2022, we are removing the Flowdock integration because the service was shut down on August 15, 2022. + +## Removed in 15.6 + +### NFS as Git repository storage is no longer supported + +As of November 22, 2022, we have removed support for customers using NFS for Git repository storage. This was +originally planned for May 22, 2022, but in an effort to allow continued maturity of Gitaly Cluster, we delayed +our removal of support date until now. Please see our official [Statement of Support](https://about.gitlab.com/support/statement-of-support/#gitaly-and-nfs) +for further information. + +This change in support follows the development deprecation for NFS for Git repository storage that occurred in GitLab 14.0. + +Gitaly Cluster offers tremendous benefits for our customers such as: + +- [Variable replication factors](https://docs.gitlab.com/ee/administration/gitaly/index.html#replication-factor). +- [Strong consistency](https://docs.gitlab.com/ee/administration/gitaly/index.html#strong-consistency). +- [Distributed read capabilities](https://docs.gitlab.com/ee/administration/gitaly/index.html#distributed-reads). + +We encourage customers currently using NFS for Git repositories to migrate as soon as possible by reviewing our documentation on +[migrating to Gitaly Cluster](https://docs.gitlab.com/ee/administration/gitaly/index.html#migrate-to-gitaly-cluster). + ## Removed in 15.4 ### SAST analyzer consolidation and CI/CD template changes @@ -730,7 +770,7 @@ WARNING: This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. -The `push_rules_supersede_code_owners` feature flag has been removed in GitLab 15.0. From now on, push rules will supersede the `CODEOWNERS` file. The code owners feature is no longer available for access control. +The `push_rules_supersede_code_owners` feature flag has been removed in GitLab 15.0. From now on, push rules will supersede the `CODEOWNERS` file. Even if Code Owner approval is required, a push rule that explicitly allows a specific user to push code supersedes the Code Owners setting. ### `type` and `types` keyword from CI/CD configuration diff --git a/doc/update/restore_after_failure.md b/doc/update/restore_after_failure.md index cc0b188a0f8..92b68410dca 100644 --- a/doc/update/restore_after_failure.md +++ b/doc/update/restore_after_failure.md @@ -2,64 +2,11 @@ stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-02-28' +redirect_to: '../raketasks/backup_restore.md' --- -# Restoring from backup after a failed upgrade **(FREE SELF)** +# Restoring from backup after a failed upgrade (removed) **(FREE SELF)** -Upgrades are usually smooth and restoring from backup is a rare occurrence. -However, it's important to know how to recover when problems do arise. - -## Roll back to an earlier version and restore a backup - -In some cases after a failed upgrade, the fastest solution is to roll back to -the previous version you were using. We recommend this path because the failed -upgrade might have made database changes that cannot be readily reverted. - -First, roll back the code or package. For source installations this involves -checking out the older version (branch or tag). For Omnibus installations this -means installing the older -[`.deb` or `.rpm` package](https://packages.gitlab.com/gitlab). Then, restore from a -backup. -Follow the instructions in the -[Backup and Restore](../raketasks/backup_restore.md#restore-gitlab) -documentation. - -## Potential problems on the next upgrade - -When a rollback is necessary it can produce problems on subsequent upgrade -attempts. This is because some tables may have been added during the failed -upgrade. If these tables are still present after you restore from the -older backup it can lead to migration failures on future upgrades. - -We drop all tables prior to importing the backup to prevent this problem. - -Example error: - -```plaintext -== 20151103134857 CreateLfsObjects: migrating ================================= --- create_table(:lfs_objects) -rake aborted! -StandardError: An error has occurred, this and all later migrations canceled: - -PG::DuplicateTable: ERROR: relation "lfs_objects" already exists -``` - -Copy the version from the error. In this case the version number is -`20151103134857`. - -WARNING: -Use the following steps only if you are certain you must do them. - -1. Pass the version to a database Rake task to manually mark the migration as - complete. - - ```shell - # Source install - sudo -u git -H bundle exec rake gitlab:db:mark_migration_complete[20151103134857] RAILS_ENV=production - - # Omnibus install - sudo gitlab-rake gitlab:db:mark_migration_complete[20151103134857] - ``` - -1. After the migration is successfully marked, run the Rake `db:migrate` task again. -1. Repeat this process until all failed migrations are complete. +This content was removed in GitLab 15.7. +Use the [backup and restore](../raketasks/backup_restore.md) documentation instead. diff --git a/doc/update/upgrading_from_source.md b/doc/update/upgrading_from_source.md index f5b85330f3b..852b54c7339 100644 --- a/doc/update/upgrading_from_source.md +++ b/doc/update/upgrading_from_source.md @@ -29,7 +29,7 @@ to identify the ideal upgrade path. Before upgrading to a new major version, you should ensure that any background migration jobs from previous releases have been completed. To see the current size of the `background_migration` queue, -[Check for background migrations before upgrading](index.md#checking-for-background-migrations-before-upgrading). +[Check for background migrations before upgrading](background_migrations.md). ## Guidelines for all versions diff --git a/doc/update/upgrading_postgresql_using_slony.md b/doc/update/upgrading_postgresql_using_slony.md index a645eb220ad..6d2abee3fc6 100644 --- a/doc/update/upgrading_postgresql_using_slony.md +++ b/doc/update/upgrading_postgresql_using_slony.md @@ -2,479 +2,11 @@ stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-02-28' +redirect_to: '../administration/postgresql/replication_and_failover.md' --- -# Upgrading PostgreSQL Using Slony **(FREE SELF)** +# Upgrading PostgreSQL Using Slony (removed) **(FREE SELF)** -This guide describes the steps one can take to upgrade their PostgreSQL database -to the latest version without the need for hours of downtime. This guide assumes -you have two database servers: one database server running an older version of -PostgreSQL (for example, 9.2.18) and one server running a newer version (for example, 9.6.0). - -For this process, a PostgreSQL replication tool called -[Slony](https://www.slony.info/) is used. Slony allows replication between different -PostgreSQL versions and as such can be used to upgrade a cluster with a minimal -amount of downtime. - -This guide often refers to the user `gitlab-psql`, which is the -user used to run the various PostgreSQL OS processes. If you are using a -different user (for example, `postgres`), replace `gitlab-psql` with the name -of said user. This guide also assumes your database is called -`gitlabhq_production`. If you happen to use a different database name you should -change this accordingly. - -## Database Dumps - -Slony only replicates data and not any schema changes. As a result you must -ensure that all databases have the same database structure. - -To do so, generate a dump of the current database. This dump only -contains the structure, not any data. To generate this dump run the following -command on your active database server: - -```shell -sudo -u gitlab-psql /opt/gitlab/embedded/bin/pg_dump -h /var/opt/gitlab/postgresql -p 5432 -U gitlab-psql -s -f /tmp/structure.sql gitlabhq_production -``` - -If you're not using the Omnibus GitLab package you may have to adjust the paths to -`pg_dump` and the PostgreSQL installation directory to match the paths of your -configuration. - -After the structure dump is generated, generate another dump for the -`schema_migrations` table. This table doesn't have any primary keys and as such -can't be replicated by Slony. To generate a dump of the `schema_migrations` table, run the following command on your active database server: - -```shell -sudo -u gitlab-psql /opt/gitlab/embedded/bin/pg_dump -h /var/opt/gitlab/postgresql/ -p 5432 -U gitlab-psql -a -t schema_migrations -f /tmp/migrations.sql gitlabhq_production -``` - -Next, move these files somewhere accessible by the new database -server. The easiest way is to download these files to your local system: - -```shell -scp your-user@production-database-host:/tmp/*.sql /tmp -``` - -This copies all the SQL files located in `/tmp` to your local system's -`/tmp` directory. Once copied you can safely remove the files from the database -server. - -## Installing Slony - -Use Slony to upgrade the database without requiring a long downtime. -Slony can be downloaded from <https://www.slony.info/>. If you have installed -PostgreSQL using your operating system's package manager you may also be able to -install Slony using said package manager. - -When compiling Slony from source you *must* use the following commands to do so: - -```shell -./configure --prefix=/path/to/installation/directory --with-perltools --with-pgconfigdir=/path/to/directory/containing/pg_config/bin -make -make install -``` - -Omnibus users can use the following commands: - -```shell -./configure --prefix=/opt/gitlab/embedded --with-perltools --with-pgconfigdir=/opt/gitlab/embedded/bin -make -make install -``` - -This assumes you have installed GitLab into `/opt/gitlab`. - -To test if Slony is installed properly, run the following commands: - -```shell -test -f /opt/gitlab/embedded/bin/slonik && echo 'Slony installed' || echo 'Slony not installed' -test -f /opt/gitlab/embedded/bin/slonik_init_cluster && echo 'Slony Perl tools are available' || echo 'Slony Perl tools are not available' -/opt/gitlab/embedded/bin/slonik -v -``` - -This assumes Slony was installed to `/opt/gitlab/embedded`. If Slony was -installed properly the output of these commands is (the mentioned `slonik` -version may be different): - -```plaintext -Slony installed -Slony Perl tools are available -slonik version 2.2.5 -``` - -## Slony User - -Next, set up a PostgreSQL user that Slony can use to replicate your -database. To do so, sign in to your production database using `psql` using a -super-user account. After signing in, run the following SQL queries: - -```sql -CREATE ROLE slony WITH SUPERUSER LOGIN REPLICATION ENCRYPTED PASSWORD 'password string here'; -ALTER ROLE slony SET statement_timeout TO 0; -``` - -Make sure you replace "password string here" with an actual password for the -user. A password is required. This user must be created on both the old and -new database server using the same password. - -After creating the user, be sure to note the password, as the password is needed -later. - -## Configuring Slony - -You can now start configuring Slony. Slony uses a configuration file for -most of the work so it is important to set this up with care. Your configuration -specifies where to put log files, how Slony should connect to the databases, -etc. - -First, create some required directories and set the correct -permissions. To do so, run the following commands on both the old and new -database server: - -```shell -sudo mkdir -p /var/log/gitlab/slony /var/run/slony1 /var/opt/gitlab/postgresql/slony -sudo chown gitlab-psql:root /var/log/gitlab/slony /var/run/slony1 /var/opt/gitlab/postgresql/slony -``` - -Here `gitlab-psql` is the user used to run the PostgreSQL database processes. If -you are using a different user you should replace this with the name of said -user. - -Now that the directories are in place you can create the configuration file -by using the following template: - -```perl -if ($ENV{"SLONYNODES"}) { - require $ENV{"SLONYNODES"}; -} else { - $CLUSTER_NAME = 'slony_replication'; - $LOGDIR = '/var/log/gitlab/slony'; - $MASTERNODE = 1; - $DEBUGLEVEL = 2; - - add_node(host => 'OLD_HOST', dbname => 'gitlabhq_production', port =>5432, - user=>'slony', password=>'SLONY_PASSWORD', node=>1); - - add_node(host => 'NEW_HOST', dbname => 'gitlabhq_production', port =>5432, - user=>'slony', password=>'SLONY_PASSWORD', node=>2, parent=>1 ); -} - -$SLONY_SETS = { - "set1" => { - "set_id" => 1, - "table_id" => 1, - "sequence_id" => 1, - "pkeyedtables" => [ - TABLES - ], - }, -}; - -if ($ENV{"SLONYSET"}) { - require $ENV{"SLONYSET"}; -} - -# Please do not add or change anything below this point. -1; -``` - -Replace the following placeholders in this file to use it: - -- `OLD_HOST`: the address of the old database server. -- `NEW_HOST`: the address of the new database server. -- `SLONY_PASSWORD`: the password of the Slony user created earlier. -- `TABLES`: the tables to replicate. - -Generate the list of tables to replicate by running the following -command on your old PostgreSQL database: - -```shell -sudo gitlab-psql gitlabhq_production -c "select concat('\"', schemaname, '.', tablename, '\",') from pg_catalog.pg_tables where schemaname = 'public' and tableowner = 'gitlab' and tablename != 'schema_migrations' order by tablename asc;" -t -``` - -If you're not using Omnibus you should replace `gitlab-psql` with the -appropriate path to the `psql` executable. - -The above command outputs a list of tables in a format that can be copy-pasted -directly into the above configuration file. Make sure to _replace_ `TABLES` with -this output, don't just append it below it. The result looks like this: - -```perl -"pkeyedtables" => [ - "public.abuse_reports", - "public.appearances", - "public.application_settings", - ... more rows here ... -] -``` - -After you have the configuration file generated you must install it on both the -old and new database. To do so, place it in -`/var/opt/gitlab/postgresql/slony/slon_tools.conf` (for which you created the -directory earlier on). - -Now that the configuration file is in place, you can _finally_ start replicating -the database. First, set up the schema in the new database by making -sure that the SQL files generated earlier are in the `/tmp` -directory of the new server. After these files are in place start a `psql` -session on this server: - -```shell -sudo gitlab-psql gitlabhq_production -``` - -Now run the following commands: - -```plaintext -\i /tmp/structure.sql -\i /tmp/migrations.sql -``` - -To verify if the structure is in place close the session (`\q`), start it again, then -run `\d`. If all went well you should see output along the lines of the -following: - -```plaintext - List of relations - Schema | Name | Type | Owner ---------+---------------------------------------------+----------+------------- - public | abuse_reports | table | gitlab - public | abuse_reports_id_seq | sequence | gitlab - public | appearances | table | gitlab - public | appearances_id_seq | sequence | gitlab - public | application_settings | table | gitlab - public | application_settings_id_seq | sequence | gitlab - public | approvals | table | gitlab - ... more rows here ... -``` - -Now you can initialize the required tables and other processes for -the replication process. To do so, run the following on the old database: - -```shell -sudo -u gitlab-psql /opt/gitlab/embedded/bin/slonik_init_cluster --conf /var/opt/gitlab/postgresql/slony/slon_tools.conf | /opt/gitlab/embedded/bin/slonik -``` - -If all went well this produces something along the lines of: - -```plaintext -<stdin>:10: Set up replication nodes -<stdin>:13: Next: configure paths for each node/origin -<stdin>:16: Replication nodes prepared -<stdin>:17: Please start a slon replication daemon for each node -``` - -Next, start a replication node on every server. To do so, run the -following on the old database: - -```shell -sudo -u gitlab-psql /opt/gitlab/embedded/bin/slon_start 1 --conf /var/opt/gitlab/postgresql/slony/slon_tools.conf -``` - -This should produce an output like the following: - -```plaintext -Invoke slon for node 1 - /opt/gitlab/embedded/bin/slon -p /var/run/slony1/slony_replication_node1.pid -s 1000 -d2 slony_replication 'host=192.168.0.7 dbname=gitlabhq_production user=slony port=5432 password=hieng8ezohHuCeiqu0leeghai4aeyahp' > /var/log/gitlab/slony/node1/gitlabhq_production-2016-10-06.log 2>&1 & -Slon successfully started for cluster slony_replication, node node1 -PID [26740] -Start the watchdog process as well... -``` - -Next, run the following command on the _new_ database server: - -```shell -sudo -u gitlab-psql /opt/gitlab/embedded/bin/slon_start 2 --conf /var/opt/gitlab/postgresql/slony/slon_tools.conf -``` - -This produces similar output if all went well. - -After Slony starts, you must tell the new database server what it should replicate. Run the following command on the _new_ database server: - -```shell -sudo -u gitlab-psql /opt/gitlab/embedded/bin/slonik_create_set 1 --conf /var/opt/gitlab/postgresql/slony/slon_tools.conf | /opt/gitlab/embedded/bin/slonik -``` - -This should produce an output like the following: - -```plaintext -<stdin>:11: Subscription set 1 (set1) created -<stdin>:12: Adding tables to the subscription set -<stdin>:16: Add primary keyed table public.abuse_reports -<stdin>:20: Add primary keyed table public.appearances -<stdin>:24: Add primary keyed table public.application_settings -... more rows here ... -<stdin>:327: Adding sequences to the subscription set -<stdin>:328: All tables added -``` - -Finally, you can start the replication process by running the following on the -_new_ database server: - -```shell -sudo -u gitlab-psql /opt/gitlab/embedded/bin/slonik_subscribe_set 1 2 --conf /var/opt/gitlab/postgresql/slony/slon_tools.conf | /opt/gitlab/embedded/bin/slonik -``` - -This should produce the following output: - -```plaintext -<stdin>:6: Subscribed nodes to set 1 -``` - -At this point the new database server starts replicating the data of the old -database server. This process can take anywhere from a few minutes to hours, if -not days. Unfortunately Slony itself doesn't really provide a way of knowing -when the two databases are in sync. To get an estimate of the progress you can -use the following shell script: - -```shell -#!/usr/bin/env bash - -set -e - -user='slony' -pass='SLONY_PASSWORD' - -function main { - while : - do - local source - local target - - source=$(PGUSER="${user}" PGPASSWORD="${pass}" /opt/gitlab/embedded/bin/psql -h OLD_HOST gitlabhq_production -c "select pg_size_pretty(pg_database_size('gitlabhq_production'));" -t -A) - target=$(PGUSER="${user}" PGPASSWORD="${pass}" /opt/gitlab/embedded/bin/psql -h NEW_HOST gitlabhq_production -c "select pg_size_pretty(pg_database_size('gitlabhq_production'));" -t -A) - - echo "$(date): ${target} of ${source}" >> progress.log - echo "$(date): ${target} of ${source}" - - sleep 60 - done -} - -main -``` - -This script compares the sizes of the old and new database every minute and -prints the results to STDOUT as well as logging it to a file. Make sure to replace -`SLONY_PASSWORD`, `OLD_HOST`, and `NEW_HOST` with the correct values. - -## Stopping Replication - -Eventually, the two databases become in sync. At this point, there is a few minutes of downtime that you must plan for before the replicated database is available. During this time, the replication process should stop and all Slony data should be removed from both databases. After the replication process finishes, GitLab can restart and is able to use the newly-replicated database. - -First, stop all of GitLab. Omnibus users can do so by running the -following on their GitLab servers: - -```shell -sudo gitlab-ctl stop puma -sudo gitlab-ctl stop sidekiq -sudo gitlab-ctl stop mailroom -``` - -If you have any other processes that use PostgreSQL, you should also stop those. - -After everything successfully stops, be sure to update any configuration settings -and DNS records so they all point to the new database. - -When the configuration is complete, stop the replication -process. It's crucial that no new data is written to the databases at this point, -as this data is discarded. - -To stop replication, run the following on both database servers: - -```shell -sudo -u gitlab-psql /opt/gitlab/embedded/bin/slon_kill --conf /var/opt/gitlab/postgresql/slony/slon_tools.conf -``` - -This stops all the Slony processes on the host the command was executed on. - -## Resetting Sequences - -The above setup does not replicate database sequences, as such these must be -reset manually in the target database. You can use the following script for -this: - -```shell -#!/usr/bin/env bash -set -e - -function main { - local fix_sequences - local fix_owners - - fix_sequences='/tmp/fix_sequences.sql' - fix_owners='/tmp/fix_owners.sql' - - # The SQL queries were taken from - # https://wiki.postgresql.org/wiki/Fixing_Sequences - sudo gitlab-psql gitlabhq_production -t -c " - SELECT 'ALTER SEQUENCE '|| quote_ident(MIN(schema_name)) ||'.'|| quote_ident(MIN(seq_name)) - ||' OWNED BY '|| quote_ident(MIN(TABLE_NAME)) ||'.'|| quote_ident(MIN(column_name)) ||';' - FROM ( - SELECT - n.nspname AS schema_name, - c.relname AS TABLE_NAME, - a.attname AS column_name, - SUBSTRING(d.adsrc FROM E'^nextval\\(''([^'']*)''(?:::text|::regclass)?\\)') AS seq_name - FROM pg_class c - JOIN pg_attribute a ON (c.oid=a.attrelid) - JOIN pg_attrdef d ON (a.attrelid=d.adrelid AND a.attnum=d.adnum) - JOIN pg_namespace n ON (c.relnamespace=n.oid) - WHERE has_schema_privilege(n.oid,'USAGE') - AND n.nspname NOT LIKE 'pg!_%' escape '!' - AND has_table_privilege(c.oid,'SELECT') - AND (NOT a.attisdropped) - AND d.adsrc ~ '^nextval' - ) seq - GROUP BY seq_name HAVING COUNT(*)=1; - " > "${fix_owners}" - - sudo gitlab-psql gitlabhq_production -t -c " - SELECT 'SELECT SETVAL(' || - quote_literal(quote_ident(PGT.schemaname) || '.' || quote_ident(S.relname)) || - ', COALESCE(MAX(' ||quote_ident(C.attname)|| '), 1) ) FROM ' || - quote_ident(PGT.schemaname)|| '.'||quote_ident(T.relname)|| ';' - FROM pg_class AS S, - pg_depend AS D, - pg_class AS T, - pg_attribute AS C, - pg_tables AS PGT - WHERE S.relkind = 'S' - AND S.oid = D.objid - AND D.refobjid = T.oid - AND D.refobjid = C.attrelid - AND D.refobjsubid = C.attnum - AND T.relname = PGT.tablename - ORDER BY S.relname; - " > "${fix_sequences}" - - sudo gitlab-psql gitlabhq_production -f "${fix_owners}" - sudo gitlab-psql gitlabhq_production -f "${fix_sequences}" - - rm "${fix_owners}" "${fix_sequences}" -} - -main -``` - -Upload this script to the _target_ server and execute it as follows: - -```shell -sudo bash path/to/the/script/above.sh -``` - -This corrects the ownership of sequences and reset the next value for the -`id` column to the next available value. - -## Removing Slony - -The final step is to remove all Slony related data. To do so, run the following -command on the _target_ server: - -```shell -sudo gitlab-psql gitlabhq_production -c "DROP SCHEMA _slony_replication CASCADE;" -``` - -Once done you can safely remove any Slony related files (for example, the log -directory), and uninstall Slony if desired. At this point you can start your -GitLab instance again and if all went well it should be using your new database -server. +This content was removed in GitLab 15.7. +Patroni has been used for database replication since GitLab 14.0. To perform upgrades, use the [Patroni replication documentation](../administration/postgresql/replication_and_failover.md) instead. diff --git a/doc/update/with_downtime.md b/doc/update/with_downtime.md index dfe64a3c2a9..2ad928927a4 100644 --- a/doc/update/with_downtime.md +++ b/doc/update/with_downtime.md @@ -6,10 +6,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Multi-node upgrades with downtime **(FREE SELF)** -NOTE: -This process is a work in progress. You're welcome to provide feedback by either raising a ticket to support, -or [commenting on this issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6244). - While you can upgrade a multi-node GitLab deployment [with zero downtime](zero_downtime.md), there are a number of constraints. In particular, you can upgrade to only one minor release at a time, for example, from 14.6 to 14.7, then to 14.8, etc. @@ -37,9 +33,6 @@ At a high level, the process is: substitute the instructions for Omnibus GitLab with your cloud provider's instructions. 1. Upgrade the GitLab application (Sidekiq, Puma) and start the application up. -If you are a Community Edition user, replace `gitlab-ee` with -`gitlab-ce` in the following commands. - ## Stop writes to the database Shut down Puma and Sidekiq on all servers running these processes: @@ -56,16 +49,7 @@ sudo gitlab-ctl stop puma In summary: 1. Check the Consul nodes are all healthy. -1. Upgrade the GitLab package on all your Consul servers: - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` - +1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories) on all your Consul servers. 1. Restart all GitLab services **one node at a time**: ```shell @@ -106,15 +90,7 @@ The Praefect nodes, however, can be upgraded via an AMI redeployment process: ## Upgrade the Gitaly nodes not part of Gitaly cluster -For Gitaly servers which are not part of Gitaly cluster, update the GitLab package: - -```shell -# Debian/Ubuntu -sudo apt-get update && sudo apt-get install gitlab-ee - -# Centos/RHEL -sudo yum install gitlab-ee -``` +For Gitaly servers which are not part of Gitaly cluster, [upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). If you have multiple Gitaly shards or have multiple load-balanced Gitaly nodes using NFS, it doesn't matter in which order you upgrade the Gitaly servers. @@ -123,15 +99,7 @@ using NFS, it doesn't matter in which order you upgrade the Gitaly servers. For unclustered PostgreSQL servers: -1. Upgrade the GitLab package: - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` +1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. The upgrade process does not restart PostgreSQL when the binaries are upgraded. Restart to load the new version: @@ -161,15 +129,7 @@ Follow the following process: sudo gitlab-ctl patroni members ``` -1. Upgrade the GitLab package on one of the replica nodes: - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` +1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories) on one of the replica nodes. 1. Restart to load the new version: @@ -194,27 +154,11 @@ Follow the following process: If you run PgBouncer on your Rails (application) nodes, then PgBouncer are upgraded as part of the application server upgrade. -Upgrade the PgBouncer nodes: - -```shell -# Debian/Ubuntu -sudo apt-get update && sudo apt-get install gitlab-ee - -# Centos/RHEL -sudo yum install gitlab-ee -``` +[Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories) on the PgBouncer nodes. ## Upgrade the Redis node -Upgrade a standalone Redis server by updating the GitLab package: - -```shell -# Debian/Ubuntu -sudo apt-get update && sudo apt-get install gitlab-ee - -# Centos/RHEL -sudo yum install gitlab-ee -``` +Upgrade a standalone Redis server by [upgrading the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). ## Upgrade Redis HA (using Sentinel) **(PREMIUM SELF)** @@ -269,15 +213,7 @@ running all database migrations. On the deploy node: sudo gitlab-ctl reconfigure ``` -1. Upgrade the GitLab package: - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` +1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. If you modified `gitlab.rb` on the deploy node to bypass PgBouncer: 1. Update `gitlab.rb` on the deploy node. Change `gitlab_rails['db_host']` @@ -300,15 +236,7 @@ set to anything in `gitlab.rb` on these nodes. They can be upgraded in parallel: -1. Upgrade the GitLab package: - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` +1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. Ensure all services are restarted: @@ -318,12 +246,4 @@ They can be upgraded in parallel: ## Upgrade the Monitor node -Upgrade the GitLab package: - -```shell -# Debian/Ubuntu -sudo apt-get update && sudo apt-get install gitlab-ee - -# Centos/RHEL -sudo yum install gitlab-ee -``` +[Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). diff --git a/doc/update/zero_downtime.md b/doc/update/zero_downtime.md index eb1d6cef606..deda12145da 100644 --- a/doc/update/zero_downtime.md +++ b/doc/update/zero_downtime.md @@ -54,12 +54,12 @@ Certain major/minor releases may require a set of background migrations to be finished. To guarantee this, such a release processes any remaining jobs before continuing the upgrading procedure. While this doesn't require downtime (if the above conditions are met) we require that you -[wait for background migrations to complete](index.md#checking-for-background-migrations-before-upgrading) +[wait for background migrations to complete](background_migrations.md) between each major/minor release upgrade. The time necessary to complete these migrations can be reduced by increasing the number of Sidekiq workers that can process jobs in the `background_migration` queue. To see the size of this queue, -[Check for background migrations before upgrading](index.md#checking-for-background-migrations-before-upgrading). +[Check for background migrations before upgrading](background_migrations.md). As a guideline, any database smaller than 10 GB doesn't take too much time to upgrade; perhaps an hour at most per minor release. Larger databases however may @@ -132,18 +132,7 @@ load balancer to latest GitLab version. sudo touch /etc/gitlab/skip-auto-reconfigure ``` - 1. Update the GitLab package: - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` - - If you are a Community Edition user, replace `gitlab-ee` with - `gitlab-ce` in the above command. + 1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. Get the regular migrations and latest code in place. Before running this step, the deploy node's `/etc/gitlab/gitlab.rb` configuration file must have @@ -193,17 +182,7 @@ Before you update the main GitLab application you must (in order): #### Upgrade Gitaly nodes -Upgrade the Gitaly nodes one at a time to ensure access to Git repositories is maintained: - -```shell -# Debian/Ubuntu -sudo apt-get update && sudo apt-get install gitlab-ee - -# Centos/RHEL -sudo yum install gitlab-ee -``` - -If you are a Community Edition user, replace `gitlab-ee` with `gitlab-ce` in the above command. +[Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories) on the Gitaly nodes one at a time to ensure access to Git repositories is maintained. #### Upgrade Praefect @@ -226,17 +205,7 @@ node first and run database migrations. 1. On the **Praefect deploy node**: - 1. Upgrade the GitLab package: - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` - - If you are a Community Edition user, replace `gitlab-ee` with `gitlab-ce` in the command above. + 1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. To apply the Praefect database migrations and restart Praefect, run: @@ -246,13 +215,7 @@ node first and run database migrations. 1. On all **remaining Praefect nodes**: - 1. Upgrade the GitLab package: - - ```shell - sudo apt-get update && sudo apt-get install gitlab-ee - ``` - - If you are a Community Edition user, replace `gitlab-ee` with `gitlab-ce` in the command above. + 1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. Ensure nodes are running the latest code: @@ -279,17 +242,7 @@ node throughout the process. **PostgreSQL only nodes** -- Update the GitLab package - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` - - If you are a Community Edition user, replace `gitlab-ee` with `gitlab-ce` in the above command. +- [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). - Ensure nodes are running the latest code @@ -299,17 +252,7 @@ node throughout the process. **Deploy node** -- Update the GitLab package - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` - - If you are a Community Edition user, replace `gitlab-ee` with `gitlab-ce` in the above command. +- [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). - If you're using PgBouncer: @@ -341,13 +284,7 @@ node throughout the process. **All nodes _excluding_ the Deploy node** -- Update the GitLab package - - ```shell - sudo apt-get update && sudo apt-get install gitlab-ee - ``` - - If you are a Community Edition user, replace `gitlab-ee` with `gitlab-ce` in the above command. +- [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). - Ensure nodes are running the latest code @@ -514,15 +451,7 @@ Log in to your **primary** node, executing the following: sudo gitlab-ctl reconfigure ``` -1. Update the GitLab package: - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` +1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. To get the database migrations and latest code in place, run: @@ -552,15 +481,7 @@ On each **secondary** node, executing the following: sudo gitlab-ctl reconfigure ``` -1. Update the GitLab package: - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` +1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. To get the database migrations and latest code in place, run: @@ -669,15 +590,7 @@ sudo touch /etc/gitlab/skip-auto-reconfigure **On primary Gitaly only nodes** -1. Update the GitLab package - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` +1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. Ensure nodes are running the latest code @@ -687,15 +600,7 @@ sudo touch /etc/gitlab/skip-auto-reconfigure **On the primary "deploy node"** -1. Update the GitLab package - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` +1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. If you're using PgBouncer: @@ -737,15 +642,7 @@ sudo touch /etc/gitlab/skip-auto-reconfigure **On all primary nodes _excluding_ the primary "deploy node"** -1. Update the GitLab package - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` +1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. Ensure nodes are running the latest code @@ -784,15 +681,7 @@ sudo touch /etc/gitlab/skip-auto-reconfigure **On secondary Gitaly only nodes** -1. Update the GitLab package - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` +1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. Ensure nodes are running the latest code @@ -802,15 +691,7 @@ sudo touch /etc/gitlab/skip-auto-reconfigure **On the secondary "deploy node"** -1. Update the GitLab package - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` +1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. To get the regular database migrations and latest code in place, run @@ -837,15 +718,7 @@ sudo touch /etc/gitlab/skip-auto-reconfigure **On all secondary nodes _excluding_ the secondary "deploy node"** -1. Update the GitLab package - - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee - - # Centos/RHEL - sudo yum install gitlab-ee - ``` +1. [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). 1. Ensure nodes are running the latest code diff --git a/doc/user/admin_area/external_users.md b/doc/user/admin_area/external_users.md new file mode 100644 index 00000000000..8b968a3da01 --- /dev/null +++ b/doc/user/admin_area/external_users.md @@ -0,0 +1,77 @@ +--- +stage: Manage +group: Authentication and Authorization +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# External users **(FREE SELF)** + +In cases where it is desired that a user has access only to some internal or +private projects, there is the option of creating **External Users**. This +feature may be useful when for example a contractor is working on a given +project and should only have access to that project. + +External users: + +- Cannot create project, groups, and snippets in their personal namespaces. +- Can only create projects (including forks), subgroups, and snippets within top-level groups to which they are explicitly granted access. +- Can only access public projects and projects to which they are explicitly granted access, + thus hiding all other internal or private ones from them (like being + logged out). +- Can only access public groups and groups to which they are explicitly granted access, + thus hiding all other internal or private ones from them (like being + logged out). +- Can only access public snippets. + +Access can be granted by adding the user as member to the project or group. +Like usual users, they receive a role in the project or group with all +the abilities that are mentioned in the [permissions table](../permissions.md#project-members-permissions). +For example, if an external user is added as Guest, and your project is internal or +private, they do not have access to the code; you need to grant the external +user access at the Reporter level or above if you want them to have access to the code. You should +always take into account the +[project's visibility and permissions settings](../project/settings/index.md#configure-project-visibility-features-and-permissions) +as well as the permission level of the user. + +NOTE: +External users still count towards a license seat. + +An administrator can flag a user as external by either of the following methods: + +- [Through the API](../../api/users.md#user-modification). +- Using the GitLab UI: + 1. On the top bar, select **Main menu > Admin**. + 1. On the left sidebar, select **Overview > Users** to create a new user or edit an existing one. + There, you can find the option to flag the user as external. + +Additionally, users can be set as external users using: + +- [SAML groups](../../integration/saml.md#external-groups). +- [LDAP groups](../../administration/auth/ldap/ldap_synchronization.md#external-groups). + +## Set a new user to external + +By default, new users are not set as external users. This behavior can be changed +by an administrator: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Account and limit** section. + +If you change the default behavior of creating new users as external, you +have the option to narrow it down by defining a set of internal users. +The **Internal users** field allows specifying an email address regex pattern to +identify default internal users. New users whose email address matches the regex +pattern are set to internal by default rather than an external collaborator. + +The regex pattern format is in Ruby, but it needs to be convertible to JavaScript, +and the ignore case flag is set (`/regex pattern/i`). Here are some examples: + +- Use `\.internal@domain\.com$` to mark email addresses ending with + `.internal@domain.com` as internal. +- Use `^(?:(?!\.ext@domain\.com).)*$\r?` to mark users with email addresses + not including `.ext@domain.com` as internal. + +WARNING: +Be aware that this regex could lead to a +[regular expression denial of service (ReDoS) attack](https://en.wikipedia.org/wiki/ReDoS). diff --git a/doc/user/admin_area/index.md b/doc/user/admin_area/index.md index 4c34d82dc02..c9b6a077c73 100644 --- a/doc/user/admin_area/index.md +++ b/doc/user/admin_area/index.md @@ -163,7 +163,7 @@ You can impersonate a user in the following ways: 1. Select **Impersonate**. - With the API, using [impersonation tokens](../../api/index.md#impersonation-tokens). -All impersonation activities are [captured with audit events](../../administration/audit_events.md#impersonation-data). +All impersonation activities are [captured with audit events](../../administration/audit_events.md#user-impersonation). By default, impersonation is enabled. GitLab can be configured to [disable impersonation](../../api/index.md#disable-impersonation). diff --git a/doc/user/admin_area/license_file.md b/doc/user/admin_area/license_file.md index d821d9bab23..9c35a8c1269 100644 --- a/doc/user/admin_area/license_file.md +++ b/doc/user/admin_area/license_file.md @@ -25,7 +25,8 @@ Otherwise, to add your license: 1. Select **Add license**. NOTE: -In GitLab 14.1.x through 14.10.x, you can access the **Add License** page directly from the URL, `<YourGitLabURL>/admin/license/new`. In GitLab 15.0 and later, the path is `<YourGitLabURL>/admin/subscription`. +In GitLab 14.7.x to 14.9.x, you can add the license file with the UI. +In GitLab 14.1.x to 14.7, if you have already activated your subscription with an activation code, you cannot access **Add License** from the Admin Area. You must access **Add License** directly from the URL, `<YourGitLabURL>/admin/license/new`. ## Add your license file during installation diff --git a/doc/user/admin_area/monitoring/background_migrations.md b/doc/user/admin_area/monitoring/background_migrations.md index f16cb219f2b..b4a6f7f66fb 100644 --- a/doc/user/admin_area/monitoring/background_migrations.md +++ b/doc/user/admin_area/monitoring/background_migrations.md @@ -1,244 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../../update/background_migrations.md' +remove_date: '2023-03-11' --- -# Batched background migrations **(FREE SELF)** +This document was moved to [another location](../../../update/background_migrations.md). -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/51332) in GitLab 13.11. -> - [Deployed behind a feature flag](../../../user/feature_flags.md), disabled by default. -> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/329511) in GitLab 13.12. -> - Enabled on GitLab.com. -> - Recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-batched-background-migrations). - -There can be [risks when disabling released features](../../../administration/feature_flags.md#risks-when-disabling-released-features). -Refer to this feature's version history for more details. - -To update database tables in batches, GitLab can use batched background migrations. These migrations -are created by GitLab developers and run automatically on upgrade. However, such migrations are -limited in scope to help with migrating some `integer` database columns to `bigint`. This is needed to -prevent integer overflow for some tables. - -## Check the status of background migrations - -All migrations must have a `Finished` status before you [upgrade GitLab](../../../update/index.md). -You can [check the status of existing migrations](../../../update/index.md#batched-background-migrations). - -## Enable or disable batched background migrations - -WARNING: -If you disable this feature flag, GitLab upgrades may fail. - -Batched background migrations are under development but ready for production use. -It is deployed behind a feature flag that is **enabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can opt to disable it. - -To enable it: - -```ruby -Feature.enable(:execute_batched_migrations_on_schedule) -``` - -To disable it: - -```ruby -Feature.disable(:execute_batched_migrations_on_schedule) -``` - -### Pause batched background migrations in GitLab 14.x - -To pause an ongoing batched background migration, use the `disable` command above. -This command causes the migration to complete the current batch, and then wait to start the next batch. - -Use the following database queries to see the state of the current batched background migration: - -1. Obtain the ID of the running migration: - - ```sql - SELECT - id, - job_class_name, - table_name, - column_name, - job_arguments - FROM batched_background_migrations - WHERE status <> 3; - ``` - -1. Run this query, replacing `XX` with the ID you obtained in the previous step, - to see the status of the migration: - - ```sql - SELECT - started_at, - finished_at, - finished_at - started_at AS duration, - min_value, - max_value, - batch_size, - sub_batch_size - FROM batched_background_migration_jobs - WHERE batched_background_migration_id = XX - ORDER BY id DESC - limit 10; - ``` - -1. Run the query multiple times within a few minutes to ensure no new row has been added. - If no new row has been added, the migration has been paused. - -1. After confirming the migration has paused, restart the migration (using the `enable` - command above) to proceed with the batch when ready. On larger instances, - background migrations can take as long as 48 hours to complete each batch. - -## Automatic batch size optimization - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60133) in GitLab 13.12. -> - [Deployed behind a feature flag](../../../user/feature_flags.md), disabled by default. -> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/329511) in GitLab 13.12. -> - Enabled on GitLab.com. -> - Recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-automatic-batch-size-optimization). - -There can be [risks when disabling released features](../../../administration/feature_flags.md#risks-when-disabling-released-features). -Refer to this feature's version history for more details. - -To maximize throughput of batched background migrations (in terms of the number of tuples updated per time unit), batch sizes are automatically adjusted based on how long the previous batches took to complete. - -## Enable or disable automatic batch size optimization - -Automatic batch size optimization for batched background migrations is under development but ready for production use. -It is deployed behind a feature flag that is **enabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can opt to disable it. - -To enable it: - -```ruby -Feature.enable(:optimize_batched_migrations) -``` - -To disable it: - -```ruby -Feature.disable(:optimize_batched_migrations) -``` - -## Troubleshooting - -### Database migrations failing because of batched background migration not finished - -When updating to GitLab 14.2 or later there might be a database migration failing with a message like: - -```plaintext -StandardError: An error has occurred, all later migrations canceled: - -Expected batched background migration for the given configuration to be marked as 'finished', but it is 'active': - {:job_class_name=>"CopyColumnUsingBackgroundMigrationJob", :table_name=>"push_event_payloads", :column_name=>"event_id", :job_arguments=>[["event_id"], ["event_id_convert_to_bigint"]]} -``` - -First, check if you have followed the [version-specific upgrade instructions for 14.2](../../../update/index.md#1420). -If you have, you can [manually finish the batched background migration](#manually-finishing-a-batched-background-migration). -If you haven't, choose one of the following methods: - -1. [Rollback and upgrade](#roll-back-and-follow-the-required-upgrade-path) through one of the required -versions before updating to 14.2+. -1. [Roll forward](#roll-forward-and-finish-the-migrations-on-the-upgraded-version), staying on the current -version and manually ensuring that the batched migrations complete successfully. - -#### Roll back and follow the required upgrade path - -1. [Rollback and restore the previously installed version](../../../update/restore_after_failure.md#roll-back-to-an-earlier-version-and-restore-a-backup) -1. Update to either 14.0.5 or 14.1 **before** updating to 14.2+ -1. [Check the status](#check-the-status-of-background-migrations) of the batched background migrations and -make sure they are all marked as finished before attempting to upgrade again. If any remain marked as active, -you can [manually finish them](#manually-finishing-a-batched-background-migration). - -#### Roll forward and finish the migrations on the upgraded version - -##### For a deployment with downtime - -To run all the batched background migrations, it can take a significant amount of time -depending on the size of your GitLab installation. - -1. [Check the status](#check-the-status-of-background-migrations) of the batched background migrations in the -database, and [manually run them](#manually-finishing-a-batched-background-migration) with the appropriate -arguments until the status query returns no rows. -1. When the status of all of all them is marked as complete, re-run migrations for your installation. -1. [Complete the database migrations](../../../administration/raketasks/maintenance.md#run-incomplete-database-migrations) from your GitLab upgrade: - - ```plaintext - sudo gitlab-rake db:migrate - ``` - -1. Run a reconfigure: - - ```plaintext - sudo gitlab-ctl reconfigure - ``` - -1. Finish the upgrade for your installation. - -##### For a no-downtime deployment - -As the failing migrations are post-deployment migrations, you can remain on a running instance of the upgraded -version and wait for the batched background migrations to finish normally. - -1. [Check the status](#check-the-status-of-background-migrations) of the batched background migration from -the error message, and make sure it is listed as finished. If it is still active, either wait until it is done, -or [manually finish it](#manually-finishing-a-batched-background-migration). -1. Re-run migrations for your installation, so the remaining post-deployment migrations finish. - -### Manually finishing a batched background migration - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62634) in GitLab 14.1 - -If you need to manually finish a batched background migration due to an -error, you can run: - -```shell -sudo gitlab-rake gitlab:background_migrations:finalize[<job_class_name>,<table_name>,<column_name>,'<job_arguments>'] -``` - -Replace the values in angle brackets with the correct -arguments. For example, if you receive an error similar to this: - -```plaintext -StandardError: An error has occurred, all later migrations canceled: - -Expected batched background migration for the given configuration to be marked as 'finished', but it is 'active': - {:job_class_name=>"CopyColumnUsingBackgroundMigrationJob", :table_name=>"push_event_payloads", :column_name=>"event_id", :job_arguments=>[["event_id"], ["event_id_convert_to_bigint"]]} -``` - -Plug the arguments from the error message into the command: - -```shell -sudo gitlab-rake gitlab:background_migrations:finalize[CopyColumnUsingBackgroundMigrationJob,push_event_payloads,event_id,'[["event_id"]\, ["event_id_convert_to_bigint"]]'] -``` - -If you need to manually run a batched background migration to continue an upgrade, you can -[check the status](#check-the-status-of-background-migrations) in the database and get the -arguments from the query results. For example, if the query returns this: - -```plaintext - job_class_name | table_name | column_name | job_arguments ----------------------------------------+------------+-------------+------------------------------------ - CopyColumnUsingBackgroundMigrationJob | events | id | [["id"], ["id_convert_to_bigint"]] - ``` - -The results from the query can be plugged into the command: - -```shell -sudo gitlab-rake gitlab:background_migrations:finalize[CopyColumnUsingBackgroundMigrationJob,events,id,'[["id"]\, ["id_convert_to_bigint"]]'] -``` - -### The `BackfillNamespaceIdForNamespaceRoute` batched migration job fails - -In GitLab 14.8, the `BackfillNamespaceIdForNamespaceRoute` batched background migration job -may fail to complete. When retried, a `500 Server Error` is returned. This issue was -[resolved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82387) in GitLab 14.9. - -To resolve this issue, [upgrade GitLab](../../../update/index.md) from 14.8 to 14.9. -You can ignore the failed batch migration until after you update to GitLab 14.9. +<!-- This redirect file can be deleted after <2023-03-11>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/admin_area/monitoring/health_check.md b/doc/user/admin_area/monitoring/health_check.md index 2939a8b0418..668d34af024 100644 --- a/doc/user/admin_area/monitoring/health_check.md +++ b/doc/user/admin_area/monitoring/health_check.md @@ -133,7 +133,7 @@ This check is being exempt from Rack Attack. ## Sidekiq -Learn how to configure the [Sidekiq health checks](../../../administration/sidekiq_health_check.md). +Learn how to configure the [Sidekiq health checks](../../../administration/sidekiq/sidekiq_health_check.md). <!-- ## Troubleshooting diff --git a/doc/user/admin_area/settings/account_and_limit_settings.md b/doc/user/admin_area/settings/account_and_limit_settings.md index 44a6bbb9d8e..b235b812416 100644 --- a/doc/user/admin_area/settings/account_and_limit_settings.md +++ b/doc/user/admin_area/settings/account_and_limit_settings.md @@ -57,7 +57,7 @@ You can change the maximum push size for your instance: 1. On the left sidebar, select **Settings > General**, then expand **Account and limit**. 1. Increase or decrease by changing the value in **Maximum push size (MB)**. -For GitLab.com application limits, read [GitLab application limits](../../../administration/instance_limits.md#max-push-size). +For GitLab.com push size limits, read [accounts and limit settings](../../gitlab_com/index.md#account-and-limit-settings). NOTE: When you [add files to a repository](../../project/repository/web_editor.md#create-a-file) diff --git a/doc/user/admin_area/settings/continuous_integration.md b/doc/user/admin_area/settings/continuous_integration.md index adca9c85af1..81e51aaef37 100644 --- a/doc/user/admin_area/settings/continuous_integration.md +++ b/doc/user/admin_area/settings/continuous_integration.md @@ -46,7 +46,10 @@ limit on the number of [CI/CD minutes](../../../ci/pipelines/cicd_minutes.md) yo ## Enable a specific runner for multiple projects -To enable a specific runner for one or more projects: +If you have already registered a [specific runner](../../../ci/runners/runners_scope.md#specific-runners) +you can assign that runner to other projects. + +To enable a specific runner for more than one project: 1. On the top bar, select **Main menu > Admin**. 1. From the left sidebar, select **Overview > Runners**. @@ -326,7 +329,7 @@ To set the maximum file size: 1. Enter the maximum file size, in bytes. 1. Select **Save size limits**. -## Prevent users from registering runners +## Restrict runner registration by all users in an instance > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22225) in GitLab 14.1. > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/368008) in GitLab 15.5. @@ -335,7 +338,7 @@ GitLab administrators can adjust who is allowed to register runners, by showing By default, all members of a project and group are able to register runners. -To change this: +To restrict all users in an instance from registering runners: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > CI/CD**. @@ -347,6 +350,22 @@ To change this: WARNING: When the registration sections are hidden in the UI, members of the project or group that need to register runners must contact the administrators. If you plan to prevent registration, ensure users have access to the runners they need to run jobs. +## Restrict runner registration by all members in a group + +Prerequisites: + +- Runner registration must be enabled for [all users in the instance](#restrict-runner-registration-by-all-users-in-an-instance). + +GitLab administrators can adjust group permissions to restrict runner registration by group members. + +To restrict runner registration by members in a specific group: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Overview > Groups** and find your group. +1. Select **Edit**. +1. Clear the **New group runners can be registered** checkbox if you want to disable runner registration by all members in the group. If the setting is read-only, you must enable runner registration for the [instance](#restrict-runner-registration-by-all-users-in-an-instance). +1. Select **Save changes**. + ## Troubleshooting ### 413 Request Entity Too Large diff --git a/doc/user/admin_area/settings/img/mirror_settings.png b/doc/user/admin_area/settings/img/mirror_settings.png Binary files differdeleted file mode 100644 index 090db6808a7..00000000000 --- a/doc/user/admin_area/settings/img/mirror_settings.png +++ /dev/null diff --git a/doc/user/admin_area/settings/img/mirror_settings_v15_7.png b/doc/user/admin_area/settings/img/mirror_settings_v15_7.png Binary files differnew file mode 100644 index 00000000000..5da41dbeceb --- /dev/null +++ b/doc/user/admin_area/settings/img/mirror_settings_v15_7.png diff --git a/doc/user/admin_area/settings/index.md b/doc/user/admin_area/settings/index.md index 08c4a0d0167..b2b702bde7c 100644 --- a/doc/user/admin_area/settings/index.md +++ b/doc/user/admin_area/settings/index.md @@ -200,3 +200,12 @@ for the entire GitLab instance: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > Preferences**. 1. Scroll to the **Localization** section, and select your desired first day of the week. + +## Default language + +You can change the [Default language](../../profile/preferences.md) +for the entire GitLab instance: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > Preferences**. +1. Scroll to the **Localization** section, and select your desired default language. diff --git a/doc/user/admin_area/settings/sidekiq_job_limits.md b/doc/user/admin_area/settings/sidekiq_job_limits.md index 7a437d877ca..a25bc0e85e4 100644 --- a/doc/user/admin_area/settings/sidekiq_job_limits.md +++ b/doc/user/admin_area/settings/sidekiq_job_limits.md @@ -9,7 +9,7 @@ type: reference > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68982) in GitLab 14.3. -[Sidekiq](../../../administration/sidekiq.md) jobs get stored in +[Sidekiq](../../../administration/sidekiq/index.md) jobs get stored in Redis. To avoid excessive memory for Redis, we: - Compress job arguments before storing them in Redis. diff --git a/doc/user/admin_area/settings/sign_in_restrictions.md b/doc/user/admin_area/settings/sign_in_restrictions.md index 4ea420d7ca6..6ec3d082114 100644 --- a/doc/user/admin_area/settings/sign_in_restrictions.md +++ b/doc/user/admin_area/settings/sign_in_restrictions.md @@ -68,7 +68,7 @@ For more information, see the [list of settings that can be accessed through API Open the [Rails console](../../../administration/operations/rails_console.md) and run the following: ```ruby -::Gitlab::CurrentSettings.update_attributes!(admin_mode: true) +::Gitlab::CurrentSettings.update!(admin_mode true) ``` #### Use the UI to enable Admin Mode @@ -84,7 +84,7 @@ To enable Admin Mode through the UI: To turn on Admin Mode for your current session and access potentially dangerous resources: -1. On the top bar, select **Enable Admin Mode**. +1. On the top bar, select **Main menu > Enter Admin Mode**. 1. Try to access any part of the UI with `/admin` in the URL (which requires administrator access). When Admin Mode status is disabled or turned off, administrators cannot access resources unless @@ -95,7 +95,11 @@ if they try to open a private group or project, unless they are members of that authentication are supported by Admin Mode. Admin Mode status is stored in the current user session and remains active until either: - It is explicitly disabled. -- It is disabled automatically after a timeout. +- It is disabled automatically after six hours. + +### Turn off Admin Mode for your session + +To turn off Admin Mode for your current session, on the top bar, select **Main menu > Leave Admin mode**. ### Limitations of Admin Mode diff --git a/doc/user/admin_area/settings/sign_up_restrictions.md b/doc/user/admin_area/settings/sign_up_restrictions.md index 76415596dce..8aabe503065 100644 --- a/doc/user/admin_area/settings/sign_up_restrictions.md +++ b/doc/user/admin_area/settings/sign_up_restrictions.md @@ -60,7 +60,7 @@ To enforce confirmation of the email address used for new sign ups: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > General**, and expand **Sign-up restrictions**. -1. Select the **Send confirmation email on sign-up** checkbox, then select **Save changes**. +1. Under **Email confirmation settings**, select **Hard**. ## User cap @@ -190,6 +190,12 @@ To disable it: Feature.disable(:soft_email_confirmation) ``` +## Set up LDAP user filter + +You can limit GitLab access to a subset of the LDAP users on your LDAP server. + +See the [documentation on setting up an LDAP user filter](../../../administration/auth/ldap/index.md#set-up-ldap-user-filter) for more information. + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/admin_area/settings/terraform_limits.md b/doc/user/admin_area/settings/terraform_limits.md new file mode 100644 index 00000000000..4e54c7a3459 --- /dev/null +++ b/doc/user/admin_area/settings/terraform_limits.md @@ -0,0 +1,27 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: reference +--- + +# Terraform limits **(FREE SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352951) in GitLab 15.7. + +You can limit the total storage of [Terraform state files](../../../administration/terraform_state.md). +The limit applies to each individual +state file version, and is checked whenever a new version is created. + +To add a storage limit: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > Preferences**. +1. Expand **Terraform limits**. +1. Adjust the size limit. + +## Available settings + +| Setting | Default | Description | +|------------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------| +| Terraform state size limit (bytes) | 0 | Terraform state files that exceed this size are not saved, and associated Terraform operations are rejected. Set to 0 to allow files of unlimited size. | diff --git a/doc/user/admin_area/settings/visibility_and_access_controls.md b/doc/user/admin_area/settings/visibility_and_access_controls.md index 6d878bcb01c..4da0f5da3f4 100644 --- a/doc/user/admin_area/settings/visibility_and_access_controls.md +++ b/doc/user/admin_area/settings/visibility_and_access_controls.md @@ -219,7 +219,7 @@ If only one protocol is enabled: - The project page shows only the allowed protocol's URL, with no option to change it. -- GitLab shows a tooltip when you hover over the URL's protocol, if user action +- GitLab shows a tooltip when you hover over the protocol for the URL, if user action (such as adding a SSH key or setting a password) is required:  @@ -273,7 +273,7 @@ This option is enabled by default. By disabling it, both [pull mirroring](../../project/repository/mirror/pull.md) and [push mirroring](../../project/repository/mirror/push.md) no longer work in every repository. They can only be re-enabled by an administrator user on a per-project basis. - + ## Configure globally-allowed IP address ranges diff --git a/doc/user/analytics/dora_metrics.md b/doc/user/analytics/dora_metrics.md index b5f37203817..fba0d0e98ff 100644 --- a/doc/user/analytics/dora_metrics.md +++ b/doc/user/analytics/dora_metrics.md @@ -104,7 +104,7 @@ To retrieve metrics for change failure rate, use the [GraphQL](../../api/graphql ### Insights: Custom DORA reporting -Custom charts to visualize DORA data with Insights YAML-based reports. +Custom charts to visualize DORA data with [Insights YAML-based reports](../../user/project/insights/index.md#dora-query-parameters). With this new visualization, software leaders can track metrics improvements, understand patterns in their metrics trends, and compare performance between groups and projects. @@ -113,7 +113,7 @@ With this new visualization, software leaders can track metrics improvements, un Deployment frequency is calculated based on the deployments record, which is created for typical push-based deployments. These deployment records are not created for pull-based deployments, for example when Container Images are connected to GitLab with an agent. -To track DORA metrics in these cases, you can [create a deployment record](../../api/deployments.md#create-a-deployment) using the Deployments API. +To track DORA metrics in these cases, you can [create a deployment record](../../api/deployments.md#create-a-deployment) using the Deployments API. See also the documentation page for [Track deployments of an external deployment tool](../../ci/environments/external_deployment_tools.md). ### Supported DORA metrics in GitLab diff --git a/doc/user/analytics/index.md b/doc/user/analytics/index.md index 01fe466755c..38d92180c7d 100644 --- a/doc/user/analytics/index.md +++ b/doc/user/analytics/index.md @@ -12,7 +12,7 @@ Instance-level analytics make it possible to aggregate analytics across GitLab, so that users can view information across multiple projects and groups in one place. -[Learn more about instance-level analytics](../admin_area/analytics/index.md). +For more information, see [instance-level analytics](../admin_area/analytics/index.md). ## Group-level analytics diff --git a/doc/user/analytics/value_stream_analytics.md b/doc/user/analytics/value_stream_analytics.md index ab68c897da8..0906f7d17a7 100644 --- a/doc/user/analytics/value_stream_analytics.md +++ b/doc/user/analytics/value_stream_analytics.md @@ -204,7 +204,7 @@ Value stream analytics records the following times for each stage: - **Review**: 14:00 to 19:00: 5 hrs - **Staging**: 19:00 to 19:30: 30 minutes -There are some additional considerations for this example: +Keep in mind the following observations related to this example: - Although this example specifies the issue number in a later commit, the process still collects analytics data for the issue. diff --git a/doc/user/application_security/api_fuzzing/create_har_files.md b/doc/user/application_security/api_fuzzing/create_har_files.md index e7eaf47121b..5e033a75902 100644 --- a/doc/user/application_security/api_fuzzing/create_har_files.md +++ b/doc/user/application_security/api_fuzzing/create_har_files.md @@ -109,7 +109,7 @@ responses in HAR format. have an account, first create an account. 1. Browse pages that call an API. Fiddler automatically captures the requests. 1. Select one or more requests, then from the context menu, select **Export > Selected Sessions**. -1. In the **Choose Format** dropdown list select **HTTPArchive v1.2**. +1. In the **Choose Format** dropdown list select **HTTP Archive v1.2**. 1. Enter a filename and select **Save**. Fiddler shows a popup message confirming the export has succeeded. diff --git a/doc/user/application_security/api_fuzzing/index.md b/doc/user/application_security/api_fuzzing/index.md index 03eed6fdbf8..e4ca512bdc6 100644 --- a/doc/user/application_security/api_fuzzing/index.md +++ b/doc/user/application_security/api_fuzzing/index.md @@ -1551,13 +1551,13 @@ When testing an API it can be useful to exclude certain paths. For example, you To verify the paths are excluded, review the `Tested Operations` and `Excluded Operations` portion of the job output. You should not see any excluded paths listed under `Tested Operations`. ```plaintext -2021-05-27 21:51:08 [INF] API Security: --[ Tested Operations ]------------------------- -2021-05-27 21:51:08 [INF] API Security: 201 POST http://target:7777/api/users CREATED -2021-05-27 21:51:08 [INF] API Security: ------------------------------------------------ -2021-05-27 21:51:08 [INF] API Security: --[ Excluded Operations ]----------------------- -2021-05-27 21:51:08 [INF] API Security: GET http://target:7777/api/messages -2021-05-27 21:51:08 [INF] API Security: POST http://target:7777/api/messages -2021-05-27 21:51:08 [INF] API Security: ------------------------------------------------ +2021-05-27 21:51:08 [INF] API Fuzzing: --[ Tested Operations ]------------------------- +2021-05-27 21:51:08 [INF] API Fuzzing: 201 POST http://target:7777/api/users CREATED +2021-05-27 21:51:08 [INF] API Fuzzing: ------------------------------------------------ +2021-05-27 21:51:08 [INF] API Fuzzing: --[ Excluded Operations ]----------------------- +2021-05-27 21:51:08 [INF] API Fuzzing: GET http://target:7777/api/messages +2021-05-27 21:51:08 [INF] API Fuzzing: POST http://target:7777/api/messages +2021-05-27 21:51:08 [INF] API Fuzzing: ------------------------------------------------ ``` #### Examples of excluding paths @@ -1589,7 +1589,7 @@ variables: While testing an API you may might want to exclude a parameter (query string, header, or body element) from testing. This may be needed because a parameter always causes a failure, slows down testing, or for other reasons. To exclude parameters you can use one of the following variables: `FUZZAPI_EXCLUDE_PARAMETER_ENV` or `FUZZAPI_EXCLUDE_PARAMETER_FILE`. -The `FUZZAPI_EXCLUDE_PARAMETER_ENV` allows providing a JSON string containing excluded parameters. This is a good option if the JSON is short and will not often change. Another option is the variable `FUZZAPI_EXCLUDE_PARAMETER_FILE`. This variable is set to a file path that can be checked into the repository, created by another job as an artifact, or generated at runtime from a pre script using `FUZZAPI_PRE_SCRIPT`. +The `FUZZAPI_EXCLUDE_PARAMETER_ENV` allows providing a JSON string containing excluded parameters. This is a good option if the JSON is short and will not often change. Another option is the variable `FUZZAPI_EXCLUDE_PARAMETER_FILE`. This variable is set to a file path that can be checked into the repository, created by another job as an artifact, or generated at runtime from a pre-script using `FUZZAPI_PRE_SCRIPT`. #### Exclude parameters using a JSON document @@ -1821,13 +1821,13 @@ As an alternative to excluding by paths, you can filter by any other component i In your job output you can check if any URLs matched any provided regular expression from `FUZZAPI_EXCLUDE_URLS`. Matching operations are listed in the **Excluded Operations** section. Operations listed in the **Excluded Operations** should not be listed in the **Tested Operations** section. For example the following portion of a job output: ```plaintext -2021-05-27 21:51:08 [INF] API Security: --[ Tested Operations ]------------------------- -2021-05-27 21:51:08 [INF] API Security: 201 POST http://target:7777/api/users CREATED -2021-05-27 21:51:08 [INF] API Security: ------------------------------------------------ -2021-05-27 21:51:08 [INF] API Security: --[ Excluded Operations ]----------------------- -2021-05-27 21:51:08 [INF] API Security: GET http://target:7777/api/messages -2021-05-27 21:51:08 [INF] API Security: POST http://target:7777/api/messages -2021-05-27 21:51:08 [INF] API Security: ------------------------------------------------ +2021-05-27 21:51:08 [INF] API Fuzzing: --[ Tested Operations ]------------------------- +2021-05-27 21:51:08 [INF] API Fuzzing: 201 POST http://target:7777/api/users CREATED +2021-05-27 21:51:08 [INF] API Fuzzing: ------------------------------------------------ +2021-05-27 21:51:08 [INF] API Fuzzing: --[ Excluded Operations ]----------------------- +2021-05-27 21:51:08 [INF] API Fuzzing: GET http://target:7777/api/messages +2021-05-27 21:51:08 [INF] API Fuzzing: POST http://target:7777/api/messages +2021-05-27 21:51:08 [INF] API Fuzzing: ------------------------------------------------ ``` NOTE: @@ -2242,18 +2242,18 @@ The first step to resolving performance issues is to understand what is contribu The API Fuzzing job output contains helpful information about how fast we are testing, how fast each operation being tested responds, and summary information. Let's take a look at some sample output to see how it can be used in tracking down performance issues: ```shell -API Security: Loaded 10 operations from: assets/har-large-response/large_responses.har -API Security: -API Security: Testing operation [1/10]: 'GET http://target:7777/api/large_response_json'. -API Security: - Parameters: (Headers: 4, Query: 0, Body: 0) -API Security: - Request body size: 0 Bytes (0 bytes) -API Security: -API Security: Finished testing operation 'GET http://target:7777/api/large_response_json'. -API Security: - Excluded Parameters: (Headers: 0, Query: 0, Body: 0) -API Security: - Performed 767 requests -API Security: - Average response body size: 130 MB -API Security: - Average call time: 2 seconds and 82.69 milliseconds (2.082693 seconds) -API Security: - Time to complete: 14 minutes, 8 seconds and 788.36 milliseconds (848.788358 seconds) +API Fuzzing: Loaded 10 operations from: assets/har-large-response/large_responses.har +API Fuzzing: +API Fuzzing: Testing operation [1/10]: 'GET http://target:7777/api/large_response_json'. +API Fuzzing: - Parameters: (Headers: 4, Query: 0, Body: 0) +API Fuzzing: - Request body size: 0 Bytes (0 bytes) +API Fuzzing: +API Fuzzing: Finished testing operation 'GET http://target:7777/api/large_response_json'. +API Fuzzing: - Excluded Parameters: (Headers: 0, Query: 0, Body: 0) +API Fuzzing: - Performed 767 requests +API Fuzzing: - Average response body size: 130 MB +API Fuzzing: - Average call time: 2 seconds and 82.69 milliseconds (2.082693 seconds) +API Fuzzing: - Time to complete: 14 minutes, 8 seconds and 788.36 milliseconds (848.788358 seconds) ``` This job console output snippet starts by telling us how many operations were found (10), followed by notifications that testing has started on a specific operation and a summary of the operation has been completed. The summary is the most interesting part of this log output. In the summary, we can see that it took API Fuzzing 767 requests to fully test this operation and its related fields. We can also see that the average response time was 2 seconds and the time to complete was 14 minutes for this one operation. @@ -2443,7 +2443,7 @@ See the following documentation sections for assistance: See [Performance Tuning and Testing Speed](#performance-tuning-and-testing-speed) -### Error waiting for API Security 'http://127.0.0.1:5000' to become available +### Error waiting for API Fuzzing 'http://127.0.0.1:5000' to become available A bug exists in versions of the API Fuzzing analyzer prior to v1.6.196 that can cause a background process to fail under certain conditions. The solution is to update to a newer version of the API Fuzzing analyzer. @@ -2456,6 +2456,11 @@ If the issue is occurring with versions v1.6.196 or greater, contact Support and 1. The `gl-api-security-scanner.log` file available as a job artifact. In the right-hand panel of the job details page, select the **Browse** button. 1. The `apifuzzer_fuzz` job definition from your `.gitlab-ci.yml` file. +**Error message** + +- In [GitLab 15.6 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/376078), `Error waiting for API Fuzzing 'http://127.0.0.1:5000' to become available` +- In GitLab 15.5 and earlier, `Error waiting for API Security 'http://127.0.0.1:5000' to become available`. + ### `Failed to start session with scanner. Please retry, and if the problem persists reach out to support.` The API Fuzzing engine outputs an error message when it cannot establish a connection with the scanner application component. The error message is shown in the job output window of the `apifuzzer_fuzz` job. A common cause for this issue is that the background component cannot use the selected port as it's already in use. This error can occur intermittently if timing plays a part (race condition). This issue occurs most often with Kubernetes environments when other services are mapped into the container causing port conflicts. @@ -2648,6 +2653,50 @@ API Fuzzing uses the specified media types in the OpenAPI document to generate r 1. Review the supported media types in the [OpenAPI Specification](#openapi-specification) section. 1. Edit your OpenAPI document, allowing at least a given operation to accept any of the supported media types. Alternatively, a supported media type could be set in the OpenAPI document level and get applied to all operations. This step may require changes in your application to ensure the supported media type is accepted by the application. +### ``Error, error occurred trying to download `<URL>`: There was an error when retrieving content from Uri:' <URL>'. Error:The SSL connection could not be established, see inner exception.`` + +API fuzzing is compatible with a broad range of TLS configurations, including outdated protocols and ciphers. +Despite broad support, you might encounter connection errors. This error occurs because API fuzzing could not establish a secure connection with the server at the given URL. + +To resolve the issue: + +If the host in the error message supports non-TLS connections, change `https://` to `http://` in your configuration. +For example, if an error occurs with the following configuration: + +```yaml +stages: + - fuzz + +include: + - template: API-Fuzzing.gitlab-ci.yml + +variables: + FUZZAPI_TARGET_URL: https://test-deployment/ + FUZZAPI_OPENAPI: https://specs/openapi.json +``` + +Change the prefix of `FUZZAPI_OPENAPI` from `https://` to `http://`: + +```yaml +stages: + - fuzz + +include: + - template: API-Fuzzing.gitlab-ci.yml + +variables: + FUZZAPI_TARGET_URL: https://test-deployment/ + FUZZAPI_OPENAPI: http://specs/openapi.json +``` + +If you cannot use a non-TLS connection to access the URL, contact the Support team for help. + +You can expedite the investigation with the [testssl.sh tool](https://testssl.sh/). From a machine with a bash shell and connectivity to the affected server: + +1. Download the latest release `zip` or `tar.gz` file and extract from <https://github.com/drwetter/testssl.sh/releases>. +1. Run `./testssl.sh --log https://specs`. +1. Attach the log file to your support ticket. + ## Get support or request an improvement To get support for your particular problem use the [getting help channels](https://about.gitlab.com/get-help/). diff --git a/doc/user/application_security/configuration/index.md b/doc/user/application_security/configuration/index.md index d9ba4640855..ea422f0b33c 100644 --- a/doc/user/application_security/configuration/index.md +++ b/doc/user/application_security/configuration/index.md @@ -67,7 +67,7 @@ You can configure the following security controls: - Can be configured by adding a configuration block to your agent configuration. For more details, read [Operational Container Scanning](../../clusters/agent/vulnerabilities.md#enable-operational-container-scanning). - [Secret Detection](../secret_detection/index.md) - Select **Configure with a merge request** to create a merge request with the changes required to - enable Secret Detection. For more details, read [Enable Secret Detection using a merge request](../secret_detection/index.md#enable-secret-detection-using-a-merge-request). + enable Secret Detection. For more details, read [Use an automatically configured merge request](../secret_detection/index.md#use-an-automatically-configured-merge-request). - [API Fuzzing](../api_fuzzing/index.md) - Select **Enable API Fuzzing** to use API Fuzzing for the current project. For more details, read [API Fuzzing](../../../user/application_security/api_fuzzing/index.md#enable-web-api-fuzzing). - [Coverage Fuzzing](../coverage_fuzzing/index.md) diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 6fc01a716b2..e198f967eea 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -90,12 +90,12 @@ To enable container scanning in your pipeline, you need the following: ## Configuration To enable container scanning, add the -[`Container-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml) +[`Container-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml) to your `.gitlab-ci.yml` file: ```yaml include: - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml ``` The included template: @@ -117,7 +117,7 @@ registry, and scans the image: ```yaml include: - template: Jobs/Build.gitlab-ci.yml - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -142,7 +142,7 @@ enables verbose output for the analyzer: ```yaml include: - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml variables: SECURE_LOG_LEVEL: 'debug' @@ -154,7 +154,7 @@ To scan images located in a registry other than the project's, use the following ```yaml include: - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -178,7 +178,7 @@ container_scanning: - export AWS_ECR_PASSWORD=$(aws ecr get-login-password --region region) include: - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml CS_IMAGE: <aws_account_id>.dkr.ecr.<region>.amazonaws.com/<image>:<tag> CS_REGISTRY_USER: AWS CS_REGISTRY_PASSWORD: "$AWS_ECR_PASSWORD" @@ -199,7 +199,7 @@ For example: ```yaml include: - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -223,7 +223,7 @@ By default, the report only includes packages managed by the Operating System (O ```yaml include: - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -345,7 +345,7 @@ This example sets `GIT_STRATEGY` to `fetch`: ```yaml include: - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -391,7 +391,7 @@ duplicated: ```yaml include: - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -567,7 +567,7 @@ and you may be able to make occasional updates on your own. For more information, see [the specific steps on how to update an image with a pipeline](#automating-container-scanning-vulnerability-database-updates-with-a-pipeline). -For details on saving and transporting Docker images as a file, see Docker's documentation on +For details on saving and transporting Docker images as a file, see the Docker documentation on [`docker save`](https://docs.docker.com/engine/reference/commandline/save/), [`docker load`](https://docs.docker.com/engine/reference/commandline/load/), [`docker export`](https://docs.docker.com/engine/reference/commandline/export/), and [`docker import`](https://docs.docker.com/engine/reference/commandline/import/). @@ -577,7 +577,7 @@ For details on saving and transporting Docker images as a file, see Docker's doc ```yaml include: - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml container_scanning: image: $CI_REGISTRY/namespace/gitlab-container-scanning @@ -628,7 +628,7 @@ This example shows the configuration needed to scan images in a private [Google ```yaml include: - - template: Security/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml container_scanning: variables: diff --git a/doc/user/application_security/dast/authentication.md b/doc/user/application_security/dast/authentication.md new file mode 100644 index 00000000000..d4f91639dbc --- /dev/null +++ b/doc/user/application_security/dast/authentication.md @@ -0,0 +1,527 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: reference, howto +--- + +# Authentication (DAST) **(ULTIMATE)** + +WARNING: +**Never** run an authenticated scan against a production server. +Authenticated scans may perform *any* function that the authenticated user can, +including modifying or deleting data, submitting forms, and following links. +Only run an authenticated scan against a test server. + +Authentication logs a user in before a DAST scan so that the analyzer can test +as much of the application as possible when searching for vulnerabilities. + +DAST uses a browser to authenticate the user so that the login form has the necessary JavaScript +and styling required to submit the form. DAST finds the username and password fields and fills them with their respective values. +The login form is submitted, and when the response returns, a series of checks verify if authentication was successful. +DAST saves the credentials for reuse when crawling the target application. + +If DAST fails to authenticate, the scan halts and the CI job fails. + +Authentication supports single-step login forms, multi-step login forms, single sign-on, and authenticating to URLs outside of the configured target URL. + +## Getting started + +NOTE: +We recommend periodically confirming that the analyzer's authentication is still working, as this tends to break over +time due to changes to the application. + +To run a DAST authenticated scan: + +- Read the [prerequisite](#prerequisites) conditions for authentication. +- [Update your target website](#update-the-target-website) to a landing page of an authenticated user. +- If your login form has the username, password and submit button on a single page, use the [CI/CD variables](#available-cicd-variables) to configure [single-step](#configuration-for-a-single-step-login-form) login form authentication. +- If your login form has the username and password fields on different pages, use the [CI/CD variables](#available-cicd-variables) to configure [multi-step](#configuration-for-a-multi-step-login-form) login form authentication. +- Make sure the user isn't [logged out](#excluding-logout-urls) during the scan. + +### Prerequisites + +- You are using either the [DAST proxy-based analyzer](proxy-based.md) or the [DAST browser-based analyzer](browser_based.md). +- You know the URL of the login form of your application. Alternatively, you know how to navigate to the login form from the authentication URL (see [clicking to navigate to the login form](#clicking-to-navigate-to-the-login-form)). +- You have the username and password of the user you would like to authenticate as during the scan. +- You know the [selectors](#finding-an-elements-selector) of the username and password HTML fields that DAST will use to input the respective values. +- You know the element's [selector](#finding-an-elements-selector) that will submit the login form when selected. +- You have thought about how you can [verify](#verifying-authentication-is-successful) whether or not authentication was successful. +- You have checked the [known limitations](#known-limitations) to ensure DAST can authenticate to your application. + +### Available CI/CD variables + +| CI/CD variable | Type | Description | +|:-----------------------------------------------|:------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `DAST_AUTH_COOKIES` | string | Set to a comma-separated list of cookie names to specify which cookies are used for authentication. | +| `DAST_AUTH_REPORT` | boolean | Used in combination with exporting the `gl-dast-debug-auth-report.html` artifact to aid in debugging authentication issues. | +| `DAST_AUTH_URL` <sup>1</sup> | URL | The URL of the page containing the sign-in HTML form on the target website. `DAST_USERNAME` and `DAST_PASSWORD` are submitted with the login form to create an authenticated scan. Example: `https://login.example.com`. | +| `DAST_AUTH_VERIFICATION_LOGIN_FORM` | boolean | Verifies successful authentication by checking for the absence of a login form once the login form has been submitted. | +| `DAST_AUTH_VERIFICATION_SELECTOR` | [selector](#finding-an-elements-selector) | Verifies successful authentication by checking for presence of a selector once the login form has been submitted. Example: `css:.user-photo`. | +| `DAST_AUTH_VERIFICATION_URL` <sup>1</sup> | URL | Verifies successful authentication by checking the URL in the browser once the login form has been submitted. Example: `"https://example.com/loggedin_page"`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207335) in GitLab 13.8. | +| `DAST_BROWSER_PATH_TO_LOGIN_FORM` <sup>1</sup> | [selector](#finding-an-elements-selector) | Comma-separated list of selectors that are selected prior to attempting to enter `DAST_USERNAME` and `DAST_PASSWORD` into the login form. Example: `"css:.navigation-menu,css:.login-menu-item"`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/326633) in GitLab 14.1. | +| `DAST_EXCLUDE_URLS` <sup>1</sup> | URLs | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. | +| `DAST_FIRST_SUBMIT_FIELD` | string | The `id` or `name` of the element that when selected submits the username form of a multi-page login process. For example, `css:button[type='user-submit']`. [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/9894) in GitLab 12.4. | +| `DAST_PASSWORD` <sup>1</sup> | string | The password to authenticate to in the website. Example: `P@55w0rd!` | +| `DAST_PASSWORD_FIELD` | string | The selector of password field at the sign-in HTML form. Example: `id:password` | +| `DAST_SUBMIT_FIELD` | string | The `id` or `name` of the element that when selected submits the login form or the password form of a multi-page login process. For example, `css:button[type='submit']`. [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/9894) in GitLab 12.4. | +| `DAST_USERNAME` <sup>1</sup> | string | The username to authenticate to in the website. Example: `admin` | +| `DAST_USERNAME_FIELD` <sup>1</sup> | string | The selector of username field at the sign-in HTML form. Example: `name:username` | + +1. Available to an on-demand proxy-based DAST scan. + +### Update the target website + +The target website, defined using the CI/CD variable `DAST_WEBSITE`, is the URL DAST uses to begin crawling your application. + +For best crawl results on an authenticated scan, the target website should be a URL accessible only after the user is authenticated. +Often, this is the URL of the page the user lands on after they're logged in. + +For example: + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_WEBSITE: "https://example.com/dashboard/welcome" + DAST_AUTH_URL: "https://example.com/login" +``` + +### Configuration for a single-step login form + +A single-step login form has all login form elements on a single page. +Configuration requires the CI/CD variables `DAST_AUTH_URL`, `DAST_USERNAME`, `DAST_USERNAME_FIELD`, `DAST_PASSWORD`, `DAST_PASSWORD_FIELD`, and `DAST_SUBMIT_FIELD` to be defined for the DAST job. + +It is recommended to set up the URL and selectors of fields in the job definition YAML, for example: + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_WEBSITE: "https://example.com" + DAST_AUTH_URL: "https://example.com/login" + DAST_USERNAME_FIELD: "css:[name=username]" + DAST_PASSWORD_FIELD: "css:[name=password]" + DAST_SUBMIT_FIELD: "css:button[type=submit]" +``` + +Do **not** define `DAST_USERNAME` and `DAST_PASSWORD` in the YAML job definition file as this could present a security risk. Instead, create them as masked CI/CD variables using the GitLab UI. +See [Custom CI/CI variables](../../../ci/variables/index.md#custom-cicd-variables) for more information. + +### Configuration for a multi-step login form + +A multi-step login form has two pages. The first page has a form with the username and a next submit button. +If the username is valid, a second form on the subsequent page has the password and the form submit button. + +Configuration requires the CI/CD variables `DAST_AUTH_URL`, `DAST_USERNAME`, `DAST_USERNAME_FIELD`, `DAST_FIRST_SUBMIT_FIELD`, `DAST_PASSWORD`, `DAST_PASSWORD_FIELD`, and `DAST_SUBMIT_FIELD` to be defined for the DAST job. + +It is recommended to set up the URL and selectors of fields in the job definition YAML, for example: + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_WEBSITE: "https://example.com" + DAST_AUTH_URL: "https://example.com/login" + DAST_USERNAME_FIELD: "css:[name=username]" + DAST_FIRST_SUBMIT_FIELD: "css:button[name=next]" + DAST_PASSWORD_FIELD: "css:[name=password]" + DAST_SUBMIT_FIELD: "css:button[type=submit]" +``` + +Do **not** define `DAST_USERNAME` and `DAST_PASSWORD` in the YAML job definition file as this could present a security risk. Instead, create them as masked CI/CD variables using the GitLab UI. +See [Custom CI/CI variables](../../../ci/variables/index.md#custom-cicd-variables) for more information. + +### Configuration for Single Sign-On (SSO) + +If a user can log into an application, then in most cases, DAST will also be able to log in. +This is the case even when an application uses Single Sign-on. Applications using SSO solutions should configure DAST +authentication using the [single-step](#configuration-for-a-single-step-login-form) or [multi-step](#configuration-for-a-multi-step-login-form) login form configuration guides. + +DAST supports authentication processes where a user is redirected to an external Identity Provider's site to log in. +Check the [known limitations](#known-limitations) of DAST authentication to determine if your SSO authentication process is supported. + +### Clicking to navigate to the login form + +Define `DAST_BROWSER_PATH_TO_LOGIN_FORM` to provide a path of elements to click on from the `DAST_AUTH_URL` so that DAST can access the +login form. This is useful for applications that show the login form in a pop-up (modal) window or when the login form does not +have a unique URL. + +For example: + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_WEBSITE: "https://example.com" + DAST_AUTH_URL: "https://example.com/login" + DAST_BROWSER_PATH_TO_LOGIN_FORM: "css:.navigation-menu,css:.login-menu-item" +``` + +### Excluding logout URLs + +If DAST crawls the logout URL while running an authenticated scan, the user will be logged out, resulting in the remainder of the scan being unauthenticated. +It is therefore recommended to exclude logout URLs using the CI/CD variable `DAST_EXCLUDE_URLS`. DAST will not access any excluded URLs, ensuring the user remains logged in. + +Provided URLs can be either absolute URLs, or regular expressions of URL paths relative to the base path of the `DAST_WEBSITE`. For example: + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_WEBSITE: "https://example.com/welcome/home" + DAST_EXCLUDE_URLS: "https://example.com/logout,/user/.*/logout" +``` + +### Finding an element's selector + +Selectors are used by CI/CD variables to specify the location of an element displayed on a page in a browser. +Selectors have the format `type`:`search string`. DAST searches for the selector using the search string based on the type. + +| Selector type | Example | Description | +|---------------|------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `css` | `css:.password-field` | Searches for a HTML element having the supplied CSS selector. Selectors should be as specific as possible for performance reasons. | +| `id` | `id:element` | Searches for an HTML element with the provided element ID. | +| `name` | `name:element` | Searches for an HTML element with the provided element name. | +| `xpath` | `xpath://input[@id="my-button"]/a` | Searches for a HTML element with the provided XPath. Note that XPath searches are expected to be less performant than other searches. | +| None provided | `a.click-me` | Defaults to searching using a CSS selector. **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383348)** in GitLab 15.8. Replaced by explicitly declaring the selector type. | + +#### Find selectors with Google Chrome + +Chrome DevTools element selector tool is an effective way to find a selector. + +1. Open Chrome and navigate to the page where you would like to find a selector, for example, the login page for your site. +1. Open the `Elements` tab in Chrome DevTools with the keyboard shortcut `Command + Shift + c` in macOS or `Ctrl + Shift + c` in Windows. +1. Select the `Select an element in the page to select it` tool. +  +1. Select the field on your page that you would like to know the selector for. +1. Once the tool is active, highlight a field you wish to view the details of. +  +1. Once highlighted, you can see the element's details, including attributes that would make a good candidate for a selector. + +In this example, the `id="user_login"` appears to be a good candidate. You can use this as a selector as the DAST username field by setting +`DAST_USERNAME_FIELD: "id:user_login"`. + +#### Choose the right selector + +Judicious choice of selector leads to a scan that is resilient to the application changing. + +In order of preference, it is recommended to choose as selectors: + +- `id` fields. These are generally unique on a page, and rarely change. +- `name` fields. These are generally unique on a page, and rarely change. +- `class` values specific to the field, such as the selector `"css:.username"` for the `username` class on the username field. +- Presence of field specific data attributes, such as the selector, `"css:[data-username]"` when the `data-username` field has any value on the username field. +- Multiple `class` hierarchy values, such as the selector `"css:.login-form .username"` when there are multiple elements with class `username` but only one nested inside the element with the class `login-form`. + +When using selectors to locate specific fields we recommend you avoid searching on: + +- Any `id`, `name`, `attribute`, `class` or `value` that is dynamically generated. +- Generic class names, such as `column-10` and `dark-grey`. +- XPath searches as they are less performant than other selector searches. +- Unscoped searches, such as those beginning with `css:*` and `xpath://*`. + +## Verifying authentication is successful + +Once DAST has submitted the login form, a verification process takes place +to determine if authentication succeeded. The scan will halt with an error if authentication is unsuccessful. + +Following the submission of the login form, authentication is determined to be unsuccessful when: + +- The login submit HTTP response has a `400` or `500` series status code. +- Any [verification check](#verification-checks) fails. +- An [authentication token](#authentication-tokens) with a sufficiently random value is not set during the authentication process. + +### Verification checks + +Verification checks run checks on the state of the browser once authentication is complete +to determine further if authentication succeeded. + +DAST will test for the absence of a login form if no verification checks are configured. + +#### Verify based on the URL + +Define `DAST_AUTH_VERIFICATION_URL` as the URL displayed in the browser tab once the login form is successfully submitted. + +DAST will compare the verification URL to the URL in the browser after authentication. +If they are not the same, authentication is unsuccessful. + +For example: + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_WEBSITE: "https://example.com" + DAST_AUTH_VERIFICATION_URL: "https://example.com/user/welcome" +``` + +#### Verify based on presence of an element + +Define `DAST_AUTH_VERIFICATION_SELECTOR` as a [selector](#finding-an-elements-selector) that will find one or many elements on the page +displayed once the login form is successfully submitted. If no element is found, authentication is unsuccessful. +Searching for the selector on the page displayed when login fails should return no elements. + +For example: + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_WEBSITE: "https://example.com" + DAST_AUTH_VERIFICATION_SELECTOR: "css:.welcome-user" +``` + +#### Verify based on absence of a login form + +Define `DAST_AUTH_VERIFICATION_LOGIN_FORM` as `"true"` to indicate that DAST should search for the login form on the +page displayed once the login form is successfully submitted. If a login form is still present after logging in, authentication is unsuccessful. + +For example: + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_WEBSITE: "https://example.com" + DAST_AUTH_VERIFICATION_LOGIN_FORM: "true" +``` + +### Authentication tokens + +DAST records authentication tokens set during the authentication process. +Authentication tokens are loaded into new browsers when DAST opens them so the user can remain logged in throughout the scan. + +To record tokens, DAST takes a snapshot of cookies, local storage, and session storage values set by the application before +the authentication process. DAST does the same after authentication and uses the difference to determine which were created +by the authentication process. + +DAST considers cookies, local storage and session storage values set with sufficiently "random" values to be authentication tokens. +For example, `sessionID=HVxzpS8GzMlPAc2e39uyIVzwACIuGe0H` would be viewed as an authentication token, while `ab_testing_group=A1` would not. + +The CI/CD variable `DAST_AUTH_COOKIES` can be used to specify the names of authentication cookies and bypass the randomness check used by DAST. +Not only can this make the authentication process more robust, but it can also increase vulnerability check accuracy for checks that +inspect authentication tokens. + +For example: + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_WEBSITE: "https://example.com" + DAST_AUTH_COOKIES: "sessionID,refreshToken" +``` + +## Known limitations + +- DAST cannot bypass a CAPTCHA if the authentication flow includes one. Please turn these off in the testing environment for the application being scanned. +- DAST cannot handle multi-factor authentication like one-time passwords (OTP) by using SMS, biometrics, or authenticator apps. Please turn these off in the testing environment for the application being scanned. +- DAST cannot authenticate to applications that do not set an [authentication token](#authentication-tokens) during login. +- DAST cannot authenticate to applications that require more than two inputs to be filled out. Two inputs must be supplied, username and password. + +## Troubleshooting + +The [logs](#read-the-logs) provide insight into what DAST is doing and expecting during the authentication process. For more detailed +information, configure the [authentication report](#configure-the-authentication-report). + +For more information about particular error messages or situations see [known problems](#known-problems). + +The browser-based analyzer is used to authenticate the user. For advanced troubleshooting, see [browser-based troubleshooting](browser_based_troubleshooting.md). + +### Read the logs + +The console output of the DAST CI/CD job shows information about the authentication process using the `AUTH` log module. +For example, the following log shows failed authentication for a multi-step login form. +Authentication failed because a home page should be displayed after login. Instead, the login form was still present. + +```plaintext +2022-11-16T13:43:02.000 INF AUTH attempting to authenticate +2022-11-16T13:43:02.000 INF AUTH loading login page LoginURL=https://example.com/login +2022-11-16T13:43:10.000 INF AUTH multi-step authentication detected +2022-11-16T13:43:15.000 INF AUTH verifying if user submit was successful true_when="HTTP status code < 400" +2022-11-16T13:43:15.000 INF AUTH requirement is satisfied, no login HTTP message detected want="HTTP status code < 400" +2022-11-16T13:43:20.000 INF AUTH verifying if login attempt was successful true_when="HTTP status code < 400 and has authentication token and no login form found (no element found when searching using selector css:[id=email] or css:[id=password] or css:[id=submit])" +2022-11-24T14:43:20.000 INF AUTH requirement is satisfied, HTTP login request returned status code 200 url=https://example.com/user/login?error=invalid%20credentials want="HTTP status code < 400" +2022-11-16T13:43:21.000 INF AUTH requirement is unsatisfied, login form was found want="no login form found (no element found when searching using selector css:[id=email] or css:[id=password] or css:[id=submit])" +2022-11-16T13:43:21.000 INF AUTH login attempt failed error="authentication failed: failed to authenticate user" +``` + +### Configure the authentication report + +An authentication report can be saved as a CI/CD job artifact to assist with understanding the cause of an authentication failure. + +The report contains steps during the login process, HTTP requests and responses, the Document Object Model (DOM) and screenshots. + + + +An example configuration where the authentication debug report is exported may look like the following: + +```yaml +dast: + variables: + DAST_WEBSITE: "https://example.com" + DAST_AUTH_REPORT: "true" + artifacts: + paths: [gl-dast-debug-auth-report.html] + when: always +``` + +### Known problems + +#### Login form not found + +DAST failed to find a login form when loading the login page, often because the authentication URL could not be loaded. +The log reports a fatal error such as: + +```plaintext +2022-12-07T12:44:02.838 INF AUTH loading login page LoginURL=[authentication URL] +2022-12-07T12:44:11.119 FTL MAIN authentication failed: login form not found +``` + +Suggested actions: + +- Generate the [authentication report](#configure-the-authentication-report) to inspect HTTP response. +- Check the target application authentication is deployed and running. +- Check the `DAST_AUTH_URL` is correct. +- Check the GitLab Runner can access the `DAST_AUTH_URL`. +- Check the `DAST_BROWSER_PATH_TO_LOGIN_FORM` is valid if used. + +#### Scan doesn't crawl authenticated pages + +If DAST captures the wrong [authentication tokens](#authentication-tokens) during the authentication process then +the scan can't crawl authenticated pages. Names of cookies and storage authentication tokens are written to the log. For example: + +```plaintext +2022-11-24T14:42:31.492 INF AUTH authentication token cookies names=["sessionID"] +2022-11-24T14:42:31.492 INF AUTH authentication token storage events keys=["token"] +``` + +Suggested actions: + +- Generate the [authentication report](#configure-the-authentication-report) and look at the screenshot from the `Login submit` to verify that the login worked as expected. +- Verify the logged authentication tokens are those used by your application. +- If using cookies to store authentication tokens, set the names of the authentication token cookies using `DAST_AUTH_COOKIES`. + +#### Unable to find elements with selector + +DAST failed to find the username, password, first submit button, or submit button elements. The log reports a fatal error such as: + +```plaintext +2022-12-07T13:14:11.545 FTL MAIN authentication failed: unable to find elements with selector: css:#username +``` + +Suggested actions: + +- Generate the [authentication report](#configure-the-authentication-report) to use the screenshot from the `Login page` to verify that the page loaded correctly. +- Load the login page in a browser and verify the [selectors](#finding-an-elements-selector) configured in `DAST_USERNAME_FIELD`, `DAST_PASSWORD_FIELD`, `DAST_FIRST_SUBMIT_FIELD`, and `DAST_SUBMIT_FIELD` are correct. + +#### Failed to authenticate user + +DAST failed to authenticate due to a failed login verification check. The log reports a fatal error such as: + +```plaintext +2022-12-07T06:39:49.483 INF AUTH verifying if login attempt was successful true_when="HTTP status code < 400 and has authentication token and no login form found (no element found when searching using selector css:[name=username] or css:[name=password] or css:button[type=\"submit\"])" +2022-12-07T06:39:49.484 INF AUTH requirement is satisfied, HTTP login request returned status code 303 url=http://auth-manual:8090/login want="HTTP status code < 400" +2022-12-07T06:39:49.513 INF AUTH requirement is unsatisfied, login form was found want="no login form found (no element found when searching using selector css:[name=username] or css:[name=password] or css:button[type=\"submit\"])" +2022-12-07T06:39:49.589 INF AUTH login attempt failed error="authentication failed: failed to authenticate user" +2022-12-07T06:39:53.626 FTL MAIN authentication failed: failed to authenticate user +``` + +Suggested actions: + +- Look in the log for the `requirement is unsatisfied`. Respond to the appropriate error. + +#### Requirement unsatisfied, login form was found + +Applications typically display a dashboard when the user logs in and the login form with an error message when the +username or password is incorrect. + +This error occurs when DAST detects the login form on the page displayed after authenticating the user, +indicating that the login attempt failed. + +```plaintext +2022-12-07T06:39:49.513 INF AUTH requirement is unsatisfied, login form was found want="no login form found (no element found when searching using selector css:[name=username] or css:[name=password] or css:button[type=\"submit\"])" +``` + +Suggested actions: + +- Verify that the username and password/authentication credentials used are correct. +- Generate the [authentication report](#configure-the-authentication-report) and verify the `Request` for the `Login submit` is correct. +- It's possible that the authentication report `Login submit` request and response are empty. This occurs when there is no request that would result + in a full page reload, such as a request made when submitting a HTML form. This occurs when using websockets or AJAX to submit the login form. +- If the page displayed following user authentication genuinely has elements matching the login form selectors, configure `DAST_AUTH_VERIFICATION_URL` + or `DAST_AUTH_VERIFICATION_SELECTOR` to use an alternate method of verifying the login attempt. + +#### Requirement unsatisfied, selector returned no results + +DAST cannot find an element matching the selector provided in `DAST_AUTH_VERIFICATION_SELECTOR` on the page displayed following user login. + +```plaintext +2022-12-07T06:39:33.239 INF AUTH requirement is unsatisfied, searching DOM using selector returned no results want="has element css:[name=welcome]" +``` + +Suggested actions: + +- Generate the [authentication report](#configure-the-authentication-report) and look at the screenshot from the `Login submit` to verify that the expected page is displayed. +- Ensure the `DAST_AUTH_VERIFICATION_SELECTOR` [selector](#finding-an-elements-selector) is correct. + +#### Requirement unsatisfied, browser not at URL + +DAST detected that the page displayed following user login has a URL different to what was expected according to `DAST_AUTH_VERIFICATION_URL`. + +```plaintext +2022-12-07T11:28:00.241 INF AUTH requirement is unsatisfied, browser is not at URL browser_url="https://example.com/home" want="is at url https://example.com/user/dashboard" +``` + +Suggested actions: + +- Generate the [authentication report](#configure-the-authentication-report) and look at the screenshot from the `Login submit` to verify that the expected page is displayed. +- Ensure the `DAST_AUTH_VERIFICATION_URL` is correct. + +#### Requirement unsatisfied, HTTP login request status code + +The HTTP response when loading the login form or submitting the form had a status code of 400 (client error) +or 500 (server error). + +```plaintext +2022-12-07T06:39:53.626 INF AUTH requirement is unsatisfied, HTTP login request returned status code 502 url="https://example.com/user/login" want="HTTP status code < 400" +``` + +- Verify that the username and password/authentication credentials used are correct. +- Generate the [authentication report](#configure-the-authentication-report) and verify the `Request` for the `Login submit` is correct. +- Verify the target application works as expected. + +#### Requirement unsatisfied, no authentication token + +DAST could not detect an [authentication token](#authentication-tokens) created during the authentication process. + +```plaintext +2022-12-07T11:25:29.010 INF AUTH authentication token cookies names=[] +2022-12-07T11:25:29.010 INF AUTH authentication token storage events keys=[] +2022-12-07T11:25:29.010 INF AUTH requirement is unsatisfied, no basic authentication, cookie or storage event authentication token detected want="has authentication token" +``` + +Suggestion actions: + +- Generate the [authentication report](#configure-the-authentication-report) and look at the screenshot from the `Login submit` to verify that the login worked as expected. +- Using the browser's developer tools, investigate the cookies and local/session storage objects created while logging in. Ensure there is an authentication token created with sufficiently random value. +- If using cookies to store authentication tokens, set the names of the authentication token cookies using `DAST_AUTH_COOKIES`. diff --git a/doc/user/application_security/dast/browser_based.md b/doc/user/application_security/dast/browser_based.md index c0a97c0ff92..7377f31d0ce 100644 --- a/doc/user/application_security/dast/browser_based.md +++ b/doc/user/application_security/dast/browser_based.md @@ -10,39 +10,144 @@ type: reference, howto > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/323423) in GitLab 13.12. WARNING: -This product is in an early-access stage and is considered a [beta](../../../policy/alpha-beta-support.md#beta-features) feature. +This product is in an early-access stage and is considered a [beta](../../../policy/alpha-beta-support.md#beta-features) +feature. -GitLab DAST's browser-based analyzer was built by GitLab to test Single Page Applications (SPAs) and -traditional web applications. It both crawls the web application and analyzes the resulting output -for vulnerabilities. Analysis of modern applications, heavily reliant on JavaScript, is vital to -ensuring DAST coverage. +WARNING: +Do not run DAST scans against a production server. Not only can it perform *any* function that +a user can, such as clicking buttons or submitting forms, but it may also trigger bugs, leading to modification or loss of production data. Only run DAST scans against a test server. + +The DAST browser-based analyzer was built by GitLab to scan modern-day web applications for vulnerabilities. +Scans run in a browser to optimize testing applications heavily dependent on JavaScript, such as single-page applications. +See [how DAST scans an application](#how-dast-scans-an-application) for more information. + +To add the analyzer to your CI/CD pipeline, see [getting started](#getting-started). + +## How DAST scans an application + +A scan performs the following steps: + +1. [Authenticate](authentication.md), if configured. +1. [Crawl](#crawling-an-application) the target application to discover the surface area of the application by performing user actions such as following links, clicking buttons, and filling out forms. +1. [Passive scan](#passive-scans) to search for vulnerabilities in HTTP messages and pages discovered while crawling. +1. [Active scan](#active-scans) to search for vulnerabilities by injecting payloads into HTTP requests recorded during the crawl phase. + +### Crawling an application + +A "navigation" is an action a user might take on a page, such as clicking buttons, clicking anchor links, opening menu items, or filling out forms. +A "navigation path" is a sequence of navigation actions representing how a user might traverse an application. +DAST discovers the surface area of an application by crawling pages and content and identifying navigation paths. + +Crawling is initialized with a navigation path containing one navigation that loads the target application URL in a specially-instrumented Chromium browser. +DAST then crawls navigation paths until all have been crawled. + +To crawl a navigation path, DAST opens a browser window and instructs it to perform all the navigation actions in the navigation path. +When the browser has finished loading the result of the final action, DAST inspects the page for actions a user might take, +creates a new navigation for each found, and adds them to the navigation path to form new navigation paths. For example: + +- DAST processes navigation path `LoadURL[https://example.com]`. +- DAST finds two user actions, `LeftClick[class=menu]` and `LeftClick[id=users]`. +- DAST creates two new navigation paths, `LoadURL[https://example.com] -> LeftClick[class=menu]` and `LoadURL[https://example.com] -> LeftClick[id=users]`. +- Crawling begins on the two new navigation paths. + +It's common for an HTML element to exist in multiple places in an application, such as a menu visible on every page. +Duplicate elements can cause crawlers to crawl the same pages again or become stuck in a loop. +DAST uses an element uniqueness calculation based on HTML attributes to discard new navigation actions it has previously crawled. + +### Passive scans + +Passive scans check for vulnerabilities in the pages discovered during the crawl phase of the scan. +Passive scans are enabled by default. + +The checks search HTTP messages, cookies, storage events, console events, and DOM for vulnerabilities. +Examples of passive checks include searching for exposed credit cards, exposed secret tokens, missing content security policies, and redirection to untrusted locations. + +See [checks](checks/index.md) for more information about individual checks. + +### Active scans + +Active scans check for vulnerabilities by injecting attack payloads into HTTP requests recorded during the crawl phase of the scan. +Active scans are disabled by default due to the nature of their probing attacks. + +DAST analyzes each recorded HTTP request for injection locations, such as query values, header values, cookie values, form posts, and JSON string values. +Attack payloads are injected into the injection location, forming a new request. +DAST sends the request to the target application and uses the HTTP response to determine attack success. + +Active scans run two types of active check: + +- A match response attack analyzes the response content to determine attack success. For example, if an attack attempts to read the system password file, a finding is created when the response body contains evidence of the password file. +- A timing attack uses the response time to determine attack success. For example, if an attack attempts to force the target application to sleep, a finding is created when the application takes longer to respond than the sleep time. Timing attacks are repeated multiple times with different attack payloads to minimize false positives. + +A simplified timing attack works as follows: + +1. The crawl phase records the HTTP request `https://example.com?search=people`. +1. DAST analyzes the URL and finds a URL parameter injection location `https://example.com?search=[INJECT]`. +1. The active check defines a payload, `sleep 10`, that attempts to get a Linux host to sleep. +1. DAST send a new HTTP request to the target application with the injected payload `https://example.com?search=sleep%2010`. +1. The target application is vulnerable if it executes the query parameter value as a system command without validation, for example, `system(params[:search])` +1. DAST creates a finding if the response time takes longer than 10 seconds. + +## Getting started + +To run a DAST scan: + +- Read the [prerequisite](index.md#prerequisites) conditions for running a DAST scan. +- Create a [DAST job](#create-a-dast-cicd-job) in your CI/CD pipeline. +- [Authenticate](#authentication) as a user if your application requires it. + +### Create a DAST CI/CD job + +> - This template was [updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62597) to DAST_VERSION: 2 in + GitLab 14.0. +> - This template was [updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87183) to DAST_VERSION: 3 in + GitLab 15.0. + +To add DAST scanning to your application, use the DAST job defined +in the GitLab DAST CI/CD template file. Updates to the template are provided with GitLab +upgrades, allowing you to benefit from any improvements and additions. + +To create the CI/CD job: + +1. Include the appropriate CI/CD template: -The browser-based scanner works by loading the target application into a specially-instrumented -Chromium browser. A snapshot of the page is taken before a search to find any actions that a user -might perform, such as selecting on a link or filling in a form. For each action found, the -browser-based scanner executes it, takes a new snapshot, and determines what in the page changed -from the previous snapshot. Crawling continues by taking more snapshots and finding subsequent -actions. The benefit of scanning by following user actions in a browser is that the crawler can -interact with the target application much like a real user would, identifying complex flows that -traditional web crawlers don't understand. This results in better coverage of the website. + - [`DAST.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml): + Stable version of the DAST CI/CD template. + - [`DAST.latest.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml): + Latest version of the DAST template. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/254325) + in GitLab 13.8). -The browser-based scanner should provide greater coverage for most web applications, compared -with the current DAST AJAX crawler. While both crawlers are -used together with the current DAST scanner, the combination of the browser-based crawler with the -current DAST scanner is much more effective at finding and testing every page in an application. + WARNING: + The latest version of the template may include breaking changes. Use the + stable template unless you need a feature provided only in the latest template. -## Enable browser-based analyzer + For more information about template versioning, see the + [CI/CD documentation](../../../development/cicd/templates.md#latest-version). -To enable the browser-based analyzer: +1. Add a `dast` stage to your GitLab CI/CD stages configuration. -1. Ensure the DAST [prerequisites](index.md#prerequisites) are met. -1. Include the [DAST CI/CD template](proxy-based.md#include-the-dast-template). -1. Set the target website using the [`DAST_WEBSITE` CI/CD variable](proxy-based.md#available-cicd-variables). -1. Set the CI/CD variable `DAST_BROWSER_SCAN` to `true`. +1. Define the URL to be scanned by DAST by using one of these methods: -Example extract of `.gitlab-ci.yml` file: + - Set the `DAST_WEBSITE` [CI/CD variable](../../../ci/yaml/index.md#variables). + If set, this value takes precedence. + + - Adding the URL in an `environment_url.txt` file at your project's root is great for testing in + dynamic environments. To run DAST against an application dynamically created during a GitLab CI/CD + pipeline, write the application URL to an `environment_url.txt` file. DAST automatically reads the + URL to find the scan target. + + You can see an [example of this in our Auto DevOps CI YAML](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml). + +1. Set the `DAST_BROWSER_SCAN` [CI/CD variable](../../../ci/yaml/index.md#variables) to `"true"`. + +For example: ```yaml +stages: + - build + - test + - deploy + - dast + include: - template: DAST.gitlab-ci.yml @@ -52,36 +157,58 @@ dast: DAST_BROWSER_SCAN: "true" ``` +### Authentication + +The browser-based analyzer can authenticate a user prior to a scan. See [Authentication](authentication.md) for +configuration instructions. + ### Available CI/CD variables -The browser-based crawler can be configured using CI/CD variables. - -| CI/CD variable | Type | Example | Description | -|----------------------------------------------| ----------------| --------------------------------- | ------------| -| `DAST_WEBSITE` | URL | `http://www.site.com` | The URL of the website to scan. | -| `DAST_BROWSER_SCAN` | boolean | `true` | Configures DAST to use the browser-based crawler engine. | -| `DAST_BROWSER_ALLOWED_HOSTS` | List of strings | `site.com,another.com` | Hostnames included in this variable are considered in scope when crawled. By default the `DAST_WEBSITE` hostname is included in the allowed hosts list. | -| `DAST_BROWSER_EXCLUDED_HOSTS` | List of strings | `site.com,another.com` | Hostnames included in this variable are considered excluded and connections are forcibly dropped. | -| `DAST_BROWSER_EXCLUDED_ELEMENTS` | selector | `a[href='2.html'],css:.no-follow` | Comma-separated list of selectors that are ignored when scanning. | -| `DAST_BROWSER_IGNORED_HOSTS` | List of strings | `site.com,another.com` | Hostnames included in this variable are accessed but not reported against. | -| `DAST_BROWSER_MAX_ACTIONS` | number | `10000` | The maximum number of actions that the crawler performs. For example, selecting a link, or filling a form. | -| `DAST_BROWSER_MAX_DEPTH` | number | `10` | The maximum number of chained actions that the crawler takes. For example, `Click -> Form Fill -> Click` is a depth of three. | -| `DAST_BROWSER_NUMBER_OF_BROWSERS` | number | `3` | The maximum number of concurrent browser instances to use. For shared runners on GitLab.com, we recommended a maximum of three. Private runners with more resources may benefit from a higher number, but are likely to produce little benefit after five to seven instances. | -| `DAST_BROWSER_COOKIES` | dictionary | `abtesting_group:3,region:locked` | A cookie name and value to be added to every request. | -| `DAST_BROWSER_LOG` | List of strings | `brows:debug,auth:debug` | A list of modules and their intended log level. | -| `DAST_BROWSER_NAVIGATION_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `15s` | The maximum amount of time to wait for a browser to navigate from one page to another. | -| `DAST_BROWSER_ACTION_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to complete an action. | -| `DAST_BROWSER_STABILITY_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis. | -| `DAST_BROWSER_NAVIGATION_STABILITY_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis after a navigation completes. | -| `DAST_BROWSER_ACTION_STABILITY_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `800ms` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis after completing an action. | -| `DAST_BROWSER_SEARCH_ELEMENT_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `3s` | The maximum amount of time to allow the browser to search for new elements or navigations. | -| `DAST_BROWSER_EXTRACT_ELEMENT_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `5s` | The maximum amount of time to allow the browser to extract newly found elements or navigations. | -| `DAST_BROWSER_ELEMENT_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `600ms` | The maximum amount of time to wait for an element before determining it is ready for analysis. | -| `DAST_BROWSER_PAGE_READY_SELECTOR` | selector | `css:#page-is-ready` | Selector that when detected as visible on the page, indicates to the analyzer that the page has finished loading and the scan can continue. Note: When this selector is set, but the element is not found, the scanner waits for the period defined in `DAST_BROWSER_STABILITY_TIMEOUT` before continuing the scan. This can significantly increase scanning time if the element is not present on multiple pages within the site. | - -The [DAST variables](proxy-based.md#available-cicd-variables) `SECURE_ANALYZERS_PREFIX`, `DAST_FULL_SCAN_ENABLED`, `DAST_AUTO_UPDATE_ADDONS`, `DAST_EXCLUDE_RULES`, `DAST_REQUEST_HEADERS`, `DAST_HTML_REPORT`, `DAST_MARKDOWN_REPORT`, `DAST_XML_REPORT`, -`DAST_AUTH_URL`, `DAST_USERNAME`, `DAST_PASSWORD`, `DAST_USERNAME_FIELD`, `DAST_PASSWORD_FIELD`, `DAST_FIRST_SUBMIT_FIELD`, `DAST_SUBMIT_FIELD`, `DAST_EXCLUDE_URLS`, `DAST_AUTH_VERIFICATION_URL`, `DAST_BROWSER_AUTH_VERIFICATION_SELECTOR`, `DAST_BROWSER_AUTH_VERIFICATION_LOGIN_FORM`, `DAST_BROWSER_AUTH_REPORT`, -`DAST_INCLUDE_ALPHA_VULNERABILITIES`, `DAST_PATHS_FILE`, `DAST_PATHS`, `DAST_ZAP_CLI_OPTIONS`, and `DAST_ZAP_LOG_CONFIGURATION` are also compatible with browser-based crawler scans. +These CI/CD variables are specific to the browser-based DAST analyzer. They can be used to customize the behavior of +DAST to your requirements. +For authentication CI/CD variables, see [Authentication](authentication.md). + +| CI/CD variable | Type | Example | Description | +|:--------------------------------------------|:---------------------------------------------------------|----------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `DAST_ADVERTISE_SCAN` | boolean | `true` | Set to `true` to add a `Via` header to every request sent, advertising that the request was sent as part of a GitLab DAST scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334947) in GitLab 14.1. | +| `DAST_BROWSER_ACTION_STABILITY_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `800ms` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis after completing an action. | +| `DAST_BROWSER_ACTION_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to complete an action. | +| `DAST_BROWSER_ALLOWED_HOSTS` | List of strings | `site.com,another.com` | Hostnames included in this variable are considered in scope when crawled. By default the `DAST_WEBSITE` hostname is included in the allowed hosts list. | +| `DAST_BROWSER_COOKIES` | dictionary | `abtesting_group:3,region:locked` | A cookie name and value to be added to every request. | +| `DAST_BROWSER_CRAWL_GRAPH` | boolean | `true` | Set to `true` to generate an SVG graph of navigation paths visited during crawl phase of the scan. | +| `DAST_BROWSER_DEVTOOLS_LOG` | string | `Default:messageAndBody,truncate:2000` | Set to log protocol messages between DAST and the Chromium browser. | | +| `DAST_BROWSER_ELEMENT_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `600ms` | The maximum amount of time to wait for an element before determining it is ready for analysis. | +| `DAST_BROWSER_EXCLUDED_ELEMENTS` | selector | `a[href='2.html'],css:.no-follow` | Comma-separated list of selectors that are ignored when scanning. | +| `DAST_BROWSER_EXCLUDED_HOSTS` | List of strings | `site.com,another.com` | Hostnames included in this variable are considered excluded and connections are forcibly dropped. | +| `DAST_BROWSER_EXTRACT_ELEMENT_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `5s` | The maximum amount of time to allow the browser to extract newly found elements or navigations. | +| `DAST_BROWSER_FILE_LOG` | List of strings | `brows:debug,auth:debug` | A list of modules and their intended logging level for use in the file log. | +| `DAST_BROWSER_FILE_LOG_PATH` | string | `/output/browserker.log` | Set to the path of the file log. | +| `DAST_BROWSER_IGNORED_HOSTS` | List of strings | `site.com,another.com` | Hostnames included in this variable are accessed, not attacked, and not reported against. | +| `DAST_BROWSER_INCLUDE_ONLY_RULES` | List of strings | `16.1,16.2,16.3` | Comma-separated list of check identifiers to use for the scan. | +| `DAST_BROWSER_LOG` | List of strings | `brows:debug,auth:debug` | A list of modules and their intended logging level for use in the console log. | +| `DAST_BROWSER_LOG_CHROMIUM_OUTPUT` | boolean | `true` | Set to `true` to log Chromium `STDOUT` and `STDERR`. | +| `DAST_BROWSER_MAX_ACTIONS` | number | `10000` | The maximum number of actions that the crawler performs. For example, selecting a link, or filling a form. | +| `DAST_BROWSER_MAX_DEPTH` | number | `10` | The maximum number of chained actions that the crawler takes. For example, `Click -> Form Fill -> Click` is a depth of three. | +| `DAST_BROWSER_MAX_RESPONSE_SIZE_MB` | number | `15` | The maximum size of a HTTP response body. Responses with bodies larger than this are blocked by the browser. Defaults to 10 MB. | +| `DAST_BROWSER_NAVIGATION_STABILITY_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis after a navigation completes. | +| `DAST_BROWSER_NAVIGATION_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `15s` | The maximum amount of time to wait for a browser to navigate from one page to another. | +| `DAST_BROWSER_NUMBER_OF_BROWSERS` | number | `3` | The maximum number of concurrent browser instances to use. For shared runners on GitLab.com, we recommended a maximum of three. Private runners with more resources may benefit from a higher number, but are likely to produce little benefit after five to seven instances. | +| `DAST_BROWSER_PAGE_READY_SELECTOR` | selector | `css:#page-is-ready` | Selector that when detected as visible on the page, indicates to the analyzer that the page has finished loading and the scan can continue. | +| `DAST_BROWSER_SCAN` | boolean | `true` | Required to be `true` to run a browser-based scan. | +| `DAST_BROWSER_SEARCH_ELEMENT_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `3s` | The maximum amount of time to allow the browser to search for new elements or user actions. | +| `DAST_BROWSER_STABILITY_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis. | +| `DAST_EXCLUDE_RULES` | string | `10020,10026` | Set to a comma-separated list of ZAP Vulnerability Rule IDs to exclude them from running during the scan. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). | +| `DAST_EXCLUDE_URLS` | URLs | `https://example.com/.*/sign-out` | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. | +| `DAST_FULL_SCAN_ENABLED` | boolean | `true` | Set to `true` to run both passive and active checks. Default: `false` | +| `DAST_PATHS` | string | `/page1.html,/category1/page3.html` | Set to a comma-separated list of URL paths relative to `DAST_WEBSITE` for DAST to scan. | +| `DAST_PATHS_FILE` | string | `/builds/project/urls.txt` | Set to a file path containing a list of URL paths relative to `DAST_WEBSITE` for DAST to scan. The file must be plain text with one path per line. | +| `DAST_PKCS12_CERTIFICATE_BASE64` | string | `ZGZkZ2p5NGd...` | The PKCS12 certificate used for sites that require Mutual TLS. Must be encoded as base64 text. | +| `DAST_PKCS12_PASSWORD` | string | `password` | The password of the certificate used in `DAST_PKCS12_CERTIFICATE_BASE64`. Create sensitive [custom CI/CI variables](../../../ci/variables/index.md#custom-cicd-variables) using the GitLab UI. | +| `DAST_REQUEST_HEADERS` | string | `Cache-control:no-cache` | Set to a comma-separated list of request header names and values. Headers are added to every request made to `DAST_BROWSER_ALLOWED_HOSTS` by DAST. | +| `DAST_SKIP_TARGET_CHECK` | boolean | `true` | Set to `true` to prevent DAST from checking that the target is available before scanning. Default: `false`. | +| `DAST_TARGET_AVAILABILITY_TIMEOUT` | number | `60` | Time limit in seconds to wait for target availability. | +| `DAST_WEBSITE` | URL | `https://example.com` | The URL of the website to scan. | +| `SECURE_ANALYZERS_PREFIX` | URL | `registry.organization.com` | Set the Docker registry base address from which to download the analyzer. | ## Vulnerability detection @@ -150,53 +277,8 @@ dast: NOTE: Adjusting these values may impact scan time because they adjust how long each browser waits for various activities to complete. -## Debugging scans using logging - -Logging can be used to help you troubleshoot a scan. - -The CI/CD variable `DAST_BROWSER_LOG` configures the logging level for particular modules of the crawler. Each module represents a component of the browser-based crawler and is separated so that debug logs can be configured just for the area of the crawler that requires further inspection. For more details, see [Crawler modules](#crawler-modules). - -For example, the following job definition enables the browsing module and the authentication module to be logged in debug-mode: - -```yaml -include: - - template: DAST.gitlab-ci.yml - -dast: - variables: - DAST_WEBSITE: "https://my.site.com" - DAST_BROWSER_SCAN: "true" - DAST_BROWSER_LOG: "brows:debug,auth:debug" -``` - -### Log message format +## Artifacts -Log messages have the format `[time] [log level] [log module] [message] [additional properties]`. For example, the following log entry has level `INFO`, is part of the `CRAWL` log module, and has the message `Crawled path`. - -```txt -2021-04-21T00:34:04.000 INF CRAWL Crawled path nav_id=0cc7fd path="LoadURL [https://my.site.com:8090]" -``` - -### Crawler modules - -The modules that can be configured for logging are as follows: - -| Log module | Component overview | -| ---------- | ----------- | -| `AUTH` | Used for creating an authenticated scan. | -| `BROWS` | Used for querying the state or page of the browser. | -| `BPOOL` | The set of browsers that are leased out for crawling. | -| `CRAWL` | Used for the core crawler algorithm. | -| `DATAB` | Used for persisting data to the internal database. | -| `LEASE` | Used to create browsers to add them to the browser pool. | -| `MAIN` | Used for the flow of the main event loop of the crawler. | -| `NAVDB` | Used for persistence mechanisms to store navigation entries. | -| `REPT` | Used for generating reports. | -| `STAT` | Used for general statistics while running the scan. | - -### Artifacts - -DAST's browser-based analyzer generates artifacts that can help you understand how the scanner works. Using the latest version of the DAST [template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml) these artifacts are exposed for download by default. The list of artifacts includes the following files: @@ -204,3 +286,7 @@ The list of artifacts includes the following files: - `gl-dast-debug-auth-report.html` - `gl-dast-debug-crawl-report.html` - `gl-dast-crawl-graph.svg` + +## Troubleshooting + +See [troubleshooting](browser_based_troubleshooting.md) for more information. diff --git a/doc/user/application_security/dast/browser_based_troubleshooting.md b/doc/user/application_security/dast/browser_based_troubleshooting.md new file mode 100644 index 00000000000..78f2723ee38 --- /dev/null +++ b/doc/user/application_security/dast/browser_based_troubleshooting.md @@ -0,0 +1,300 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: reference, howto +--- + +# Troubleshooting DAST browser-based analyzer **(ULTIMATE)** + +The following troubleshooting scenarios have been collected from customer support cases. If you +experience a problem not addressed here, or the information here does not fix your problem, create a +support ticket. For more details, see the [GitLab Support](https://about.gitlab.com/support/) page. + +## When something goes wrong + +When something goes wrong with a DAST scan, if you have a particular error message then check [known problems](#known-problems). + +Otherwise, try to discover the problem by answering the following questions: + +- [What is the expected outcome?](#what-is-the-expected-outcome) +- [Is the outcome achievable by a human?](#is-the-outcome-achievable-by-a-human) +- [Any reason why DAST would not work?](#any-reason-why-dast-would-not-work) +- [How does your application work?](#how-does-your-application-work) +- [What is DAST doing?](#what-is-dast-doing) + +### What is the expected outcome? + +Many users who encounter issues with a DAST scan have a good high-level idea of what they think the scanner should be doing. For example, +it's not scanning particular pages, or it's not selecting a button on the page. + +As much as possible, try to isolate the problem to help narrow the search for a solution. For example, take the situation where DAST isn't scanning a particular page. +From where should DAST have found the page? What path did it take to navigate there? Were there elements on the referring page that DAST should have selected, but did not? + +### Is the outcome achievable by a human? + +DAST cannot scan an application if a human cannot manually traverse the application. + +Knowing the outcome you expect, try to replicate it manually using a browser on your machine. For example: + +- Open a new incognito/private browser window. +- Open Developer Tools. Keep an eye on the console for error messages. + - In Chrome: `View -> Developer -> Developer Tools`. + - In Firefox: `Tools -> Browser Tools -> Web Developer Tools`. +- If authenticating: + - Navigate to the `DAST_AUTH_URL`. + - Type in the `DAST_USERNAME` in the `DAST_USERNAME_FIELD`. + - Type in the `DAST_PASSWORD` in the `DAST_PASSWORD_FIELD`. + - Select the `DAST_SUBMIT_FIELD`. +- Select links and fill in forms. Navigate to the pages that aren't scanning correctly. +- Observe how your application behaves. Notice if there is anything that might cause problems for an automated scanner. + +### Any reason why DAST would not work? + +DAST cannot scan correctly when: + +- There is a CAPTCHA. Please turn these off in the testing environment for the application being scanned. +- It does not have access to the target application. Ensure the GitLab Runner can access the application using the URLs used in the DAST configuration. + +### How does your application work? + +Understanding how your application works is vital to figuring out why a DAST scan isn't working. For example, the following situations +may require additional configuration settings. + +- Is there a popup modal that hides elements? +- Does a loaded page change dramatically after a certain period of time? +- Is the application especially slow or fast to load? +- Is the target application jerky while loading? +- Does the application work differently based on the client's location? +- Is the application a single-page application? +- Does the application submit HTML forms, or does it use JavaScript and AJAX? +- Does the application use websockets? +- Does the application use a specific web framework? +- Does selecting buttons run JavaScript before continuing the form submit? Is it fast, slow? +- Is it possible DAST could be selecting or searching for elements before either the element or page is ready? + +### What is DAST doing? + +Logging remains the best way to understand what DAST is doing: + +- [Browser-based analyzer logging](#browser-based-analyzer-logging), useful for understanding what the analyzer is doing. +- [Chromium DevTools logging](#chromium-devtools-logging), useful to inspect the communication between DAST and Chromium. +- [Chromium Logs](#chromium-logs), useful for logging errors when Chromium crashes unexpectedly. + +## Browser-based analyzer logging + +The analyzer log is one of the most useful tools to help diagnose problems with a scan. Different parts of the analyzer can be logged at different levels. + +### Log message format + +Log messages have the format `[time] [log level] [log module] [message] [additional properties]`. + +For example, the following log entry has level `INFO`, is part of the `CRAWL` log module, has the message `Crawled path` and the additional properties `nav_id` and `path`. + +```txt +2021-04-21T00:34:04.000 INF CRAWL Crawled path nav_id=0cc7fd path="LoadURL [https://my.site.com:8090]" +``` + +### Log destination + +Logs are sent either to file or to console (the CI/CD job log). You can configure each destination to accept different logs using +the environment variables `DAST_BROWSER_LOG` for console logs and `DAST_BROWSER_FILE_LOG` for file logs. + +In the following example, the file log defaults to `DEBUG` level, the console log defaults to `INFO` level and logs the `AUTH` module at `DEBUG` level. + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_BROWSER_LOG: "auth:debug" + DAST_BROWSER_FILE_LOG: "loglevel:debug" + DAST_BROWSER_FILE_LOG_PATH: "/zap/wrk/dast-scan.log" + artifacts: + paths: + - dast-scan.log + when: always +``` + +### Log levels + +The log levels that can be configured are as follows: + +| Log module | Component overview | More | +|-------------------------|--------------------------------------------------------------------------|----------------------------------| +| `TRACE` | Used for specific, often noisy inner workings of a feature. | | +| `DEBUG` | Describes the inner-workings of a feature. Used for diagnostic purposes. | | +| `INFO` | Describes the high level flow of the scan and the results. | Default level if none specified. | +| `WARN` | Describes an error situation where DAST recovers and continues the scan. | | +| `FATAL`/`ERROR`/`PANIC` | Describes unrecoverable errors prior to exit. | | + +### Log modules + +`LOGLEVEL` configures the default log level for the log destination. If any of the following modules are configured, +DAST uses the log level for that module in preference to the default log level. + +The modules that can be configured for logging are as follows: + +| Log module | Component overview | +|------------|---------------------------------------------------------------------------------------------------| +| `ACTIV` | Used for active attacks. | +| `AUTH` | Used for creating an authenticated scan. | +| `BPOOL` | The set of browsers that are leased out for crawling. | +| `BROWS` | Used for querying the state or page of the browser. | +| `CACHE` | Used for reporting on cache hit and miss for cached HTTP resources. | +| `CHROM` | Used to log Chrome DevTools messages. | +| `CONTA` | Used for the container that collects parts of HTTP requests and responses from DevTools messages. | +| `CRAWL` | Used for the core crawler algorithm. | +| `DATAB` | Used for persisting data to the internal database. | +| `LEASE` | Used to create browsers to add them to the browser pool. | +| `MAIN` | Used for the flow of the main event loop of the crawler. | +| `NAVDB` | Used for persistence mechanisms to store navigation entries. | +| `REGEX` | Used for recording performance statistics when running regular expressions. | +| `REPT` | Used for generating reports. | +| `STAT` | Used for general statistics while running the scan. | +| `VLDFN` | Used for loading and parsing vulnerability definitions. | +| `WEBGW` | Used to log messages sent to the target application when running active checks. | + +### Example - log crawled paths + +Set the log module `CRAWL` to `DEBUG` to log navigation paths found during the crawl phase of the scan. This is useful for understanding +if DAST is crawling your target application correctly. + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_BROWSER_LOG: "crawl:debug" +``` + +For example, the following output shows that four anchor links we discovered during the crawl of the page at `https://example.com`. + +```plaintext +2022-11-17T11:18:05.578 DBG CRAWL executing step nav_id=6ec647d8255c729160dd31cb124e6f89 path="LoadURL [https://example.com]" step=1 +... +2022-11-17T11:18:11.900 DBG CRAWL found new navigations browser_id=2243909820020928961 nav_count=4 nav_id=6ec647d8255c729160dd31cb124e6f89 of=1 step=1 +2022-11-17T11:18:11.901 DBG CRAWL adding navigation action="LeftClick [a href=/page1.html]" nav=bd458cc1fc2d7c6fb984464b6d968866 parent_nav=6ec647d8255c729160dd31cb124e6f89 +2022-11-17T11:18:11.901 DBG CRAWL adding navigation action="LeftClick [a href=/page2.html]" nav=6dcb25f9f9ece3ee0071ac2e3166d8e6 parent_nav=6ec647d8255c729160dd31cb124e6f89 +2022-11-17T11:18:11.901 DBG CRAWL adding navigation action="LeftClick [a href=/page3.html]" nav=89efbb0c6154d6c6d85a63b61a7cdc6f parent_nav=6ec647d8255c729160dd31cb124e6f89 +2022-11-17T11:18:11.901 DBG CRAWL adding navigation action="LeftClick [a href=/page4.html]" nav=f29b4f4e0bdee70f5255de7fc080f04d parent_nav=6ec647d8255c729160dd31cb124e6f89 +``` + +## Chromium DevTools logging + +WARNING: +Logging DevTools messages is a security risk. The output contains secrets such as usernames, passwords and authentication tokens. +The output is uploaded to the GitLab server and may be visible in job logs. + +The DAST Browser-based scanner orchestrates a Chromium browser using the [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/). +Logging DevTools messages helps provide transparency into what the browser is doing. For example, if selecting a button does not work, a DevTools message might show that the cause is a CORS error in a browser console log. +Logs that contain DevTools messages can be very large in size. For this reason, it should only be enabled on jobs with a short duration. + +To log all DevTools messages, turn the `CHROM` log module to `trace` and configure logging levels. The following are examples of DevTools logs: + +```plaintext +2022-12-05T06:27:24.280 TRC CHROM event received {"method":"Fetch.requestPaused","params":{"requestId":"interception-job-3.0","request":{"url":"http://auth-auto:8090/font-awesome.min.css","method":"GET","headers":{"Accept":"text/css,*/*;q=0.1","Referer":"http://auth-auto:8090/login.html","User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/105.0.5195.102 Safari/537.36"},"initialPriority":"VeryHigh","referrerPolicy":"strict-origin-when-cross-origin"},"frameId":"A706468B01C2FFAA2EB6ED365FF95889","resourceType":"Stylesheet","networkId":"39.3"}} method=Fetch.requestPaused +2022-12-05T06:27:24.280 TRC CHROM request sent {"id":47,"method":"Fetch.continueRequest","params":{"requestId":"interception-job-3.0","headers":[{"name":"Accept","value":"text/css,*/*;q=0.1"},{"name":"Referer","value":"http://auth-auto:8090/login.html"},{"name":"User-Agent","value":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/105.0.5195.102 Safari/537.36"}]}} id=47 method=Fetch.continueRequest +2022-12-05T06:27:24.281 TRC CHROM response received {"id":47,"result":{}} id=47 method=Fetch.continueRequest +``` + +### Customizing DevTools log levels + +Chrome DevTools requests, responses and events are namespaced by domain. DAST allows each domain and each domain with message to have different logging configuration. +The environment variable `DAST_BROWSER_DEVTOOLS_LOG` accepts a semi-colon separated list of logging configurations. +Logging configurations are declared using the structure `[domain/message]:[what-to-log][,truncate:[max-message-size]]`. + +- `domain/message` references what is being logged. + - `Default` can be used as a value to represent all domains and messages. + - Can be a domain, for example, `Browser`, `CSS`, `Page`, `Network`. + - Can be a domain with a message, for example, `Network.responseReceived`. + - If multiple configurations apply, the most specific configuration is used. +- `what-to-log` references whether and what to log. + - `message` logs that a message was received and does not log the message content. + - `messageAndBody` logs the message with the message content. Recommended to be used with `truncate`. + - `suppress` does not log the message. Used to silence noisy domains and messages. +- `truncate` is an optional configuration to limit the size of the message printed. + +### Example - log all DevTools messages + +Used to log everything when you're not sure where to start. + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_BROWSER_FILE_LOG: "chrom:trace" + DAST_BROWSER_FILE_LOG_PATH: "/zap/wrk/dast-scan.log" + DAST_BROWSER_DEVTOOLS_LOG: "Default:messageAndBody,truncate:2000" + artifacts: + paths: + - dast-scan.log + when: always +``` + +### Example - log HTTP messages + +Useful for when a resource isn't loading correctly. HTTP message events are logged, as is the decision to continue or +fail the request. Any errors in the browser console are also logged. + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_BROWSER_FILE_LOG: "chrom:trace" + DAST_BROWSER_FILE_LOG_PATH: "/zap/wrk/dast-scan.log" + DAST_BROWSER_DEVTOOLS_LOG: "Default:suppress;Fetch:messageAndBody,truncate:2000;Network:messageAndBody,truncate:2000;Log:messageAndBody,truncate:2000;Console:messageAndBody,truncate:2000" + artifacts: + paths: + - dast-scan.log + when: always +``` + +## Chromium logs + +In the rare event that Chromium crashes, it can be helpful to write the Chromium process `STDOUT` and `STDERR` to log. +Setting the environment variable `DAST_BROWSER_LOG_CHROMIUM_OUTPUT` to `true` achieves this purpose. + +DAST starts and stops many Chromium processes. DAST sends each process output to all log destinations with the log module `LEASE` and log level `INFO`. + +For example: + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_BROWSER_LOG_CHROMIUM_OUTPUT: "true" +``` + +## Known problems + +### Logs contain `response body exceeds allowed size` + +By default DAST processes HTTP requests where the HTTP response body is 10 MB or less. Otherwise, DAST blocks the response +which can cause scans to fail. This constraint is intended to reduce memory consumption during a scan. + +An example log is as follows, where DAST blocked the JavaScript file found at `https://example.com/large.js` as it's size is greater than the limit: + +```plaintext +2022-12-05T06:28:43.093 WRN BROWS response body exceeds allowed size allowed_size_bytes=1000000 browser_id=752944257619431212 nav_id=ae23afe2acbce2c537657a9112926f1a of=1 request_id=interception-job-2.0 response_size_bytes=9333408 step=1 url=https://example.com/large.js +2022-12-05T06:28:58.104 WRN CONTA request failed, attempting to continue scan error=net::ERR_BLOCKED_BY_RESPONSE index=0 requestID=38.2 url=https://example.com/large.js +``` + +This can be changed using the configuration `DAST_MAX_RESPONSE_SIZE_MB`. For example, + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_MAX_RESPONSE_SIZE_MB: "25" +``` diff --git a/doc/user/application_security/dast/checks/16.2.md b/doc/user/application_security/dast/checks/16.2.md index a317b9418a1..2051b118009 100644 --- a/doc/user/application_security/dast/checks/16.2.md +++ b/doc/user/application_security/dast/checks/16.2.md @@ -40,5 +40,5 @@ the `Server` header. - [CWE](https://cwe.mitre.org/data/definitions/16.html) - [Apache ServerTokens](https://blog.mozilla.org/security/2016/08/26/mitigating-mime-confusion-attacks-in-firefox/) -- [NGINX server_tokens](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_tokens) +- [NGINX `server_tokens`](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_tokens) - [IIS 10 Remove Server Header](https://learn.microsoft.com/en-us/iis/configuration/system.webserver/security/requestfiltering/#attributes) diff --git a/doc/user/application_security/dast/checks/16.3.md b/doc/user/application_security/dast/checks/16.3.md index d9e6f6f8d92..d1799baa517 100644 --- a/doc/user/application_security/dast/checks/16.3.md +++ b/doc/user/application_security/dast/checks/16.3.md @@ -32,4 +32,4 @@ information from the `X-Powered-By` header. ## Links - [CWE](https://cwe.mitre.org/data/definitions/16.html) -- [PHP expose_php](https://www.php.net/manual/en/ini.core.php#ini.expose-php) +- [PHP `expose_php`](https://www.php.net/manual/en/ini.core.php#ini.expose-php) diff --git a/doc/user/application_security/dast/checks/548.1.md b/doc/user/application_security/dast/checks/548.1.md index b6907db5928..6cef8ccdb63 100644 --- a/doc/user/application_security/dast/checks/548.1.md +++ b/doc/user/application_security/dast/checks/548.1.md @@ -41,5 +41,5 @@ indexing. - [CWE](https://cwe.mitre.org/data/definitions/548.html) - [Apache Options](https://httpd.apache.org/docs/2.4/mod/core.html#options) -- [NGINX autoindex](https://nginx.org/en/docs/http/ngx_http_autoindex_module.html) -- [IIS directoryBrowse element](https://learn.microsoft.com/en-us/iis/configuration/system.webserver/directorybrowse) +- [NGINX `autoindex`](https://nginx.org/en/docs/http/ngx_http_autoindex_module.html) +- [IIS `directoryBrowse` element](https://learn.microsoft.com/en-us/iis/configuration/system.webserver/directorybrowse) diff --git a/doc/user/application_security/dast/checks/798.33.md b/doc/user/application_security/dast/checks/798.33.md index 536faefdb51..4761ac9d157 100644 --- a/doc/user/application_security/dast/checks/798.33.md +++ b/doc/user/application_security/dast/checks/798.33.md @@ -4,11 +4,11 @@ group: Dynamic Analysis info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Exposure of confidential secret or token Droneci Access Token +# Exposure of confidential secret or token Drone CI Access Token ## Description -The response body contains content that matches the pattern of a Droneci Access Token. +The response body contains content that matches the pattern of a Drone CI Access Token. Exposing this value could allow attackers to gain access to all resources granted by this token. ## Remediation diff --git a/doc/user/application_security/dast/checks/798.49.md b/doc/user/application_security/dast/checks/798.49.md index 7ea3a65fbfa..41a3e8ace3d 100644 --- a/doc/user/application_security/dast/checks/798.49.md +++ b/doc/user/application_security/dast/checks/798.49.md @@ -4,11 +4,11 @@ group: Dynamic Analysis info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Exposure of confidential secret or token Freshbooks Access Token +# Exposure of confidential secret or token FreshBooks Access Token ## Description -The response body contains content that matches the pattern of a Freshbooks Access Token. +The response body contains content that matches the pattern of a FreshBooks Access Token. Exposing this value could allow attackers to gain access to all resources granted by this token. ## Remediation diff --git a/doc/user/application_security/dast/checks/798.65.md b/doc/user/application_security/dast/checks/798.65.md index f2ebfb988b2..083bfec3350 100644 --- a/doc/user/application_security/dast/checks/798.65.md +++ b/doc/user/application_security/dast/checks/798.65.md @@ -4,11 +4,11 @@ group: Dynamic Analysis info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Exposure of confidential secret or token Launchdarkly Access Token +# Exposure of confidential secret or token LaunchDarkly Access Token ## Description -The response body contains content that matches the pattern of a Launchdarkly Access Token. +The response body contains content that matches the pattern of a LaunchDarkly Access Token. Exposing this value could allow attackers to gain access to all resources granted by this token. ## Remediation diff --git a/doc/user/application_security/dast/checks/798.97.md b/doc/user/application_security/dast/checks/798.97.md index d3035b05bbb..711288eba9c 100644 --- a/doc/user/application_security/dast/checks/798.97.md +++ b/doc/user/application_security/dast/checks/798.97.md @@ -4,11 +4,11 @@ group: Dynamic Analysis info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Exposure of confidential secret or token Rubygem API token +# Exposure of confidential secret or token RubyGems API token ## Description -The response body contains content that matches the pattern of a Rubygem API token. +The response body contains content that matches the pattern of a RubyGems API token. Exposing this value could allow attackers to gain access to all resources granted by this token. ## Remediation diff --git a/doc/user/application_security/dast/checks/829.1.md b/doc/user/application_security/dast/checks/829.1.md index f18634b72d9..7df250c2047 100644 --- a/doc/user/application_security/dast/checks/829.1.md +++ b/doc/user/application_security/dast/checks/829.1.md @@ -20,7 +20,7 @@ applications users would be protected from the malicious alterations. All identified resources should be sourced from the same domain as the target application. If this is not possible, it is strongly recommended that all `script` tags that implement `src` values, or `link` tags that implement the `href` values include Sub-Resource Integrity. To generate SRI integrity values the -[srihash](https://www.srihash.org/) tool can be used, or by running one of the following commands: +[SRI hash](https://www.srihash.org/) tool can be used, or by running one of the following commands: - `cat FILENAME.js | openssl dgst -sha384 -binary | openssl base64 -A` - `shasum -b -a 384 FILENAME.js | awk '{ print $1 }' | xxd -r -p | base64` diff --git a/doc/user/application_security/dast/checks/829.2.md b/doc/user/application_security/dast/checks/829.2.md index 19490afe676..d9d3e5a6341 100644 --- a/doc/user/application_security/dast/checks/829.2.md +++ b/doc/user/application_security/dast/checks/829.2.md @@ -19,7 +19,7 @@ them with known good versions. All identified resources should be sourced from the same domain as the target application. If this is not possible, it is strongly recommended that all `script` tags that implement `src` values, or `link` tags that implement the `href` values include Sub-Resource Integrity. To generate SRI integrity values the -[srihash](https://www.srihash.org/) tool can be used, or by running one of the following commands: +[SRI hash](https://www.srihash.org/) tool can be used, or by running one of the following commands: - `cat FILENAME.js | openssl dgst -sha384 -binary | openssl base64 -A` - `shasum -b -a 384 FILENAME.js | awk '{ print $1 }' | xxd -r -p | base64` diff --git a/doc/user/application_security/dast/checks/index.md b/doc/user/application_security/dast/checks/index.md index 9466734f9cf..56406b24586 100644 --- a/doc/user/application_security/dast/checks/index.md +++ b/doc/user/application_security/dast/checks/index.md @@ -69,7 +69,7 @@ The [DAST browser-based crawler](../browser_based.md) provides a number of vulne | [798.30](798.30.md) | Exposure of confidential secret or token Dropbox API secret | High | Passive | | [798.31](798.31.md) | Exposure of confidential secret or token Dropbox long lived API token | High | Passive | | [798.32](798.32.md) | Exposure of confidential secret or token Dropbox short lived API token | High | Passive | -| [798.33](798.33.md) | Exposure of confidential secret or token Droneci Access Token | High | Passive | +| [798.33](798.33.md) | Exposure of confidential secret or token Drone CI Access Token | High | Passive | | [798.34](798.34.md) | Exposure of confidential secret or token Duffel API token | High | Passive | | [798.35](798.35.md) | Exposure of confidential secret or token Dynatrace API token | High | Passive | | [798.36](798.36.md) | Exposure of confidential secret or token EasyPost API token | High | Passive | @@ -84,7 +84,7 @@ The [DAST browser-based crawler](../browser_based.md) provides a number of vulne | [798.46](798.46.md) | Exposure of confidential secret or token Flutterwave Secret Key | High | Passive | | [798.47](798.47.md) | Exposure of confidential secret or token Flutterwave Encryption Key | High | Passive | | [798.48](798.48.md) | Exposure of confidential secret or token Frame.io API token | High | Passive | -| [798.49](798.49.md) | Exposure of confidential secret or token Freshbooks Access Token | High | Passive | +| [798.49](798.49.md) | Exposure of confidential secret or token FreshBooks Access Token | High | Passive | | [798.50](798.50.md) | Exposure of confidential secret or token GoCardless API token | High | Passive | | [798.52](798.52.md) | Exposure of confidential secret or token GitHub Personal Access Token | High | Passive | | [798.53](798.53.md) | Exposure of confidential secret or token GitHub OAuth Access Token | High | Passive | @@ -99,7 +99,7 @@ The [DAST browser-based crawler](../browser_based.md) provides a number of vulne | [798.62](798.62.md) | Exposure of confidential secret or token Kraken Access Token | High | Passive | | [798.63](798.63.md) | Exposure of confidential secret or token Kucoin Access Token | High | Passive | | [798.64](798.64.md) | Exposure of confidential secret or token Kucoin Secret Key | High | Passive | -| [798.65](798.65.md) | Exposure of confidential secret or token Launchdarkly Access Token | High | Passive | +| [798.65](798.65.md) | Exposure of confidential secret or token LaunchDarkly Access Token | High | Passive | | [798.66](798.66.md) | Exposure of confidential secret or token Linear API Token | High | Passive | | [798.67](798.67.md) | Exposure of confidential secret or token Linear Client Secret | High | Passive | | [798.68](798.68.md) | Exposure of confidential secret or token LinkedIn Client ID | High | Passive | @@ -126,7 +126,7 @@ The [DAST browser-based crawler](../browser_based.md) provides a number of vulne | [798.94](798.94.md) | Exposure of confidential secret or token Private Key | High | Passive | | [798.95](798.95.md) | Exposure of confidential secret or token Pulumi API token | High | Passive | | [798.96](798.96.md) | Exposure of confidential secret or token PyPI upload token | High | Passive | -| [798.97](798.97.md) | Exposure of confidential secret or token Rubygem API token | High | Passive | +| [798.97](798.97.md) | Exposure of confidential secret or token RubyGem API token | High | Passive | | [798.98](798.98.md) | Exposure of confidential secret or token RapidAPI Access Token | High | Passive | | [798.99](798.99.md) | Exposure of confidential secret or token Sendbird Access ID | High | Passive | | [798.100](798.100.md) | Exposure of confidential secret or token Sendbird Access Token | High | Passive | diff --git a/doc/user/application_security/dast/dast_troubleshooting.md b/doc/user/application_security/dast/dast_troubleshooting.md index 61a7520bf7c..0dcf203a3a9 100644 --- a/doc/user/application_security/dast/dast_troubleshooting.md +++ b/doc/user/application_security/dast/dast_troubleshooting.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference, howto --- -# Troubleshooting Dynamic Application Security Testing (DAST) **(ULTIMATE)** +# Troubleshooting DAST proxy-based analyzer **(ULTIMATE)** The following troubleshooting scenarios have been collected from customer support cases. If you experience a problem not addressed here, or the information here does not fix your problem, create a diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index d78a8fca98f..283e48ec499 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -25,7 +25,7 @@ whitepaper. GitLab provides the following DAST analyzers, one or more of which may be useful depending on the kind of application you're testing. -For scanning websites, use one of: +For scanning websites, use one of: - The [DAST proxy-based analyzer](proxy-based.md) for scanning traditional applications serving simple HTML. The proxy-based analyzer can be run automatically or on-demand. - The [DAST browser-based analyzer](browser_based.md) for scanning applications that make heavy use of JavaScript. This includes single page web applications. diff --git a/doc/user/application_security/dast/proxy-based.md b/doc/user/application_security/dast/proxy-based.md index ec98b809fb7..fc78018bdad 100644 --- a/doc/user/application_security/dast/proxy-based.md +++ b/doc/user/application_security/dast/proxy-based.md @@ -24,6 +24,40 @@ The analyzer uses the [OWASP Zed Attack Proxy](https://www.zaproxy.org/) (ZAP) t to attack your application and produce a more extensive security report. It can be very useful when combined with [Review Apps](../../../ci/review_apps/index.md). +## Templates + +> - The DAST latest template was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/254325) in GitLab 13.8. +> - All DAST templates were [updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62597) to DAST_VERSION: 2 in GitLab 14.0. +> - All DAST templates were [updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87183) to DAST_VERSION: 3 in GitLab 15.0. + +GitLab DAST configuration is defined in CI/CD templates. Updates to the template are provided with +GitLab upgrades, allowing you to benefit from any improvements and additions. + +Available templates: + +- [`DAST.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml): Stable version of the DAST CI/CD template. +- [`DAST.latest.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml): Latest version of the DAST template. + +WARNING: +The latest version of the template may include breaking changes. Use the stable template unless you +need a feature provided only in the latest template. + +For more information about template versioning, see the +[CI/CD documentation](../../../development/cicd/templates.md#latest-version). + +## DAST versions + +By default, the DAST template uses the latest major version of the DAST Docker image. You can choose +how DAST updates, using the `DAST_VERSION` variable: + +- Automatically update DAST with new features and fixes by pinning to a major + version (such as `1`). +- Only update fixes by pinning to a minor version (such as `1.6`). +- Prevent all updates by pinning to a specific version (such as `1.6.4`). + +Find the latest DAST versions on the [DAST releases](https://gitlab.com/gitlab-org/security-products/dast/-/releases) +page. + ## DAST run options You can use DAST to examine your web application: @@ -46,58 +80,32 @@ To enable DAST to run automatically, either: - Enable [Auto DAST](../../../topics/autodevops/stages.md#auto-dast) (provided by [Auto DevOps](../../../topics/autodevops/index.md)). -- [Include the DAST template](#include-the-dast-template) in your existing - `.gitlab-ci.yml` file. -- [Configure DAST using the UI](#configure-dast-using-the-ui). - -#### Include the DAST template +- [Edit the `.gitlab.ci.yml` file manually](#edit-the-gitlabciyml-file-manually). +- [Use an automatically configured merge request](#configure-dast-using-the-ui). -> - This template was [updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62597) to DAST_VERSION: 2 in GitLab 14.0. -> - This template was [updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87183) to DAST_VERSION: 3 in GitLab 15.0. +#### Edit the `.gitlab.ci.yml` file manually -If you want to manually add DAST to your application, the DAST job is defined -in a CI/CD template file. Updates to the template are provided with GitLab -upgrades, allowing you to benefit from any improvements and additions. +In this method you manually edit the existing `.gitlab-ci.yml` file. Use this method if your GitLab CI/CD configuration file is complex. To include the DAST template: -1. Select the CI/CD template you want to use: - - - [`DAST.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml): - Stable version of the DAST CI/CD template. - - [`DAST.latest.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml): - Latest version of the DAST template. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/254325) - in GitLab 13.8). - - WARNING: - The latest version of the template may include breaking changes. Use the - stable template unless you need a feature provided only in the latest template. - - For more information about template versioning, see the - [CI/CD documentation](../../../development/cicd/templates.md#latest-version). - -1. Add a `dast` stage to your GitLab CI stages configuration: - - ```yaml - stages: - - dast - ``` - -1. Add the template to GitLab, based on your version of GitLab: +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **CI/CD > Editor**. +1. Copy and paste the following to the bottom of the `.gitlab-ci.yml` file. - - In GitLab 11.9 and later, [include](../../../ci/yaml/index.md#includetemplate) - the template by adding the following to your `.gitlab-ci.yml` file: + To use the DAST stable template: - ```yaml - include: - - template: <template_file.yml> + ```yaml + include: + - template: DAST.gitlab-ci.yml + ``` - variables: - DAST_WEBSITE: https://example.com - ``` + To use the DAST latest template: - - In GitLab 11.8 and earlier, add the contents of the template to your - `.gitlab_ci.yml` file. + ```yaml + include: + - template: DAST.latest.gitlab-ci.yml + ``` 1. Define the URL to be scanned by DAST by using one of these methods: @@ -125,9 +133,13 @@ To include the DAST template: You can see an example of this in our [Auto DevOps CI YAML](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml) file. +1. Select the **Validate** tab, then select **Validate pipeline**. + The message **Simulation completed successfully** indicates the file is valid. +1. Select the **Edit** tab. +1. Optional. In **Commit message**, customize the commit message. +1. Select **Commit changes**. -The included template creates a `dast` job in your CI/CD pipeline and scans -your project's running application for possible vulnerabilities. +Pipelines now include a DAST job. The results are saved as a [DAST report artifact](../../../ci/yaml/artifacts_reports.md#artifactsreportsdast) @@ -137,21 +149,12 @@ always take the latest DAST artifact available. Behind the scenes, the is used to run the tests on the specified URL and scan it for possible vulnerabilities. -By default, the DAST template uses the latest major version of the DAST Docker -image. Using the `DAST_VERSION` variable, you can choose how DAST updates: - -- Automatically update DAST with new features and fixes by pinning to a major - version (such as `1`). -- Only update fixes by pinning to a minor version (such as `1.6`). -- Prevent all updates by pinning to a specific version (such as `1.6.4`). - -Find the latest DAST versions on the [Releases](https://gitlab.com/gitlab-org/security-products/dast/-/releases) -page. - #### Configure DAST using the UI -You can enable or configure DAST settings using the UI. The generated settings are formatted so they -can be conveniently pasted into the `.gitlab-ci.yml` file. +In this method you select options in the UI. Based on your selections, a code +snippet is created that you paste into the `.gitlab-ci.yml` file. + +To configure DAST using the UI: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Security & Compliance > Configuration**. @@ -168,102 +171,17 @@ can be conveniently pasted into the `.gitlab-ci.yml` file. 1. To add the snippet to your project's `.gitlab-ci.yml` file, select **Copy code and open `.gitlab-ci.yml` file**. The Pipeline Editor opens. 1. Paste the snippet into the `.gitlab-ci.yml` file. - 1. Select the **Lint** tab to confirm the edited `.gitlab-ci.yml` file is valid. - 1. Select the **Edit** tab, then select **Commit changes**. + 1. Select the **Validate** tab, then select **Validate pipeline**. + The message **Simulation completed successfully** indicates the file is valid. + 1. Select the **Edit** tab. + 1. Optional. In **Commit message**, customize the commit message. + 1. Select **Commit changes**. -When the snippet is committed to the `.gitlab-ci.yml` file, pipelines include a DAST job. +Pipelines now include a DAST job. ### API scan -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10928) in GitLab 12.10. -> - A new DAST API scanning engine was introduced in GitLab 13.10. - -Using an API specification as a scan's target is a useful way to seed URLs for scanning an API. -Vulnerability rules in an API scan are different than those in a normal website scan. - -A new DAST API scanning engine is available in GitLab 13.12 and later. For more details, see [DAST API scanning engine](../dast_api). The new scanning engine supports REST, SOAP, GraphQL, and generic APIs using forms, XML, and JSON. Testing can be performed using OpenAPI, Postman Collections, and HTTP Archive (HAR) documents. - -The target API instance's base URL is provided by using the `DAST_API_TARGET_URL` variable or an `environment_url.txt` file. - -#### Specification format - -API scans support OpenAPI V2 and OpenAPI V3 specifications. You can define these specifications using `JSON` or `YAML`. - -#### Import API specification from a URL - -If your API specification is accessible at a URL, you can pass that URL in directly as the target. -The specification does not have to be hosted on the same host as the API being tested. - -```yaml -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_SPECIFICATION: http://my.api/api-specification.yml -``` - -#### Import API specification from a file - -If your API specification file is in your repository, you can provide its filename as the target. - -```yaml -dast: - variables: - GIT_STRATEGY: fetch - DAST_API_SPECIFICATION: api-specification.yml -``` - -#### Full API scan - -API scans support full scanning, which can be enabled by using the `DAST_FULL_SCAN_ENABLED` -CI/CD variable. Domain validation is not supported for full API scans. - -#### Host override - -Specifications often define a host, which contains a domain name and a port. The -host referenced may be different than the host of the API's review instance. -This can cause incorrect URLs to be imported, or a scan on an incorrect host. -Use the `DAST_API_HOST_OVERRIDE` CI/CD variable to override these values. - -WARNING: -When using the API host override feature, you cannot use the `$DAST_WEBSITE` variable to override the hostname. -A host override is _only_ supported when importing the API specification from a URL. Attempts to override the -host throw an error when the API specification is imported from a file. This is due to a limitation in the -ZAP OpenAPI extension. - -For example, with a OpenAPI V3 specification containing: - -```yaml -servers: - - url: https://api.host.com -``` - -If the test version of the API is running at `https://api-test.host.com`, then -the following DAST configuration can be used: - -```yaml -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_SPECIFICATION: http://api-test.host.com/api-specification.yml - DAST_API_HOST_OVERRIDE: api-test.host.com -``` - -#### Authentication using headers - -Tokens in request headers are often used as a way to authenticate API requests. -You can achieve this by using the `DAST_REQUEST_HEADERS` CI/CD variable. -Headers are applied to every request DAST makes. - -```yaml -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_SPECIFICATION: http://api-test.api.com/api-specification.yml - DAST_REQUEST_HEADERS: "Authorization: Bearer my.token" -``` +- The [DAST API analyzer](../dast_api/index.md) is used for scanning web APIs. Web API technologies such as GraphQL, REST, and SOAP are supported. ### URL scan @@ -336,6 +254,10 @@ When using `DAST_PATHS` and `DAST_PATHS_FILE`, note the following: To perform a [full scan](#full-scan) on the listed paths, use the `DAST_FULL_SCAN_ENABLED` CI/CD variable. +## Authentication + +The proxy-based analyzer uses the browser-based analyzer to authenticate a user prior to a scan. See [Authentication](authentication.md) for configuration instructions. + ## Customize DAST settings You can customize the behavior of DAST using both CI/CD variables and command-line options. Use of CI/CD @@ -427,61 +349,49 @@ To enable Mutual TLS: #### Available CI/CD variables These CI/CD variables are specific to DAST. They can be used to customize the behavior of DAST to your requirements. +For authentication CI/CD variables, see [Authentication](authentication.md). WARNING: All customization of GitLab security scanning tools should be tested in a merge request before merging these changes to the default branch. Failure to do so can give unexpected results, including a large number of false positives. -| CI/CD variable | Type | Description | -|:-------------------------------------------------|:--------------|:------------------------------| -| `DAST_ADVERTISE_SCAN` | boolean | Set to `true` to add a `Via` header to every request sent, advertising that the request was sent as part of a GitLab DAST scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334947) in GitLab 14.1. | -| `DAST_AGGREGATE_VULNERABILITIES` | boolean | Vulnerability aggregation is set to `true` by default. To disable this feature and see each vulnerability individually set to `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/254043) in GitLab 14.0. | -| `DAST_API_HOST_OVERRIDE` <sup>1</sup> | string | Used to override domains defined in API specification files. Only supported when importing the API specification from a URL. Example: `example.com:8080`. | -| `DAST_API_SPECIFICATION` <sup>1</sup> | URL or string | The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | -| `DAST_AUTH_REPORT` <sup>2</sup> | boolean | Used in combination with exporting the `gl-dast-debug-auth-report.html` artifact to aid in debugging authentication issues. | -| `DAST_AUTH_EXCLUDE_URLS` <sup>2</sup> | URLs | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/289959)** in GitLab 14.0. Replaced by `DAST_EXCLUDE_URLS`. The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. Not supported for API scans. | -| `DAST_AUTH_URL` <sup>1,2</sup> | URL | The URL of the page containing the sign-in HTML form on the target website. `DAST_USERNAME` and `DAST_PASSWORD` are submitted with the login form to create an authenticated scan. Not supported for API scans. Example: `https://login.example.com`. | -| `DAST_AUTH_VERIFICATION_LOGIN_FORM` <sup>2</sup> | boolean | Verifies successful authentication by checking for the lack of a login form once the login form has been submitted. | -| `DAST_AUTH_VERIFICATION_SELECTOR` <sup>2</sup> | selector | Verifies successful authentication by checking for presence of a selector once the login form has been submitted. Example: `css:.user-photo`. | -| `DAST_AUTH_VERIFICATION_URL` <sup>1,2</sup> | URL | A URL only accessible to logged in users that DAST can use to confirm successful authentication. If provided, DAST exits if it cannot access the URL. Example: `"http://example.com/loggedin_page"`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207335) in GitLab 13.8. | -| `DAST_AUTO_UPDATE_ADDONS` | boolean | ZAP add-ons are pinned to specific versions in the DAST Docker image. Set to `true` to download the latest versions when the scan starts. Default: `false`. | -| `DAST_BROWSER_PATH_TO_LOGIN_FORM` <sup>1,2</sup> | selector | Comma-separated list of selectors that are selected prior to attempting to enter `DAST_USERNAME` and `DAST_PASSWORD` into the login form. Example: `"css:.navigation-menu,css:.login-menu-item"`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/326633) in GitLab 14.1. | -| `DAST_DEBUG` <sup>1</sup> | boolean | Enable debug message output. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_EXCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from running during the scan. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). For example, `HTTP Parameter Override` has a rule ID of `10026`. Cannot be used when `DAST_ONLY_INCLUDE_RULES` is set. **Note:** In earlier versions of GitLab the excluded rules were executed but vulnerabilities they generated were suppressed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118641) in GitLab 12.10. | -| `DAST_EXCLUDE_URLS` <sup>1,2</sup> | URLs | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. Not supported for API scans. Example, `http://example.com/sign-out`. | -| `DAST_FIRST_SUBMIT_FIELD` <sup>2</sup> | string | The `id` or `name` of the element that when selected submits the username form of a multi-page login process. For example, `css:button[type='user-submit']`. [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/9894) in GitLab 12.4. | -| `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` | boolean | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/293595)** in GitLab 14.0. Set to `true` to require domain validation when running DAST full scans. Not supported for API scans. Default: `false` | -| `DAST_FULL_SCAN_ENABLED` <sup>1</sup> | boolean | Set to `true` to run a [ZAP Full Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan) instead of a [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan). Default: `false` | -| `DAST_HTML_REPORT` | string | The filename of the HTML report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | boolean | Set to `true` to include alpha passive and active scan rules. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_MARKDOWN_REPORT` | string | The filename of the Markdown report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_MASK_HTTP_HEADERS` | string | Comma-separated list of request and response headers to be masked (GitLab 13.1). Must contain **all** headers to be masked. Refer to [list of headers that are masked by default](#hide-sensitive-information). | -| `DAST_MAX_URLS_PER_VULNERABILITY` | number | The maximum number of URLs reported for a single vulnerability. `DAST_MAX_URLS_PER_VULNERABILITY` is set to `50` by default. To list all the URLs set to `0`. [Introduced](https://gitlab.com/gitlab-org/security-products/dast/-/merge_requests/433) in GitLab 13.12. | -| `DAST_ONLY_INCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to configure the scan to run only them. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). Cannot be used when `DAST_EXCLUDE_RULES` is set. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/250651) in GitLab 13.12. | -| `DAST_PASSWORD` <sup>1,2</sup> | string | The password to authenticate to in the website. Example: `P@55w0rd!` | -| `DAST_PASSWORD_FIELD` <sup>1,2</sup> | string | The selector of password field at the sign-in HTML form. Example: `id:password` | -| `DAST_PATHS` | string | Set to a comma-separated list of URLs for DAST to scan. For example, `/page1.html,/category1/page3.html,/page2.html`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214120) in GitLab 13.4. | -| `DAST_PATHS_FILE` | string | The file path containing the paths within `DAST_WEBSITE` to scan. The file must be plain text with one path per line. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/258825) in GitLab 13.6. | -| `DAST_PKCS12_CERTIFICATE_BASE64` | string | The PKCS12 certificate used for sites that require Mutual TLS. Must be encoded as base64 text. | -| `DAST_PKCS12_PASSWORD` | string | The password of the certificate used in `DAST_PKCS12_CERTIFICATE_BASE64`. | -| `DAST_REQUEST_HEADERS` <sup>1</sup> | string | Set to a comma-separated list of request header names and values. Headers are added to every request made by DAST. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` | -| `DAST_SKIP_TARGET_CHECK` | boolean | Set to `true` to prevent DAST from checking that the target is available before scanning. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229067) in GitLab 13.8. | -| `DAST_SPIDER_MINS` <sup>1</sup> | number | The maximum duration of the spider scan in minutes. Set to `0` for unlimited. Default: One minute, or unlimited when the scan is a full scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_SPIDER_START_AT_HOST` | boolean | Set to `false` to prevent DAST from resetting the target to its host before scanning. When `true`, non-host targets `http://test.site/some_path` is reset to `http://test.site` before scan. Default: `true`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/258805) in GitLab 13.6. | -| `DAST_SUBMIT_FIELD` <sup>2</sup> | string | The `id` or `name` of the element that when selected submits the login form or the password form of a multi-page login process. For example, `css:button[type='submit']`. [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/9894) in GitLab 12.4. | -| `DAST_TARGET_AVAILABILITY_TIMEOUT` <sup>1</sup> | number | Time limit in seconds to wait for target availability. | -| `DAST_USE_AJAX_SPIDER` <sup>1</sup> | boolean | Set to `true` to use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_USERNAME` <sup>1,2</sup> | string | The username to authenticate to in the website. Example: `admin` | -| `DAST_USERNAME_FIELD` <sup>1,2</sup> | string | The selector of username field at the sign-in HTML form. Example: `name:username` | -| `DAST_XML_REPORT` | string | The filename of the XML report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_WEBSITE` <sup>1</sup> | URL | The URL of the website to scan. The variable `DAST_API_SPECIFICATION` must be specified if this is omitted. | -| `DAST_ZAP_CLI_OPTIONS` | string | ZAP server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_ZAP_LOG_CONFIGURATION` | string | Set to a semicolon-separated list of additional log4j properties for the ZAP Server. Example: `logger.httpsender.name=org.parosproxy.paros.network.HttpSender;logger.httpsender.level=debug;logger.sitemap.name=org.parosproxy.paros.model.SiteMap;logger.sitemap.level=debug;` | -| `SECURE_ANALYZERS_PREFIX` | URL | Set the Docker registry base address from which to download the analyzer. | +| CI/CD variable | Type | Description | +|:------------------------------------------------|:--------------|:------------------------------| +| `DAST_ADVERTISE_SCAN` | boolean | Set to `true` to add a `Via` header to every request sent, advertising that the request was sent as part of a GitLab DAST scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334947) in GitLab 14.1. | +| `DAST_AGGREGATE_VULNERABILITIES` | boolean | Vulnerability aggregation is set to `true` by default. To disable this feature and see each vulnerability individually set to `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/254043) in GitLab 14.0. | +| `DAST_API_HOST_OVERRIDE` <sup>1</sup> | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. Replaced by [DAST API scan](../dast_api/index.md#available-cicd-variables). Used to override domains defined in API specification files. Only supported when importing the API specification from a URL. Example: `example.com:8080`. | +| `DAST_API_SPECIFICATION` <sup>1</sup> | URL or string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. Replaced by [DAST API scan](../dast_api/index.md#available-cicd-variables). The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | +| `DAST_AUTH_EXCLUDE_URLS` | URLs | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/289959)** in GitLab 14.0. Replaced by `DAST_EXCLUDE_URLS`. The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. | +| `DAST_AUTO_UPDATE_ADDONS` | boolean | ZAP add-ons are pinned to specific versions in the DAST Docker image. Set to `true` to download the latest versions when the scan starts. Default: `false`. | +| `DAST_DEBUG` <sup>1</sup> | boolean | Enable debug message output. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_EXCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from running during the scan. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). For example, `HTTP Parameter Override` has a rule ID of `10026`. Cannot be used when `DAST_ONLY_INCLUDE_RULES` is set. **Note:** In earlier versions of GitLab the excluded rules were executed but vulnerabilities they generated were suppressed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118641) in GitLab 12.10. | +| `DAST_EXCLUDE_URLS` <sup>1</sup> | URLs | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. Example, `http://example.com/sign-out`. | +| `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` | boolean | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/293595)** in GitLab 14.0. Set to `true` to require domain validation when running DAST full scans. Default: `false` | +| `DAST_FULL_SCAN_ENABLED` <sup>1</sup> | boolean | Set to `true` to run a [ZAP Full Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan) instead of a [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan). Default: `false` | +| `DAST_HTML_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the HTML report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | boolean | Set to `true` to include alpha passive and active scan rules. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_MARKDOWN_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the Markdown report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_MASK_HTTP_HEADERS` | string | Comma-separated list of request and response headers to be masked (GitLab 13.1). Must contain **all** headers to be masked. Refer to [list of headers that are masked by default](#hide-sensitive-information). | +| `DAST_MAX_URLS_PER_VULNERABILITY` | number | The maximum number of URLs reported for a single vulnerability. `DAST_MAX_URLS_PER_VULNERABILITY` is set to `50` by default. To list all the URLs set to `0`. [Introduced](https://gitlab.com/gitlab-org/security-products/dast/-/merge_requests/433) in GitLab 13.12. | +| `DAST_ONLY_INCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to configure the scan to run only them. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). Cannot be used when `DAST_EXCLUDE_RULES` is set. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/250651) in GitLab 13.12. | +| `DAST_PATHS` | string | Set to a comma-separated list of URLs for DAST to scan. For example, `/page1.html,/category1/page3.html,/page2.html`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214120) in GitLab 13.4. | +| `DAST_PATHS_FILE` | string | The file path containing the paths within `DAST_WEBSITE` to scan. The file must be plain text with one path per line. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/258825) in GitLab 13.6. | +| `DAST_PKCS12_CERTIFICATE_BASE64` | string | The PKCS12 certificate used for sites that require Mutual TLS. Must be encoded as base64 text. | +| `DAST_PKCS12_PASSWORD` | string | The password of the certificate used in `DAST_PKCS12_CERTIFICATE_BASE64`. | +| `DAST_REQUEST_HEADERS` <sup>1</sup> | string | Set to a comma-separated list of request header names and values. Headers are added to every request made by DAST. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` | +| `DAST_SKIP_TARGET_CHECK` | boolean | Set to `true` to prevent DAST from checking that the target is available before scanning. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229067) in GitLab 13.8. | +| `DAST_SPIDER_MINS` <sup>1</sup> | number | The maximum duration of the spider scan in minutes. Set to `0` for unlimited. Default: One minute, or unlimited when the scan is a full scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_SPIDER_START_AT_HOST` | boolean | Set to `false` to prevent DAST from resetting the target to its host before scanning. When `true`, non-host targets `http://test.site/some_path` is reset to `http://test.site` before scan. Default: `true`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/258805) in GitLab 13.6. | +| `DAST_TARGET_AVAILABILITY_TIMEOUT` <sup>1</sup> | number | Time limit in seconds to wait for target availability. | +| `DAST_USE_AJAX_SPIDER` <sup>1</sup> | boolean | Set to `true` to use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_XML_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the XML report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_WEBSITE` <sup>1</sup> | URL | The URL of the website to scan. | +| `DAST_ZAP_CLI_OPTIONS` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. ZAP server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_ZAP_LOG_CONFIGURATION` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. Set to a semicolon-separated list of additional log4j properties for the ZAP Server. Example: `logger.httpsender.name=org.parosproxy.paros.network.HttpSender;logger.httpsender.level=debug;logger.sitemap.name=org.parosproxy.paros.model.SiteMap;logger.sitemap.level=debug;` | +| `SECURE_ANALYZERS_PREFIX` | URL | Set the Docker registry base address from which to download the analyzer. | 1. Available to an on-demand DAST scan. -1. Used for authentication. ### Customize DAST using command-line options @@ -528,242 +438,6 @@ variables: DAST_ZAP_CLI_OPTIONS: "-config replacer.full_list(0).description=auth -config replacer.full_list(0).enabled=true -config replacer.full_list(0).matchtype=REQ_HEADER -config replacer.full_list(0).matchstr=Authorization -config replacer.full_list(0).regex=false -config replacer.full_list(0).replacement=TOKEN" ``` -## Authentication - -NOTE: -We highly recommend you configure the scanner to authenticate to the application. If you don't, it cannot check most of the application for security risks, as most -of your application is likely not accessible without authentication. We also recommend -you periodically confirm the scanner's authentication is still working, as this tends to break over -time due to authentication changes to the application. - -Create masked CI/CD variables to pass the credentials that DAST uses. -To create masked variables for the username and password, see [Create a custom variable in the UI](../../../ci/variables/index.md#custom-cicd-variables). -The key of the username variable must be `DAST_USERNAME`, -and the key of the password variable must be `DAST_PASSWORD`. - -After DAST has authenticated with the application, all cookies are collected from the web browser. -For each cookie a matching session token is created for use by ZAP. This ensures ZAP is recognized -by the application as correctly authenticated. - -Authentication supports single form logins, multi-step login forms, and authenticating to URLs outside of the configured target URL. - -WARNING: -**Never** run an authenticated scan against a production server. When an authenticated -scan is run, it may perform *any* function that the authenticated user can. This -includes actions like modifying and deleting data, submitting forms, and following links. -Only run an authenticated scan against a test server. - -### SSO - -DAST can authenticate to websites making use of SSO, with the following restrictions: - -- DAST cannot bypass a CAPTCHA if the authentication flow includes one. -- DAST cannot handle multi-factor authentication like one-time passwords (OTP) by using SMS or authenticator apps. -- DAST must get a cookie, or a local or session storage, with a sufficiently random value. - -The [authentication debug output](#configure-the-authentication-debug-output) can be helpful for troubleshooting SSO authentication -with DAST. - -### Log in using automatic detection of the login form - -By providing a `DAST_USERNAME`, `DAST_PASSWORD`, and `DAST_AUTH_URL`, DAST attempts to authenticate to the -target application by locating the login form based on a determination about whether or not the form contains username or password fields. - -Automatic detection is "best-effort", and depending on the application being scanned may provide either a resilient login experience or one that fails to authenticate the user. - -Login process: - -1. The `DAST_AUTH_URL` is loaded into the browser, and any forms on the page are located. - 1. If a form contains a username and password field, `DAST_USERNAME` and `DAST_PASSWORD` is inputted into the respective fields, the form submit button is selected and the user is logged in. - 1. If a form contains only a username field, it is assumed that the login form is multi-step. - 1. The `DAST_USERNAME` is inputted into the username field and the form submit button is selected. - 1. The subsequent pages loads where it is expected that a form exists and contains a password field. If found, `DAST_PASSWORD` is inputted, form submit button is selected and the user is logged in. - -### Log in using explicit selection of the login form - -By providing a `DAST_USERNAME_FIELD`, `DAST_PASSWORD_FIELD`, and `DAST_SUBMIT_FIELD`, in addition to the fields required for automatic login, -DAST attempts to authenticate to the target application by locating the login form based on the selectors provided. -Most applications benefit from this approach to authentication. - -Login process: - -1. The `DAST_AUTH_URL` is loaded into the browser, and any forms on the page are located. - 1. If the `DAST_FIRST_SUBMIT_FIELD` is not defined, then `DAST_USERNAME` is inputted into `DAST_USERNAME_FIELD`, `DAST_PASSWORD` is inputted into `DAST_PASSWORD_FIELD`, `DAST_SUBMIT_FIELD` is selected and the user is logged in. - 1. If the `DAST_FIRST_SUBMIT_FIELD` is defined, then it is assumed that the login form is multi-step. - 1. The `DAST_USERNAME` is inputted into the `DAST_USERNAME_FIELD` field and the `DAST_FIRST_SUBMIT_FIELD` is selected. - 1. The subsequent pages loads where the `DAST_PASSWORD` is inputted into the `DAST_PASSWORD_FIELD` field, the `DAST_SUBMIT_FIELD` is selected and the user is logged in. - -### Verifying successful login - -Once the login form has been submitted, DAST determines if the login was successful. Unsuccessful attempts at authentication cause the scan to halt. - -Following the submission of the login form, authentication is determined to be unsuccessful when: - -- A `400` or `500` series HTTP response status code is returned. -- A new cookie/browser storage value determined to be sufficiently random has not been set. - -In addition to these checks, the user can configure their own verification checks. -Each of the following checks can be used in conjunction with one another, if none are configured by default the presence of a login form is checked. - -#### Verifying based on the URL - -When `DAST_AUTH_VERIFICATION_URL` is configured, the URL displayed in the browser tab post login form submission is directly compared to the URL in the CI/CD variable. -If these are not exactly the same, authentication is deemed to be unsuccessful. - -For example: - -```yaml -include: - - template: DAST.gitlab-ci.yml - -dast: - variables: - DAST_WEBSITE: "https://example.com" - DAST_BROWSER_SCAN: "true" # use the browser-based GitLab DAST crawler - ... - DAST_AUTH_VERIFICATION_URL: "https://example.com/user/welcome" -``` - -#### Verify based on presence of an element - -When `DAST_AUTH_VERIFICATION_SELECTOR` is configured, the page displayed in the browser tab is searched for an element described by the selector in the CI/CD variable. -If no element is found, authentication is deemed to be unsuccessful. - -For example: - -```yaml -include: - - template: DAST.gitlab-ci.yml - -dast: - variables: - DAST_WEBSITE: "https://example.com" - DAST_BROWSER_SCAN: "true" # use the browser-based GitLab DAST crawler - ... - DAST_AUTH_VERIFICATION_SELECTOR: "css:.welcome-user" -``` - -#### Verify based on presence of a login form - -When `DAST_AUTH_VERIFICATION_LOGIN_FORM` is configured, the page displayed in the browser tab is searched for a form that is detected to be a login form. -If any such form is found, authentication is deemed to be unsuccessful. - -For example: - -```yaml -include: - - template: DAST.gitlab-ci.yml - -dast: - variables: - DAST_WEBSITE: "https://example.com" - DAST_BROWSER_SCAN: "true" # use the browser-based GitLab DAST crawler - ... - DAST_AUTH_VERIFICATION_LOGIN_FORM: "true" -``` - -### View the login form - -Many web applications show the user the login form in a pop-up (modal) window. -For these applications, navigating to the form requires both: - -- A starting URL. -- A list of elements to select to display the modal window. - -When `DAST_BROWSER_PATH_TO_LOGIN_FORM` is present, like in this example: - -```yaml -include: - - template: DAST.gitlab-ci.yml - -dast: - variables: - DAST_WEBSITE: "https://my.site.com" - DAST_BROWSER_SCAN: "true" # use the browser-based GitLab DAST crawler - ... - DAST_AUTH_URL: "https://my.site.com/admin" - DAST_BROWSER_PATH_TO_LOGIN_FORM: "css:.navigation-menu,css:.login-menu-item" -``` - -DAST performs these actions: - -1. Load the `DAST_AUTH_URL` page, such as `https://my.site.com/admin`. -1. After the page loads, DAST selects elements found by the selectors described - in `DAST_BROWSER_PATH_TO_LOGIN_FORM`. This example opens the navigation menu - and selects the login menu, to display the login modal window. -1. To continue the authentication process, DAST fills in the username and password - on the login form. - -### Configure the authentication debug output - -It is often difficult to understand the cause of an authentication failure when running DAST in a CI/CD pipeline. -To assist users in debugging authentication issues, a debug report can be generated and saved as a job artifact. -This HTML report contains all steps made during the login process, along with HTTP requests and responses, the Document Object Model (DOM) and screenshots. - - - -An example configuration where the authentication debug report is exported may look like the following: - -```yaml -dast: - variables: - DAST_WEBSITE: "https://example.com" - DAST_BROWSER_SCAN: "true" # use the browser-based GitLab DAST crawler - ... - DAST_AUTH_REPORT: "true" - artifacts: - paths: [gl-dast-debug-auth-report.html] - when: always -``` - -### Selectors - -Selectors are used by CI/CD variables to specify the location of an element displayed on a page in a browser. -Selectors have the format `type`:`search string`. The crawler searches for the selector using the search string based on the type. - -| Selector type | Example | Description | -| ------------- | ---------------------------------- | ----------- | -| `css` | `css:.password-field` | Searches for a HTML element having the supplied CSS selector. Selectors should be as specific as possible for performance reasons. | -| `id` | `id:element` | Searches for an HTML element with the provided element ID. | -| `name` | `name:element` | Searches for an HTML element with the provided element name. | -| `xpath` | `xpath://input[@id="my-button"]/a` | Searches for a HTML element with the provided XPath. Note that XPath searches are expected to be less performant than other searches. | -| None provided | `a.click-me` | Defaults to searching using a CSS selector. | - -#### Find selectors with Google Chrome - -Chrome DevTools element selector tool is an effective way to find a selector. - -1. Open Chrome and navigate to the page where you would like to find a selector, for example, the login page for your site. -1. Open the `Elements` tab in Chrome DevTools with the keyboard shortcut `Command + Shift + c` in macOS or `Ctrl + Shift + c` in Windows. -1. Select the `Select an element in the page to select it` tool. -  -1. Select the field on your page that you would like to know the selector for. -1. Once the tool is active, highlight a field you wish to view the details of. -  -1. Once highlighted, you can see the element's details, including attributes that would make a good candidate for a selector. - -In this example, the `id="user_login"` appears to be a good candidate. You can use this as a selector as the DAST username field by setting -`DAST_USERNAME_FIELD: "id:user_login"`. - -#### Choose the right selector - -Judicious choice of selector leads to a scan that is resilient to the application changing. - -In order of preference, it is recommended to choose as selectors: - -- `id` fields. These are generally unique on a page, and rarely change. -- `name` fields. These are generally unique on a page, and rarely change. -- `class` values specific to the field, such as the selector `"css:.username"` for the `username` class on the username field. -- Presence of field specific data attributes, such as the selector, `"css:[data-username]"` when the `data-username` field has any value on the username field. -- Multiple `class` hierarchy values, such as the selector `"css:.login-form .username"` when there are multiple elements with class `username` but only one nested inside the element with the class `login-form`. - -When using selectors to locate specific fields we recommend you avoid searching on: - -- Any `id`, `name`, `attribute`, `class` or `value` that is dynamically generated. -- Generic class names, such as `column-10` and `dark-grey`. -- XPath searches as they are less performant than other selector searches. -- Unscoped searches, such as those beginning with `css:*` and `xpath://*`. - ### Bleeding-edge vulnerability definitions ZAP first creates rules in the `alpha` class. After a testing period with @@ -804,8 +478,6 @@ An on-demand DAST scan: - Is associated with your project's default branch. - Is saved on creation so it can be run later. -### On-demand scan modes - An on-demand scan can be run in active or passive mode: - _Passive mode_ is the default and runs a ZAP Baseline Scan. @@ -814,35 +486,20 @@ An on-demand scan can be run in active or passive mode: ### View on-demand DAST scans -To view running completed and scheduled on-demand DAST scans for a project, go to -**Security & Compliance > On-demand Scans** in the left sidebar. +To view on-demand scans, from your project's home page, go to **Security & Compliance > On-demand +scans** in the left sidebar. -- To view both running and completed scans, select **All**. -- To view running scans only, select **Running**. -- To view finished scans, select **Finished**. A finished scan is a scan that either succeeded, - failed, or was canceled. -- To view scheduled scans, select **Scheduled**. It shows on-demand scans that have a schedule - set up. Those are _not_ included in the **All** tab. -- To view saved on-demand scan profiles, select **Scan library**. - Those are _not_ included in the **All** tab. +On-demand scans are grouped by their status. The scan library contains all available on-demand +scans. -#### Cancel an on-demand scan +From the **On-demand scans** page you can: -To cancel a pending or running on-demand scan, select **Cancel** (**{cancel}**) in the -on-demand scans list. - -#### Retry an on-demand scan - -To retry a scan that failed or succeeded with warnings, select **Retry** (**{retry}**) in the -on-demand scans list. - -#### View an on-demand scan's results - -To view a finished scan's results, select **View results** in the on-demand scans list. - -#### Edit an on-demand scan - -To edit an on-demand scan's settings, select **Edit** (**{pencil}**) in the **Scheduled** tab. +- [Run](#run-an-on-demand-dast-scan) an on-demand scan. +- View the results of an on-demand scan. +- Cancel (**{cancel}**) a pending or running on-demand scan. +- Retry (**{retry}**) a scan that failed, or succeeded with warnings. +- [Edit](#edit-an-on-demand-scan) (**{pencil}**) an on-demand scan's settings. +- [Delete](#delete-an-on-demand-scan) an on-demand scan. ### Run an on-demand DAST scan @@ -925,44 +582,37 @@ To schedule a scan: 1. To run the on-demand scan immediately, select **Save and run scan**. To [run](#run-a-saved-on-demand-scan) it according to the schedule you set, select **Save scan**. -#### List saved on-demand scans - -To list saved on-demand scans: - -1. From your project's home page, go to **Security & Compliance > On-demand Scans**. -1. Select the **Scan library** tab. - -#### View details of an on-demand scan +### View details of an on-demand scan To view details of an on-demand scan: -1. From your project's home page, go to **Security & Compliance > On-demand Scans**. +1. From your project's home page, go to **Security & Compliance > On-demand scans**. 1. Select the **Scan library** tab. 1. In the saved scan's row select **More actions** (**{ellipsis_v}**), then select **Edit**. -#### Edit an on-demand scan +### Edit an on-demand scan To edit an on-demand scan: -1. From your project's home page, go to **Security & Compliance > On-demand Scans**. +1. From your project's home page, go to **Security & Compliance > On-demand scans**. 1. Select the **Scan library** tab. 1. In the saved scan's row select **More actions** (**{ellipsis_v}**), then select **Edit**. 1. Edit the form. 1. Select **Save scan**. -#### Delete an on-demand scan +### Delete an on-demand scan To delete an on-demand scan: -1. From your project's home page, go to **Security & Compliance > On-demand Scans**. +1. From your project's home page, go to **Security & Compliance > On-demand scans**. 1. Select the **Scan library** tab. 1. In the saved scan's row select **More actions** (**{ellipsis_v}**), then select **Delete**. -1. Select **Delete** to confirm the deletion. +1. On the confirmation dialog, select **Delete**. ## Site profile -> - Scan method [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/345837) in GitLab 15.6. -> - File URL [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/345837) in GitLab 15.6. +> - Site profile features, scan method and file URL, were [enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/345837) in GitLab 15.6. +> - GraphQL endpoint path feature was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/378692) in GitLab 15.7. A site profile defines the attributes and configuration details of the deployed application, website, or API to be scanned by DAST. A site profile can be referenced in `.gitlab-ci.yml` and @@ -984,10 +634,11 @@ A site profile contains: - **Password form field**: The name of password field at the sign-in HTML form. - **Submit form field**: The `id` or `name` of the element that when selected submits the sign-in HTML form. -- **Scan method**: A type of method to perform API testing. The supported methods are OpenAPI, Postman Collections, and HTTP Archive (HAR) documents. -- **File URL**: The URL of the OpenAPI, Postman Collection, or HTTP Archive file. +- **Scan method**: A type of method to perform API testing. The supported methods are OpenAPI, Postman Collections, HTTP Archive (HAR), or GraphQL. + - **GraphQL endpoint path**: The path to the GraphQL endpoint. This path is concatenated with the target URL to provide the URI for the scan to test. The GraphQL endpoint must support introspection queries. + - **File URL**: The URL of the OpenAPI, Postman Collection, or HTTP Archive file. -When an API site type is selected, a [host override](#host-override) is used to ensure the API being scanned is on the same host as the target. This is done to reduce the risk of running an active scan against the wrong API. +When an API site type is selected, a host override is used to ensure the API being scanned is on the same host as the target. This is done to reduce the risk of running an active scan against the wrong API. When configured, request headers and password fields are encrypted using [`aes-256-gcm`](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) before being stored in the database. This data can only be read and decrypted with a valid secrets file. @@ -1235,9 +886,7 @@ and DAST site profiles are included in the [audit log](../../../administration/a The DAST tool outputs a `gl-dast-report.json` report file containing details of the scan and its results. This file is included in the job's artifacts. JSON is the default format, but you can output the report in Markdown, HTML, and XML formats. To specify an alternative -format, use a [CI/CD variable](#available-cicd-variables). You can also use a CI/CD variable -to configure the job to output the `gl-dast-debug-auth-report.html` file which helps when debugging -authentication issues. +format, use a [CI/CD variable](#available-cicd-variables). For details of the report's schema, see the [schema for DAST reports](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/dast-report-format.json). Example reports can be found in the [DAST repository](https://gitlab.com/gitlab-org/security-products/dast/-/tree/main/test/end-to-end/expect). diff --git a/doc/user/application_security/dast/run_dast_offline.md b/doc/user/application_security/dast/run_dast_offline.md index 05c6b74fbcd..7cb4eff8e68 100644 --- a/doc/user/application_security/dast/run_dast_offline.md +++ b/doc/user/application_security/dast/run_dast_offline.md @@ -39,7 +39,7 @@ process by which external resources can be imported or temporarily accessed. These scanners are [periodically updated](../index.md#vulnerability-scanner-maintenance) with new definitions, and you may be able to make occasional updates on your own. -For details on saving and transporting Docker images as a file, see Docker's documentation on +For details on saving and transporting Docker images as a file, see the Docker documentation on [`docker save`](https://docs.docker.com/engine/reference/commandline/save/), [`docker load`](https://docs.docker.com/engine/reference/commandline/load/), [`docker export`](https://docs.docker.com/engine/reference/commandline/export/), and diff --git a/doc/user/application_security/dast_api/index.md b/doc/user/application_security/dast_api/index.md index d77be0f0ca9..63276eba871 100644 --- a/doc/user/application_security/dast_api/index.md +++ b/doc/user/application_security/dast_api/index.md @@ -217,8 +217,9 @@ DAST API supports testing GraphQL endpoints multiple ways: - Test using a Postman Collection containing GraphQL queries. This section documents how to test using a GraphQL schema. The GraphQL schema support in -DAST API is able to query the schema from endpoints that support introspection. +DAST API is able to query the schema from endpoints that support [introspection](https://graphql.org/learn/introspection/). Introspection is enabled by default to allow tools like GraphiQL to work. +For details on how to enable introspection, see your GraphQL framework documentation. #### DAST API scanning with a GraphQL endpoint URL @@ -1046,6 +1047,8 @@ can be added, removed, and modified by creating a custom configuration. |[`DAST_API_EXCLUDE_URLS`](#exclude-urls) | Exclude API URL from testing. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357195) in GitLab 14.10. | |[`DAST_API_EXCLUDE_PARAMETER_ENV`](#exclude-parameters) | JSON string containing excluded parameters. | |[`DAST_API_EXCLUDE_PARAMETER_FILE`](#exclude-parameters) | Path to a JSON file containing excluded parameters. | +|[`DAST_API_REQUEST_HEADERS`](#request-headers) | A comma-separated (`,`) list of headers to include on each scan request. Consider using `DAST_API_REQUEST_HEADERS_BASE64` when storing secret header values in a [masked variable](../../../ci/variables/index.md#mask-a-cicd-variable), which has character set restrictions. | +|[`DAST_API_REQUEST_HEADERS_BASE64`](#request-headers) | A comma-separated (`,`) list of headers to include on each scan request, Base64-encoded. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/378440) in GitLab 15.6. | |[`DAST_API_OPENAPI`](#openapi-specification) | OpenAPI specification file or URL. | |[`DAST_API_OPENAPI_RELAXED_VALIDATION`](#openapi-specification) | Relax document validation. Default is disabled. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345950) in GitLab 14.7. | |[`DAST_API_OPENAPI_ALL_MEDIA_TYPES`](#openapi-specification) | Use all supported media types instead of one when generating requests. Causes test duration to be longer. Default is disabled. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/333304) in GitLab 14.10. | @@ -1488,6 +1491,61 @@ variables: In the previous sample, you could use the script `user-pre-scan-set-up.sh` to also install new runtimes or applications that later on you could use in our overrides command. +## Request Headers + +The request headers feature lets you specify fixed values for the headers during the scan session. For example, you can use the configuration variable `DAST_API_REQUEST_HEADERS` to set a fixed value in the `Cache-Control` header. If the headers you need to set include sensitive values like the `Authorization` header, use the [masked variable](../../../ci/variables/index.md#mask-a-cicd-variable) feature along with the [variable `DAST_API_REQUEST_HEADERS_BASE64`](#base64). + +Note that if the `Authorization` header or any other header needs to get updated while the scan is in progress, consider using the [overrides](#overrides) feature. + +The variable `DAST_API_REQUEST_HEADERS` lets you specify a comma-separated (`,`) list of headers. These headers are included on each request that the scanner performs. Each header entry in the list consists of a name followed by a colon (`:`) and then by its value. Whitespace before the key or value is ignored. For example, to declare a header name `Cache-Control` with the value `max-age=604800`, the header entry is `Cache-Control: max-age=604800`. To use two headers, `Cache-Control: max-age=604800` and `Age: 100`, set `DAST_API_REQUEST_HEADERS` variable to `Cache-Control: max-age=604800, Age: 100`. + +The order in which the different headers are provided into the variable `DAST_API_REQUEST_HEADERS` does not affect the result. Setting `DAST_API_REQUEST_HEADERS` to `Cache-Control: max-age=604800, Age: 100` produces the same result as setting it to `Age: 100, Cache-Control: max-age=604800`. + +### Base64 + +The `DAST_API_REQUEST_HEADERS_BASE64` variable accepts the same list of headers as `DAST_API_REQUEST_HEADERS`, with the only difference that the entire value of the variable must be Base64-encoded. For example, to set `DAST_API_REQUEST_HEADERS_BASE64` variable to `Authorization: QmVhcmVyIFRPS0VO, Cache-control: bm8tY2FjaGU=`, ensure you convert the list to its Base64 equivalent: `QXV0aG9yaXphdGlvbjogUW1WaGNtVnlJRlJQUzBWTywgQ2FjaGUtY29udHJvbDogYm04dFkyRmphR1U9`, and the Base64-encoded value must be used. This is useful when storing secret header values in a [masked variable](../../../ci/variables/index.md#mask-a-cicd-variable), which has character set restrictions. + +WARNING: +Base64 is used to support the [masked variable](../../../ci/variables/index.md#mask-a-cicd-variable) feature. Base64 encoding is not by itself a security measure, because sensitive values can be easily decoded. + +### Example: Adding a list of headers on each request using plain text + +In the following example of a `.gitlab-ci.yml`, `DAST_API_REQUEST_HEADERS` configuration variable is set to provide two header values as explained in [request headers](#request-headers). + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_REQUEST_HEADERS: 'Cache-control: no-cache, Save-Data: on' +``` + +### Example: Using a masked CI/CD variable + +The following `.gitlab-ci.yml` sample assumes the [masked variable](../../../ci/variables/index.md#mask-a-cicd-variable) `SECRET_REQUEST_HEADERS_BASE64` is defined as a [group or instance level CI/CD variable defined in the UI](../../../ci/variables/index.md#add-a-cicd-variable-to-an-instance). The value of `SECRET_REQUEST_HEADERS_BASE64` is set to `WC1BQ01FLVNlY3JldDogc31jcnt0ISwgWC1BQ01FLVRva2VuOiA3MDVkMTZmNWUzZmI=`, which is the Base64-encoded text version of `X-ACME-Secret: s3cr3t!, X-ACME-Token: 705d16f5e3fb`. Then, it can be used as follows: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_REQUEST_HEADERS_BASE64: $SECRET_REQUEST_HEADERS_BASE64 +``` + +Consider using `DAST_API_REQUEST_HEADERS_BASE64` when storing secret header values in a [masked variable](../../../ci/variables/index.md#mask-a-cicd-variable), which has character set restrictions. + ## Exclude Paths > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211892) in GitLab 14.0. @@ -1497,13 +1555,13 @@ When testing an API it can be useful to exclude certain paths. For example, you To verify the paths are excluded, review the `Tested Operations` and `Excluded Operations` portion of the job output. You should not see any excluded paths listed under `Tested Operations`. ```plaintext -2021-05-27 21:51:08 [INF] API Security: --[ Tested Operations ]------------------------- -2021-05-27 21:51:08 [INF] API Security: 201 POST http://target:7777/api/users CREATED -2021-05-27 21:51:08 [INF] API Security: ------------------------------------------------ -2021-05-27 21:51:08 [INF] API Security: --[ Excluded Operations ]----------------------- -2021-05-27 21:51:08 [INF] API Security: GET http://target:7777/api/messages -2021-05-27 21:51:08 [INF] API Security: POST http://target:7777/api/messages -2021-05-27 21:51:08 [INF] API Security: ------------------------------------------------ +2021-05-27 21:51:08 [INF] DAST API: --[ Tested Operations ]------------------------- +2021-05-27 21:51:08 [INF] DAST API: 201 POST http://target:7777/api/users CREATED +2021-05-27 21:51:08 [INF] DAST API: ------------------------------------------------ +2021-05-27 21:51:08 [INF] DAST API: --[ Excluded Operations ]----------------------- +2021-05-27 21:51:08 [INF] DAST API: GET http://target:7777/api/messages +2021-05-27 21:51:08 [INF] DAST API: POST http://target:7777/api/messages +2021-05-27 21:51:08 [INF] DAST API: ------------------------------------------------ ``` ### Examples @@ -1548,7 +1606,7 @@ variables: While testing an API you may might want to exclude a parameter (query string, header, or body element) from testing. This may be needed because a parameter always causes a failure, slows down testing, or for other reasons. To exclude parameters, you can set one of the following variables: `DAST_API_EXCLUDE_PARAMETER_ENV` or `DAST_API_EXCLUDE_PARAMETER_FILE`. -The `DAST_API_EXCLUDE_PARAMETER_ENV` allows providing a JSON string containing excluded parameters. This is a good option if the JSON is short and will not often change. Another option is the variable `DAST_API_EXCLUDE_PARAMETER_FILE`. This variable is set to a file path that can be checked into the repository, created by another job as an artifact, or generated at runtime with a pre script using `DAST_API_PRE_SCRIPT`. +The `DAST_API_EXCLUDE_PARAMETER_ENV` allows providing a JSON string containing excluded parameters. This is a good option if the JSON is short and will not often change. Another option is the variable `DAST_API_EXCLUDE_PARAMETER_FILE`. This variable is set to a file path that can be checked into the repository, created by another job as an artifact, or generated at runtime with a pre-script using `DAST_API_PRE_SCRIPT`. #### Exclude parameters using a JSON document @@ -1780,13 +1838,13 @@ As an alternative to excluding by paths, you can filter by any other component i In your job output you can check if any URLs matched any provided regular expression from `DAST_API_EXCLUDE_URLS`. Matching operations are listed in the **Excluded Operations** section. Operations listed in the **Excluded Operations** should not be listed in the **Tested Operations** section. For example the following portion of a job output: ```plaintext -2021-05-27 21:51:08 [INF] API Security: --[ Tested Operations ]------------------------- -2021-05-27 21:51:08 [INF] API Security: 201 POST http://target:7777/api/users CREATED -2021-05-27 21:51:08 [INF] API Security: ------------------------------------------------ -2021-05-27 21:51:08 [INF] API Security: --[ Excluded Operations ]----------------------- -2021-05-27 21:51:08 [INF] API Security: GET http://target:7777/api/messages -2021-05-27 21:51:08 [INF] API Security: POST http://target:7777/api/messages -2021-05-27 21:51:08 [INF] API Security: ------------------------------------------------ +2021-05-27 21:51:08 [INF] DAST API: --[ Tested Operations ]------------------------- +2021-05-27 21:51:08 [INF] DAST API: 201 POST http://target:7777/api/users CREATED +2021-05-27 21:51:08 [INF] DAST API: ------------------------------------------------ +2021-05-27 21:51:08 [INF] DAST API: --[ Excluded Operations ]----------------------- +2021-05-27 21:51:08 [INF] DAST API: GET http://target:7777/api/messages +2021-05-27 21:51:08 [INF] DAST API: POST http://target:7777/api/messages +2021-05-27 21:51:08 [INF] DAST API: ------------------------------------------------ ``` NOTE: @@ -2083,18 +2141,18 @@ The first step to resolving performance issues is to understand what is contribu The DAST API job output contains helpful information about how fast we are testing, how fast each operation being tested responds, and summary information. Let's take a look at some sample output to see how it can be used in tracking down performance issues: ```shell -API Security: Loaded 10 operations from: assets/har-large-response/large_responses.har -API Security: -API Security: Testing operation [1/10]: 'GET http://target:7777/api/large_response_json'. -API Security: - Parameters: (Headers: 4, Query: 0, Body: 0) -API Security: - Request body size: 0 Bytes (0 bytes) -API Security: -API Security: Finished testing operation 'GET http://target:7777/api/large_response_json'. -API Security: - Excluded Parameters: (Headers: 0, Query: 0, Body: 0) -API Security: - Performed 767 requests -API Security: - Average response body size: 130 MB -API Security: - Average call time: 2 seconds and 82.69 milliseconds (2.082693 seconds) -API Security: - Time to complete: 14 minutes, 8 seconds and 788.36 milliseconds (848.788358 seconds) +DAST API: Loaded 10 operations from: assets/har-large-response/large_responses.har +DAST API: +DAST API: Testing operation [1/10]: 'GET http://target:7777/api/large_response_json'. +DAST API: - Parameters: (Headers: 4, Query: 0, Body: 0) +DAST API: - Request body size: 0 Bytes (0 bytes) +DAST API: +DAST API: Finished testing operation 'GET http://target:7777/api/large_response_json'. +DAST API: - Excluded Parameters: (Headers: 0, Query: 0, Body: 0) +DAST API: - Performed 767 requests +DAST API: - Average response body size: 130 MB +DAST API: - Average call time: 2 seconds and 82.69 milliseconds (2.082693 seconds) +DAST API: - Time to complete: 14 minutes, 8 seconds and 788.36 milliseconds (848.788358 seconds) ``` This job console output snippet starts by telling us how many operations were found (10), followed by notifications that testing has started on a specific operation and a summary of the operation has been completed. The summary is the most interesting part of this log output. In the summary, we can see that it took DAST API 767 requests to fully test this operation and its related fields. We can also see that the average response time was 2 seconds and the time to complete was 14 minutes for this one operation. @@ -2281,7 +2339,7 @@ See the following documentation sections for assistance: See [Performance Tuning and Testing Speed](#performance-tuning-and-testing-speed) -### Error waiting for API Security 'http://127.0.0.1:5000' to become available +### Error waiting for DAST API 'http://127.0.0.1:5000' to become available A bug exists in versions of the DAST API analyzer prior to v1.6.196 that can cause a background process to fail under certain conditions. The solution is to update to a newer version of the DAST API analyzer. @@ -2294,6 +2352,11 @@ If the issue is occurring with versions v1.6.196 or greater, contact Support and 1. The `gl-api-security-scanner.log` file available as a job artifact. In the right-hand panel of the job details page, select the **Browse** button. 1. The `dast_api` job definition from your `.gitlab-ci.yml` file. +**Error message** + +- In [GitLab 15.6 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/376078), `Error waiting for DAST API 'http://127.0.0.1:5000' to become available` +- In GitLab 15.5 and earlier, `Error waiting for API Security 'http://127.0.0.1:5000' to become available`. + ### `Failed to start scanner session (version header not found)` The DAST API engine outputs an error message when it cannot establish a connection with the scanner application component. The error message is shown in the job output window of the `dast_api` job. A common cause of this issue is changing the `DAST_API_API` variable from its default. @@ -2446,6 +2509,50 @@ DAST API uses the specified media types in the OpenAPI document to generate requ 1. Review supported media types in the [OpenAPI Specification](#openapi-specification) section. 1. Edit your OpenAPI document, allowing at least a given operation to accept any of the supported media types. Alternatively, a supported media type could be set in the OpenAPI document level and get applied to all operations. This step may require changes in your application to ensure the supported media type is accepted by the application. +### ``Error, error occurred trying to download `<URL>`: There was an error when retrieving content from Uri:' <URL>'. Error:The SSL connection could not be established, see inner exception.`` + +DAST API is compatible with a broad range of TLS configurations, including outdated protocols and ciphers. +Despite broad support, you might encounter connection errors. This error occurs because DAST API could not establish a secure connection with the server at the given URL. + +To resolve the issue: + +If the host in the error message supports non-TLS connections, change `https://` to `http://` in your configuration. +For example, if an error occurs with the following configuration: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_TARGET_URL: https://test-deployment/ + DAST_API_OPENAPI: https://specs/openapi.json +``` + +Change the prefix of `DAST_API_OPENAPI` from `https://` to `http://`: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_TARGET_URL: https://test-deployment/ + DAST_API_OPENAPI: http://specs/openapi.json +``` + +If you cannot use a non-TLS connection to access the URL, contact the Support team for help. + +You can expedite the investigation with the [testssl.sh tool](https://testssl.sh/). From a machine with a bash shell and connectivity to the affected server: + +1. Download the latest release `zip` or `tar.gz` file and extract from <https://github.com/drwetter/testssl.sh/releases>. +1. Run `./testssl.sh --log https://specs`. +1. Attach the log file to your support ticket. + ## Get support or request an improvement To get support for your particular problem, use the [getting help channels](https://about.gitlab.com/get-help/). diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 4ed9ceedb4d..a4957c96db4 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -109,7 +109,7 @@ maximum of two directory levels from the repository's root. For example, the `gemnasium-dependency_scanning` job is enabled if a repository contains either `Gemfile`, `api/Gemfile`, or `api/client/Gemfile`, but not if the only supported dependency file is `api/v1/client/Gemfile`. -For Java and Python, when a supported depedency file is detected, Dependency Scanning attempts to build the project and execute some Java or Python commands to get the list of dependencies. For all other projects, the lock file is parsed to obtain the list of dependencies without needing to build the project first. +For Java and Python, when a supported dependency file is detected, Dependency Scanning attempts to build the project and execute some Java or Python commands to get the list of dependencies. For all other projects, the lock file is parsed to obtain the list of dependencies without needing to build the project first. When a supported dependency file is detected, all dependencies, including transitive dependencies are analyzed. There is no limit to the depth of nested or transitive dependencies that are analyzed. @@ -309,7 +309,7 @@ table.supported-languages ul { <li> <a id="notes-regarding-supported-languages-and-package-managers-3"></a> <p> - npm is only supported when <code>lockfileVersion = 1</code> or <code>lockfileVersion = 2</code>. Work to add support for <code>lockfileVersion = 3</code> is being tracked in issue <a href="https://gitlab.com/gitlab-org/gitlab/-/issues/365176">GitLab#365176</a>. + npm is supported for <code>lockfileVersion = 1</code>, <code>lockfileVersion = 2</code>, and <code>lockfileVersion = 3</code>. Support for <code>lockfileVersion = 3</code> was <a href="https://gitlab.com/gitlab-org/gitlab/-/issues/365176">introduced</a> in GitLab 15.7. </p> </li> <li> @@ -437,7 +437,7 @@ To support the following package managers, the GitLab analyzers proceed in two s <li> <a id="exported-dependency-information-notes-4"></a> <p> - Because of the implementation of <code>go build</code>, the Go build process requires network access, a pre-loaded modcache via <code>go mod download</code>, or vendored dependencies. For more information, + Because of the implementation of <code>go build</code>, the Go build process requires network access, a pre-loaded mod cache via <code>go mod download</code>, or vendored dependencies. For more information, refer to the Go documentation on <a href="https://pkg.go.dev/cmd/go#hdr-Compile_packages_and_dependencies">compiling packages and dependencies</a>. </p> </li> @@ -867,13 +867,8 @@ Here's an example dependency scanning report: ### CycloneDX Software Bill of Materials -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/350509) in GitLab 14.8 in [Beta](../../../policy/alpha-beta-support.md#beta-features). - -NOTE: -CycloneDX SBOMs are a [Beta](../../../policy/alpha-beta-support.md#beta-features) feature, -and the reports are subject to change during the beta period. Do not build integrations -that rely on the format of these SBOMs staying consistent, as the format might change -before the feature is made generally available. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/350509) in GitLab 14.8 in [Beta](../../../policy/alpha-beta-support.md#beta-features). +> - Generally available in GitLab 15.7. In addition to the [JSON report file](#reports-json-format), the [Gemnasium](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium) Dependency Scanning tool outputs a [CycloneDX](https://cyclonedx.org/) Software Bill of Materials (SBOM) for @@ -996,7 +991,7 @@ process by which external resources can be imported or temporarily accessed. These scanners are [periodically updated](../index.md#vulnerability-scanner-maintenance) with new definitions, and you may be able to make occasional updates on your own. -For details on saving and transporting Docker images as a file, see Docker's documentation on +For details on saving and transporting Docker images as a file, see the Docker documentation on [`docker save`](https://docs.docker.com/engine/reference/commandline/save/), [`docker load`](https://docs.docker.com/engine/reference/commandline/load/), [`docker export`](https://docs.docker.com/engine/reference/commandline/export/), and [`docker import`](https://docs.docker.com/engine/reference/commandline/import/). @@ -1257,7 +1252,7 @@ file in the `CI_BUILDS_DIR` directory triggers the dependency scanning job. We recommend committing the lock files, which prevents this warning. -### I no longer get the latest Docker image after setting `DS_MAJOR_VERSION` or `DS_ANALYZER_IMAGE` +### You no longer get the latest Docker image after setting `DS_MAJOR_VERSION` or `DS_ANALYZER_IMAGE` If you have manually set `DS_MAJOR_VERSION` or `DS_ANALYZER_IMAGE` for specific reasons, and now must update your configuration to again get the latest patched versions of our diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index 5ddfa99fc81..6629c798cfa 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -240,9 +240,9 @@ reports are available to download. To download a report, select ### Ultimate -A merge request contains a security widget which displays a summary of the new results. New results are determined by comparing the current findings against existing findings in the target (default) branch (if there are prior findings). +A merge request contains a security widget which displays a summary of the new results. New results are determined by comparing the findings of the merge request against the findings of the most recent completed pipeline (`success`, `failed`, `canceled` or `skipped`) for the latest commit in the target branch. -We recommend you run a scan of the `default` branch before enabling feature branch scans for your developers. Otherwise, there is no base for comparison and all feature branches display the full scan results in the merge request security widget. +If security scans have not run for the most recent completed pipeline in the target branch there is no base for comparison. The vulnerabilities from the merge request findings will be listed as new in the merge request security widget. We recommend you run a scan of the `default` (target) branch before enabling feature branch scans for your developers. The merge request security widget displays only a subset of the vulnerabilities in the generated JSON artifact because it contains both new and existing findings. @@ -339,7 +339,7 @@ custom job: The above `.gitlab-ci.yml` causes a linting error: ```plaintext -Found errors in your .gitlab-ci.yml: +Unable to create pipeline - dependency_scanning job: chosen stage does not exist; available stages are .pre - unit-tests - .post @@ -590,7 +590,7 @@ like [`SAST.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/l the following error may occur, depending on your GitLab CI/CD configuration: ```plaintext -Found errors in your .gitlab-ci.yml: +Unable to create pipeline jobs:sast config key may not be used with `rules`: only/except ``` @@ -683,7 +683,7 @@ This can be used for offline setups or for anyone wishing to use [Auto DevOps](. Instructions are available in the [legacy template project](https://gitlab.com/gitlab-org/auto-devops-v12-10). -#### Vulnerabilities are found, but the job succeeds. How can I have a pipeline fail instead? +#### Vulnerabilities are found, but the job succeeds. How can you have a pipeline fail instead? In these circumstances, that the job succeeds is the default behavior. The job's status indicates success or failure of the analyzer itself. Analyzer results are displayed in the diff --git a/doc/user/application_security/offline_deployments/index.md b/doc/user/application_security/offline_deployments/index.md index 2db8e9522db..05e56560f95 100644 --- a/doc/user/application_security/offline_deployments/index.md +++ b/doc/user/application_security/offline_deployments/index.md @@ -117,7 +117,7 @@ This template should be used in a new, empty project, with a `.gitlab-ci.yml` fi ```yaml include: - - template: Secure-Binaries.gitlab-ci.yml + - template: Security/Secure-Binaries.gitlab-ci.yml ``` The pipeline downloads the Docker images needed for the Security Scanners and saves them as diff --git a/doc/user/application_security/policies/index.md b/doc/user/application_security/policies/index.md index f6d22ab28cd..a214d0d2cec 100644 --- a/doc/user/application_security/policies/index.md +++ b/doc/user/application_security/policies/index.md @@ -34,6 +34,10 @@ project and the security policy project, this is not recommended. Keeping the se project separate from the development project allows for complete separation of duties between security/compliance teams and development teams. +You should not link a security policy project to a development project and to the group +or sub-group the development project belongs to at the same time. Linking this way will result in +approval rules from the Scan Result Policy not being applied to merge requests in the development project. + All security policies are stored in the `.gitlab/security-policies/policy.yml` YAML file inside the linked security policy project. The format for this YAML is specific to the type of policy that is stored there. Examples and schema information are available for the following policy types: @@ -140,10 +144,10 @@ for more information on the product direction of security policies within GitLab ## Troubleshooting -### `Branch name does not follow the pattern 'update-policy-<timestamp>'` +### `Branch name 'update-policy-<timestamp>' does not follow the pattern '<branch_name_regex>'` When you create a new security policy or change an existing policy, a new branch is automatically created with the branch name following the pattern `update-policy-<timestamp>`. For example: `update-policy-1659094451`. -If you have group or instance push rules that do not allow branch name patterns that contain the text `update-policy-<timestamp>`, you will get an error that states `Branch name does not follow the pattern 'update-policy-<timestamp>'`. +If you have group or instance [push rules that do not allow branch name patterns](../../project/repository/push_rules.md#validate-branch-names) that contain the text `update-policy-<timestamp>`, you will get an error that states `Branch name 'update-policy-<timestamp>' does not follow the pattern '<branch_name_regex>'`. The workaround is to amend your group or instance push rules to allow branches following the pattern `update-policy-` followed by an integer timestamp. diff --git a/doc/user/application_security/policies/scan-execution-policies.md b/doc/user/application_security/policies/scan-execution-policies.md index f950d5116b1..c9c48c0c926 100644 --- a/doc/user/application_security/policies/scan-execution-policies.md +++ b/doc/user/application_security/policies/scan-execution-policies.md @@ -8,6 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - Group-level security policies were [introduced](https://gitlab.com/groups/gitlab-org/-/epics/4425) in GitLab 15.2. > - Group-level security policies were [enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/356258) in GitLab 15.4. +> - Operational container scanning [introduced](https://gitlab.com/groups/gitlab-org/-/epics/3410) in GitLab 15.5 Group, subgroup, or project owners can use scan execution policies to require that security scans run on a specified schedule or with the project (or multiple projects if the policy is defined at a group or subgroup level) pipeline. Required scans are injected into the CI pipeline as new jobs @@ -86,16 +87,20 @@ This rule enforces the defined actions and schedules a scan on the provided date | Field | Type | Possible values | Description | |------------|------|-----------------|-------------| | `type` | `string` | `schedule` | The rule's type. | -| `branches` | `array` of `string` | `*` or the branch's name | The branch the given policy applies to (supports wildcard). | +| `branches` | `array` of `string` | `*` or the branch's name | The branch the given policy applies to (supports wildcard). This field is required if the `agents` field is not set. | | `cadence` | `string` | CRON expression (for example, `0 0 * * *`) | A whitespace-separated string containing five fields that represents the scheduled time. | -| `agents` | `object` | | The name of the [GitLab agents](../../clusters/agent/index.md) where [cluster image scanning](../../clusters/agent/vulnerabilities.md) will run. The object key is the name of the Kubernetes cluster configured for your project in GitLab. You can use the optional value of the object to select and scan specific Kubernetes resources. | +| `agents` | `object` | | The name of the [GitLab agents](../../clusters/agent/index.md) where [cluster image scanning](../../clusters/agent/vulnerabilities.md) will run. The object key is the name of the Kubernetes agent configured for your project in GitLab. This field is required if the `branches` field is not set. | GitLab supports the following types of CRON syntax for the `cadence` field: - A daily cadence of once per hour at a specified hour, for example: `0 18 * * *` - A weekly cadence of once per week on a specified day and at a specified hour, for example: `0 13 * * 0` -Other elements of the CRON syntax may work in the cadence field, however, GitLab does not officially test or support them. The CRON expression is evaluated in UTC by default. If you have a self-managed GitLab instance and have [changed the server timezone](../../../administration/timezone.md), the CRON expression is evaluated with the new timezone. +NOTE: +Other elements of the [CRON syntax]((https://docs.oracle.com/cd/E12058_01/doc/doc.1014/e12030/cron_expressions.htm)) may work in the cadence field if supported by the [cron](https://github.com/robfig/cron) we are using in our implementation, however, GitLab does not officially test or support them. + +NOTE: +If using the `agents` field, required for `Operational Container Scanning`, the CRON expression is evaluated in [UTC](https://www.timeanddate.com/worldclock/timezone/utc) using the system-time of the Kubernetes-agent pod. If not using the `agents` field, the CRON expression is evaluated in standard [UTC](https://www.timeanddate.com/worldclock/timezone/utc) time from GitLab.com. If you have a self-managed GitLab instance and have [changed the server timezone](../../../administration/timezone.md), the CRON expression is evaluated with the new timezone. ### `agent` schema @@ -108,20 +113,26 @@ Use this schema to define `agents` objects in the [`schedule` rule type](#schedu #### Policy example ```yaml -- name: Enforce Container Scanning in cluster connected through gitlab-agent for production and staging namespaces +- name: Enforce Container Scanning in cluster connected through my-gitlab-agent for default and kube-system namespaces enabled: true rules: - type: schedule cadence: '0 10 * * *' agents: - gitlab-agent: + <agent-name>: namespaces: - - 'production' - - 'staging' + - 'default' + - 'kube-system' actions: - scan: container_scanning ``` +The keys for a schedule rule are: + +- `cadence` (required): a [CRON expression](https://docs.oracle.com/cd/E12058_01/doc/doc.1014/e12030/cron_expressions.htm) for when the scans will be run +- `agents:<agent-name>` (required): The name of the agent to use for scanning +- `agents:<agent-name>:namespaces` (optional): The Kubernetes namespaces to scan. If omitted, all namespaces will be scanned. + ## `scan` action type This action executes the selected `scan` with additional parameters when conditions for at least one @@ -133,6 +144,7 @@ rule in the defined policy are met. | `site_profile` | `string` | Name of the selected [DAST site profile](../dast/proxy-based.md#site-profile). | The DAST site profile to execute the DAST scan. This field should only be set if `scan` type is `dast`. | | `scanner_profile` | `string` or `null` | Name of the selected [DAST scanner profile](../dast/proxy-based.md#scanner-profile). | The DAST scanner profile to execute the DAST scan. This field should only be set if `scan` type is `dast`.| | `variables` | `object` | | A set of CI variables, supplied as an array of `key: value` pairs, to apply and enforce for the selected scan. The `key` is the variable name, with its `value` provided as a string. This parameter supports any variable that the GitLab CI job supports for the specified scan. | +| `tags` | `array` of `string` | | A list of runner tags for the policy. The policy jobs will be run by runner with the specified tags. | Note the following: @@ -152,7 +164,6 @@ Note the following: mode when executed as part of a scheduled scan. - A container scanning scan that is configured for the `pipeline` rule type ignores the agent defined in the `agents` object. The `agents` object is only considered for `schedule` rule types. An agent with a name provided in the `agents` object must be created and configured for the project. -- The Dependency Scanning and SAST scans use the default templates and run in a [child pipeline](../../../ci/pipelines/downstream_pipelines.md#parent-child-pipelines). ## Example security policies project diff --git a/doc/user/application_security/policies/scan-result-policies.md b/doc/user/application_security/policies/scan-result-policies.md index 7482df18cc3..5df910efb15 100644 --- a/doc/user/application_security/policies/scan-result-policies.md +++ b/doc/user/application_security/policies/scan-result-policies.md @@ -184,3 +184,12 @@ It corresponds to a single object from the previous example: user_approvers: - adalberto.dare ``` + +## Example situations where scan result policies require additional approval + +There are several situations where the scan result policy will require an additional approval step. For example: + +- The number of security jobs is reduced in the working branch and no longer matches the number of security jobs in the target branch. Users can't skip the Scanning Result Policies by removing scanning jobs from the CI configuration. +- Someone stops a pipeline security job, and users can't skip the security scan. +- A job in a merge request fails and is configured with `allow_failure: false`. As a result, the pipeline is in a blocked state. +- A pipeline has a manual job that must run successfully for the entire pipeline to pass. diff --git a/doc/user/application_security/sast/customize_rulesets.md b/doc/user/application_security/sast/customize_rulesets.md index a0742eb79a7..3d8ad6c8bf6 100644 --- a/doc/user/application_security/sast/customize_rulesets.md +++ b/doc/user/application_security/sast/customize_rulesets.md @@ -11,251 +11,423 @@ info: To determine the technical writer assigned to the Stage/Group associated w > passthrough chains. Expanded to include additional passthrough types of `file`, `git`, and `url` in GitLab 14.6. > - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/235359) support for overriding rules in GitLab 14.8. -You can customize the default scanning rules provided by our SAST analyzers. -Ruleset customization supports the following that can be used -simultaneously: +You can customize the behavior of our SAST analyzers by [defining a ruleset configuration file](#create-the-configuration-file) in the +repository being scanned. There are two kinds of customization: -- [Disabling predefined rules](#disable-predefined-analyzer-rules). Available for all analyzers. -- [Overriding predefined rules](#override-predefined-analyzer-rules). Available for all analyzers. -- Modifying the default behavior of a given analyzer by [synthesizing and passing a custom configuration](#synthesize-a-custom-configuration). Available for only `nodejs-scan`, `gosec`, and `semgrep`. +- Modifying the behavior of **predefined rules**. This includes: + - [Disabling predefined rules](#disable-predefined-rules). Available for all analyzers. + - [Overriding predefined rules](#override-predefined-rules). Available for all analyzers. +- Replacing predefined rules by [synthesizing a custom configuration](#synthesize-a-custom-configuration) + using **passthroughs**. Available for only [nodejs-scan](https://gitlab.com/gitlab-org/security-products/analyzers/nodejs-scan) + and [semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep). -To customize the default scanning rules, create a file containing custom rules. These rules -are passed through to the analyzer's underlying scanner tools. +## Disable predefined rules -To create a custom ruleset: +You can disable predefined rules for any SAST analyzer. Disabled rules won't appear +on the [Pipeline Security](../index.md#view-security-scan-information-in-the-pipeline-security-tab) +tab or the [Vulnerability Report](../index.md#view-security-scan-information-in-the-vulnerability-report). -1. Create a `.gitlab` directory at the root of your project, if one doesn't already exist. -1. Create a custom ruleset file named `sast-ruleset.toml` in the `.gitlab` directory. +Disabling rules has a retroactive effect. The analyzer continues to scan for the +vulnerability, but findings are omitted from the [`gl-sast-report.json` artifact](index.md#reports-json-format). + +See the [Schema](#schema) and [Examples](#examples) sections for information on how +to configure this behavior. + +## Override predefined rules + +Certain attributes of predefined rules can be overridden for any SAST analyzer. This +can be useful when adapting SAST to your existing workflow or tools. For example, you +might want to override the severity of a vulnerability based on organizational policy, +or choose a different message to display in the Vulnerability Report. + +See the [Schema](#schema) and [Examples](#examples) sections for information on how +to configure this behavior. + +## Synthesize a custom configuration + +You can completely replace the predefined rules of some SAST analyzers: + +- [nodejs-scan](https://gitlab.com/gitlab-org/security-products/analyzers/nodejs-scan) - you + can replace the default [njsscan configuration file](https://github.com/ajinabraham/njsscan#configure-njsscan) + with your own. +- [semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) - you can replace + the [GitLab-maintained ruleset](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep/-/tree/main/rules) + with your own. -## Disable predefined analyzer rules +You provide your customizations via passthroughs, which are composed into a +passthrough chain at runtime and evaluated to produce a complete configuration. The +underlying scanner is then executed against this new configuration. -To disable analyzer rules: +There are multiple passthrough types that let you provide configuration in different +ways, such as using a file committed to your repository or inline in the ruleset +configuration file. You can also choose how subsequent passthroughs in the chain are +handled; they can overwrite or append to previous configuration. -1. Set the `disabled` flag to `true` in the context of a `ruleset` section +See the [Schema](#schema) and [Examples](#examples) sections for information on how +to configure this behavior. -1. In one or more `ruleset.identifier` sub sections, list the rules that you want disabled. Every `ruleset.identifier` section has: +## Create the configuration file -- a `type` field, to name the predefined rule identifier that the targeted analyzer uses. -- a `value` field, to name the rule to be disabled. +To create the ruleset configuration file: -### Example: Disable predefined rules of SAST analyzers +1. Create a `.gitlab` directory at the root of your project, if one doesn't already exist. +1. Create a file named `sast-ruleset.toml` in the `.gitlab` directory. + +## Schema + +### The top-level section + +The top-level section contains one or more _configuration sections_, defined as [TOML tables](https://toml.io/en/v1.0.0#table). + +| Setting | Description | +| --------| ----------- | +| `[$analyzer]` | Declares a configuration section for an analyzer. The name follows the snake-case names defined in the list of [SAST analyzers](analyzers.md#sast-analyzers). | -In the following example, the disabled rules are assigned to `eslint` -and `sobelow` by matching the `type` and `value` of identifiers: +Configuration example: ```toml -[eslint] - [[eslint.ruleset]] - disable = true - [eslint.ruleset.identifier] - type = "eslint_rule_id" - value = "security/detect-object-injection" +[semgrep] +... +``` - [[eslint.ruleset]] - disable = true - [eslint.ruleset.identifier] - type = "cwe" - value = "185" +Avoid creating configuration sections that modify existing rules _and_ synthesize a custom ruleset, as +the latter replaces predefined rules completely. -[sobelow] - [[sobelow.ruleset]] - disable = true - [sobelow.ruleset.identifier] - type = "sobelow_rule_id" - value = "sql_injection" +### The `[$analyzer]` configuration section + +The `[$analyzer]` section lets you customize the behavior of an analyzer. Valid properties +differ based on the kind of configuration you're making. + +| Setting | Applies to | Description | +| --------| -------------- | ----------- | +| `[[$analyzer.ruleset]]` | Predefined rules | Defines modifications to an existing rule. | +| `interpolate` | All | If set to `true`, you can use `$VAR` in the configuration to evaluate environment variables. Use this feature with caution, so you don't leak secrets or tokens. (Default: `false`) | +| `description` | Passthroughs | Description of the custom ruleset. | +| `targetdir` | Passthroughs | The directory where the final configuration should be persisted. If empty, a directory with a random name is created. The directory can contain up to 100MB of files. | +| `validate` | Passthroughs | If set to `true`, the content of each passthrough is validated. The validation works for `yaml`, `xml`, `json` and `toml` content. The proper validator is identified based on the extension used in the `target` parameter of the `[[$analyzer.passthrough]]` section. (Default: `false`) | +| `timeout` | Passthroughs | The maximum time to spend to evaluate the passthrough chain, before timing out. The timeout cannot exceed 300 seconds. (Default: 60) | + +#### `interpolate` + +WARNING: +To reduce the risk of leaking secrets, use this feature with caution. + +The example below shows a configuration that uses the `$GITURL` environment variable to access a +private repository. The variable contains a username and token (for example `https://user:token@url`), so +they're not explicitly stored in the configuration file. + +```toml +[semgrep] + description = "My private Semgrep ruleset" + interpolate = true + + [[semgrep.passthrough]] + type = "git" + value = "$GITURL" + ref = "refs/heads/main" ``` -Those vulnerabilities containing the provided type and value are now disabled, meaning -they won't be displayed in Merge Request nor the Vulnerability Report. +### The `[[$analyzer.ruleset]]` section -## Override predefined analyzer rules +The `[[$analyzer.ruleset]]` section targets and modifies a single predefined rule. You can define +one to many of these sections per analyzer. -To override analyzer rules: +| Setting | Description | +| --------| ----------- | +| `disable` | Whether the rule should be disabled. (Default: `false`) | +| `[$analyzer.ruleset.identifier]` | Selects the predefined rule to be modified. | +| `[$analyzer.ruleset.override]` | Defines the overrides for the rule. | -1. In one or more `ruleset.identifier` subsections, list the rules that you want to override. Every `ruleset.identifier` section has: +Configuration example: - - a `type` field, to name the predefined rule identifier that the targeted analyzer uses. - - a `value` field, to name the rule to be overridden. +```toml +[semgrep] + [[semgrep.ruleset]] + disable = true + ... +``` -1. In the `ruleset.override` context of a `ruleset` section, - provide the keys to override. Any combination of keys can be - overridden. Valid keys are: +### The `[$analyzer.ruleset.identifier]` section - - description - - message - - name - - severity (valid options are: Critical, High, Medium, Low, Unknown, Info) +The `[$analyzer.ruleset.identifier]` section defines the identifiers of the predefined +rule that you wish to modify. -### Example: Override predefined rules of SAST analyzers +| Setting | Description | +| --------| ----------- | +| `type` | The type of identifier used by the predefined rule. | +| `value` | The value of the identifier used by the predefined rule. | -Before adding a ruleset, we verify which vulnerability will be overwritten by viewing the [`gl-sast-report.json`](index.md#reports-json-format): +You can look up the correct values for `type` and `value` by viewing the +[`gl-sast-report.json`](index.md#reports-json-format) produced by the analyzer. +You can download this file as a job artifact from the analyzer's CI job. + +For example, the snippet below shows a finding from a `semgrep` rule with three +identifiers. The `type` and `value` keys in the JSON object correspond to the +values you should provide in this section. ```json -"identifiers": [ +... + "vulnerabilities": [ + { + "id": "7331a4b7093875f6eb9f6eb1755b30cc792e9fb3a08c9ce673fb0d2207d7c9c9", + "category": "sast", + "message": "Key Exchange without Entity Authentication", + "description": "Audit the use of ssh.InsecureIgnoreHostKey\n", + ... + "identifiers": [ { - "type": "gosec_rule_id", - "name": "Gosec Rule ID G307", - "value": "G307" + "type": "semgrep_id", + "name": "gosec.G106-1", + "value": "gosec.G106-1" }, { - "type": "CWE", - "name": "CWE-703", - "value": "703", - "url": "https://cwe.mitre.org/data/definitions/703.html" + "type": "cwe", + "name": "CWE-322", + "value": "322", + "url": "https://cwe.mitre.org/data/definitions/322.html" + }, + { + "type": "gosec_rule_id", + "name": "Gosec Rule ID G106", + "value": "G106" } ] + } + ... + ] +... ``` -In the following example, rules from `gosec` are matched by the `type` -and `value` of identifiers and then overridden: +Configuration example: ```toml -[gosec] - [[gosec.ruleset]] - [gosec.ruleset.identifier] - type = "CWE" - value = "703" - [gosec.ruleset.override] +[semgrep] + [[semgrep.ruleset]] + [semgrep.ruleset.identifier] + type = "semgrep_id" + value = "gosec.G106-1 + ... +``` + +### The `[$analyzer.ruleset.override]` section + +The `[$analyzer.ruleset.override]` section allows you to override attributes of a predefined rule. + +| Setting | Description | +| --------| ----------- | +| `description` | A detailed description of the issue. | +| `message` | (Deprecated) A description of the issue. | +| `name` | The name of the rule. | +| `severity` | The severity of the rule. Valid options are: `Critical`, `High`, `Medium`, `Low`, `Unknown`, `Info`) | + +NOTE: +While `message` is populated by the analyzers, it has been [deprecated](https://gitlab.com/gitlab-org/security-products/analyzers/report/-/blob/1d86d5f2e61dc38c775fb0490ee27a45eee4b8b3/vulnerability.go#L22) +in favor of `name` and `description`. + +Configuration example: + +```toml +[semgrep] + [[semgrep.ruleset]] + [semgrep.ruleset.override] severity = "Critical" + name = "Command injection" + ... ``` -If a vulnerability is found with a type `CWE` with a value of `703` then -the vulnerability severity is overwritten to `Critical`. +### The `[[$analyzer.passthrough]]` section -## Synthesize a custom configuration +NOTE: +This is currently supported by the `nodejs-scan` and `semgrep` analyzers only. -To create a custom configuration, you can use passthrough chains. +The `[[$analyzer.passthrough]]` section allows you to synthesize a custom configuration for an analyzer. You +can define up to 20 of these sections per analyzer. Passthroughs are composed into a _passthrough chain_ +that evaluates into a complete configuration that replaces the predefined rules of the analyzer. -A passthrough is a single step in a passthrough chain. The passthrough is evaluated -in a sequence to incrementally build a configuration. The configuration is then -passed to the target analyzer. +Passthroughs are evaluated in order. Passthroughs listed later in the chain have +a higher precedence and can overwrite or append to data yielded by previous +passthroughs (depending on the `mode`). This is useful for cases where you need +to use or modify an existing configuration. -A configuration section for an analyzer has the following -parameters: +The amount of data generated by a single passthrough is limited to 1MB. -| Parameter | Explanation | -| ------------- | ------ | -| `description` | Description about the analyzer configuration section. | -| `targetdir` | The `targetdir` parameter defines the directory where the final configuration is located. If `targetdir` is empty, the analyzer uses a random directory. The maximum size of `targetdir` is 100MB. | -| `validate` | If set to `true`, the target files for passthroughs (`raw`, `file` and `url`) are validated. The validation works for `yaml`, `xml`, `json` and `toml` files. The proper validator is identified based on the extension of the target file. By default, `validate` is set to `false`. | -| `interpolate` | If set to `true`, environment variable interpolation is enabled so that the configuration uses secrets/tokens. We advise using this feature with caution to not leak any secrets. By default, `interpolate` is set to `false`. | -| `timeout` | The total `timeout` for the evaluation of a passthrough chain is set to 60 seconds. If `timeout` is not set, the default timeout is 60 seconds. The timeout cannot exceed 300 seconds. | +| Setting | Applies to | Description | +| ------- | ---------- | ----------- | +| `type` | All | One of `file`, `raw`, `git` or `url`. | +| `target` | All | The target file to contain the data written by the passthrough evaluation. If empty, a random filename is used. | +| `mode` | All | If `overwrite`, the `target` file is overwritten. If `append`, new content is appended to the `target` file. Note that the `git` type only supports `overwrite`. (Default: `overwrite`) | +| `ref` | `type = "git"` | Contains the name of the branch or the SHA to pull. When using a branch name, specify it in the form `refs/heads/<branch>`, not `refs/remotes/<remote_name>/<branch>`. | +| `subdir` | `type = "git"` | Used to select a subdirectory of the Git repository as the configuration source. | +| `value` | All | For the `file`, `url`, and `git` types, defines the location of the file or Git repository. For the `raw` type, contains the inline configuration. | +| `validator` | All | Used to explicitly invoke validators (`xml`, `yaml`, `json`, `toml`) on the target file after the evaluation of a passthrough. | -A configuration section can include one or more passthrough sections. The maximum number of passthrough sections is 20. -There are several types of passthroughs: +#### Passthrough types | Type | Description | -| ------ | ------ | -| `file` | Use a file that is already available in the Git repository. | +| ------ | ----------- | +| `file` | Use a file that is present in the Git repository. | | `raw` | Provide the configuration inline. | | `git` | Pull the configuration from a remote Git repository. | -| `url` | Fetch the analyzer configuration through HTTP. | +| `url` | Fetch the configuration using HTTP. | -If multiple passthrough sections are defined in a passthrough chain, their -position in the chain defines the order in which they are evaluated. +WARNING: +When using the `raw` passthrough with a YAML snippet, it's recommended to format all indentation +in the `sast-ruleset.toml` file as spaces. The YAML specification mandates spaces over tabs, and the +analyzer will fail to parse your custom ruleset unless the indentation is represented accordingly. -- Passthroughs listed later in the chain sequence have a higher precedence. -- Passthroughs with a higher precedence overwrite (default) and append data - yielded by previous passthroughs. This is useful for cases where you need to - use or modify an existing configuration. +## Examples -Configure a passthrough these parameters: +### Disable predefined rules of SAST analyzers -| Parameter | Explanation | -| ------------ | ----------- | -| `type` | One of `file`, `raw`, `git` or `url`. | -| `target` | The target file that contains the data written by the passthrough evaluation. If no value is provided, a random target file is generated. | -| `mode` | `overwrite`: if `target` exists, overwrites the file; `append`: append to file instead. The default is `overwrite`. | -| `ref` | This option only applies to the `git` passthrough type and contains the name of the branch or the SHA to be used. When using a branch name, specify it in the form `refs/heads/<branch>`, not `refs/remotes/<remote_name>/<branch>`. | -| `subdir` | This option only applies to the `git` passthrough type and can be used to only consider a certain subdirectory of the source Git repository. | -| `value` | For the `file` `url` and `git` types, `value` defines the source location of the file/Git repository; for the `raw` type, `value` carries the raw content to be passed through. | -| `validator` | Can be used to explicitly invoke validators (`xml`, `yaml`, `json`, `toml`) on the target files after the application of a passthrough. Per default, no validator is set. | +With the following custom ruleset configuration, the following rules are omitted from the report: -The amount of data generated by a single passthrough is limited to 1MB. +- `semgrep` rules with a `semgrep_id` of `gosec.G106-1` or a `cwe` of `322`. +- `sobelow` rules with a `sobelow_rule_id` of `sql_injection`. +- `flawfinder` rules with a `flawfinder_func_name` of `memcpy`. -## Passthrough configuration examples +```toml +[semgrep] + [[semgrep.ruleset]] + disable = true + [semgrep.ruleset.identifier] + type = "semgrep_id" + value = "gosec.G106-1" -### Raw passthrough for nodejs-scan + [[semgrep.ruleset]] + disable = true + [semgrep.ruleset.identifier] + type = "cwe" + value = "322" -Define a custom analyzer configuration. In this example, customized rules are -defined for the `nodejs-scan` scanner: +[sobelow] + [[sobelow.ruleset]] + disable = true + [sobelow.ruleset.identifier] + type = "sobelow_rule_id" + value = "sql_injection" + +[flawfinder] + [[flawfinder.ruleset]] + disable = true + [flawfinder.ruleset.identifier] + type = "flawfinder_func_name" + value = "memcpy" +``` + +### Override predefined rules of SAST analyzers + +With the following custom ruleset configuration, vulnerabilities found with +`semgrep` with a type `CWE` and a value `322` will have their severity +overridden to `Critical`. + +```toml +[semgrep] + [[semgrep.ruleset]] + [semgrep.ruleset.identifier] + type = "CWE" + value = "322" + [semgrep.ruleset.override] + severity = "Critical" +``` + +### Synthesize a custom configuration using a raw passthrough for `nodejs-scan` + +With the following custom ruleset configuration, the predefined behavior +of the `nodejs-scan` analyzer is replaced with a custom configuration. + +The syntax used for the `value` follows the [njsscan config format](https://github.com/ajinabraham/njsscan#configure-njsscan). ```toml [nodejs-scan] - description = 'custom ruleset for nodejs-scan' + description = "My custom ruleset for nodejs-scan" [[nodejs-scan.passthrough]] type = "raw" value = ''' +--- - nodejs-extensions: - .js - + template-extensions: - .new - .hbs - '' - + ignore-filenames: -- skip.js - + - skip.js + ignore-paths: - __MACOSX - skip_dir - node_modules - + ignore-extensions: - .hbs - + ignore-rules: - regex_injection_dos - pug_jade_template - express_xss - ''' ``` -### File passthrough for Gosec +### Synthesize a custom configuration using a file passthrough for `semgrep` -Provide the name of the file containing a custom analyzer configuration. In -this example, customized rules for the `gosec` scanner are contained in the -file `gosec-config.json`: +With the following custom ruleset configuration, the predefined ruleset +of the `semgrep` analyzer is replaced with a custom ruleset contained in +a file called `my-semgrep-rules.yaml` in the repository being scanned. + +```yaml +# my-semgrep-rules.yml +--- +rules: +- id: my-custom-rule + pattern: print("Hello World") + message: | + Unauthorized use of Hello World. + severity: CRITICAL + languages: + - python +``` ```toml -[gosec] - description = 'custom ruleset for gosec' +[semgrep] + description = "My custom ruleset for Semgrep" - [[gosec.passthrough]] + [[semgrep.passthrough]] type = "file" - value = "gosec-config.json" + value = "my-semgrep-rules.yml" ``` -### Passthrough chain for Semgrep +### Synthesize a custom configuration using a passthrough chain for `semgrep` -In the below example, we generate a custom configuration under the `/sgrules` -target directory with a total `timeout` of 60 seconds. +With the following custom ruleset configuration, the predefined ruleset +of the `semgrep` analyzer is replaced with a custom ruleset produced by +evaluating a chain of four passthroughs. Each passthrough produces a file +that's written to the `/sgrules` directory within the container. A +`timeout` of 60 seconds is set in case any Git remotes are unresponsive. -Several passthrouh types generate a configuration for the target analyzer: +Different passthrough types are demonstrated in this example: -- Two `git` passthrough sections pull the head of branch - `refs/heads/test` from the `myrules` Git repository, and revision - `97f7686` from the `sast-rules` Git repository. From the `sast-rules` Git - repository, only data from the `go` subdirectory is considered. +- Two `git` passthroughs, the first pulling `refs/heads/test` from the + `myrules` Git repository, and the second pulling revision `97f7686` + from the `sast-rules` repository, and considering only files in the + `go` subdirectory. - The `sast-rules` entry has a higher precedence because it appears later in the configuration. - - If there is a filename collision between files in both repositories, files - from the `sast` repository overwrite files from the `myrules` repository, - as `sast-rules` has higher precedence. -- The `raw` entry creates a file named `insecure.yml` under `/sgrules`. The - full path is `/sgrules/insecure.yml`. -- The `url` entry fetches a configuration made available through a URL and - stores it in the `/sgrules/gosec.yml` file. + - If there's a filename collision between the two checkouts, files + from the `sast-rules` repository will overwrite files from the + `myrules` repository. +- A `raw` passthrough, which writes its `value` to `/sgrules/insecure.yml`. +- A `url` passthrough, which fetches a configuration hosted at a URL and + writes it to `/sgrules/gosec.yml`. Afterwards, Semgrep is invoked with the final configuration located under `/sgrules`. ```toml [semgrep] - description = 'semgrep custom rules configuration' + description = "My custom ruleset for Semgrep" targetdir = "/sgrules" timeout = 60 @@ -277,15 +449,15 @@ Afterwards, Semgrep is invoked with the final configuration located under rules: - id: "insecure" patterns: - - pattern: "func insecure() {...}" + - pattern: "func insecure() {...}" message: | Insecure function insecure detected metadata: cwe: "CWE-200: Exposure of Sensitive Information to an Unauthorized Actor" severity: "ERROR" languages: - - "go" - """ + - "go" +""" [[semgrep.passthrough]] type = "url" @@ -293,89 +465,93 @@ rules: target = "gosec.yml" ``` -### Interpolation - -The code snippet below shows an example configuration that uses an environment -variable `$GITURL` to access a private repositories with a Git URL. The variable contains -a username and token in the `value` field (for example `https://user:token@url`). -It does not explicitly store credentials in the configuration file. To reduce the risk of leaking secrets through created paths and files, use this feature with caution. - -```toml -[semgrep] - description = 'semgrep custom rules configuration' - targetdir = "/sgrules" - interpolate = true - - [[semgrep.passthrough]] - type = "git" - value = "$GITURL" - ref = "refs/heads/main" -``` - -### Configure the append mode for passthroughs - -To append data to previous passthroughs, use the `append` mode for the -passthrough types `file`, `url`, and `raw`. +### Configure the mode for passthroughs in a chain -Passthroughs in `override` mode overwrite files -created when preceding passthroughs in the chain find a naming -collision. If `mode` is set to `append`, a passthrough appends data to the -files created by its predecessors instead of overwriting. +You can choose how to handle filename conflicts that occur between +passthroughs in a chain. The default behavior is to overwrite +existing files with the same name, but you can choose `mode = append` +instead to append the content of later files onto earlier ones. -In the below Semgrep configuration,`/sgrules/insecure.yml` assembles two passthroughs. The rules are: +You can use the `append` mode for the `file`, `url`, and `raw` +passthrough types only. -- `insecure` -- `secret` - -These rules add a search pattern to the analyzer and extends Semgrep capabilities. - -For passthrough chains we recommend that you enable validation. To enable validation, -you can either: - -- set `validate` to `true` - -- set a passthrough `validator` to `xml`, `json`, `yaml`, or `toml`. +With the following custom ruleset configuration, two `raw` passthroughs +are used to iteratively assemble the `/sgrules/my-rules.yml` file, which +is then provided to Semgrep as the ruleset. Each passthrough appends a +single rule to the ruleset. The first passthrough is responsible for +initialising the top-level `rules` object, according to the +[Semgrep rule syntax](https://semgrep.dev/docs/writing-rules/rule-syntax/). ```toml [semgrep] - description = 'semgrep custom rules configuration' + description = "My custom ruleset for Semgrep" targetdir = "/sgrules" validate = true [[semgrep.passthrough]] type = "raw" - target = "insecure.yml" + target = "my-rules.yml" value = """ rules: - id: "insecure" patterns: - - pattern: "func insecure() {...}" + - pattern: "func insecure() {...}" message: | - Insecure function insecure detected + Insecure function 'insecure' detected metadata: - cwe: "... + cwe: "..." severity: "ERROR" languages: - - "go" + - "go" """ [[semgrep.passthrough]] type = "raw" mode = "append" - target = "insecure.yml" + target = "my-rules.yml" value = """ - id: "secret" patterns: - - pattern-either: - - pattern: "$MASK = \"...\"" - - metavariable-regex: - metavariable: "$MASK" - regex: "(password|pass|passwd|pwd|secret|token)" + - pattern-either: + - pattern: '$MASK = "..."' + - metavariable-regex: + metavariable: "$MASK" + regex: "(password|pass|passwd|pwd|secret|token)" message: | - Use of Hard-coded Password + Use of hard-coded password + metadata: cwe: "..." severity: "ERROR" languages: - - "go" + - "go" """ ``` + +```yaml +# /sgrules/my-rules.yml +rules: +- id: "insecure" + patterns: + - pattern: "func insecure() {...}" + message: | + Insecure function 'insecure' detected + metadata: + cwe: "..." + severity: "ERROR" + languages: + - "go" +- id: "secret" + patterns: + - pattern-either: + - pattern: '$MASK = "..."' + - metavariable-regex: + metavariable: "$MASK" + regex: "(password|pass|passwd|pwd|secret|token)" + message: | + Use of hard-coded password + metadata: + cwe: "..." + severity: "ERROR" + languages: + - "go" +``` diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index b1bc9794ced..94719224254 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -105,6 +105,7 @@ Check the [SAST direction page](https://about.gitlab.com/direction/secure/static | React | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with GitLab-managed rules | 13.10 | | Ruby | [brakeman](https://gitlab.com/gitlab-org/security-products/analyzers/brakeman) | 13.9 | | Ruby on Rails | [brakeman](https://gitlab.com/gitlab-org/security-products/analyzers/brakeman) | 10.3 | +| Scala (any build system) | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with GitLab-managed rules | 15.7 | | Scala<sup>2</sup> | [SpotBugs](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) with the find-sec-bugs plugin | 11.0 (SBT) & 11.9 (Gradle, Maven) | | Swift (iOS) | [MobSF (beta)](https://gitlab.com/gitlab-org/security-products/analyzers/mobsf) | 13.5 | | TypeScript<sup>3</sup> | [ESLint security plugin](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) | 11.9, [merged](https://gitlab.com/gitlab-org/gitlab/-/issues/36059) with ESLint in 13.2 | @@ -694,7 +695,7 @@ The process for importing Docker images into a local offline Docker registry dep process by which external resources can be imported or temporarily accessed. These scanners are [periodically updated](../index.md#vulnerability-scanner-maintenance) with new definitions, and you may be able to make occasional updates on your own. -For details on saving and transporting Docker images as a file, see Docker's documentation on +For details on saving and transporting Docker images as a file, see the Docker documentation on [`docker save`](https://docs.docker.com/engine/reference/commandline/save/), [`docker load`](https://docs.docker.com/engine/reference/commandline/load/), [`docker export`](https://docs.docker.com/engine/reference/commandline/export/), and [`docker import`](https://docs.docker.com/engine/reference/commandline/import/). diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index 8a066cf1be1..d955170ece2 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -82,43 +82,57 @@ To enable Secret Detection, either: - Enable [Auto DevOps](../../../topics/autodevops/index.md), which includes [Auto Secret Detection](../../../topics/autodevops/stages.md#auto-secret-detection). -- [Enable Secret Detection by including the template](#enable-secret-detection-by-including-the-template). +- [Edit the `.gitlab.ci.yml` file manually](#edit-the-gitlabciyml-file-manually). Use this method if + your `.gitlab-ci.yml` file is complex. -- [Enable Secret Detection using a merge request](#enable-secret-detection-using-a-merge-request). +- [Use an automatically configured merge request](#use-an-automatically-configured-merge-request). -### Enable Secret Detection by including the template +### Edit the `.gitlab.ci.yml` file manually -You should use this method if you have an existing GitLab CI/CD configuration file. +This method requires you to manually edit the existing `.gitlab-ci.yml` file. Use this method if +your GitLab CI/CD configuration file is complex. -Add the following extract to your `.gitlab-ci.yml` file: +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **CI/CD > Editor**. +1. Copy and paste the following to the bottom of the `.gitlab-ci.yml` file: -```yaml -include: - - template: Jobs/Secret-Detection.gitlab-ci.yml -``` + ```yaml + include: + - template: Jobs/Secret-Detection.gitlab-ci.yml + ``` -Pipelines now include a Secret Detection job, and the results are included in the merge request -widget. +1. Select the **Validate** tab, then select **Validate pipeline**. + The message **Simulation completed successfully** indicates the file is valid. +1. Select the **Edit** tab. +1. Optional. In the **Commit message** text box, customize the commit message. +1. In the **Branch** text box, enter the name of the default branch. +1. Select **Commit changes**. -### Enable Secret Detection using a merge request +Pipelines now include a Secret Detection job. + +### Use an automatically configured merge request > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4496) in GitLab 13.11, deployed behind a feature flag, enabled by default. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/329886) in GitLab 14.1. +This method automatically prepares a merge request, with the Secret Detection template included in +the `.gitlab-ci.yml` file. You then merge the merge request to enable Secret Detection. + NOTE: This method works best with no existing `.gitlab-ci.yml` file, or with a minimal configuration file. If you have a complex GitLab configuration file it may not be parsed successfully, and an -error may occur. +error may occur. In that case, use the [manual](#edit-the-gitlabciyml-file-manually) method instead. -To enable Secret Detection using a merge request: +To enable Secret Detection automatically: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Security & Compliance > Configuration**. 1. In the **Secret Detection** row, select **Configure with a merge request**. +1. Optional. Complete the fields. +1. Select **Create merge request**. 1. Review and merge the merge request. -Pipelines now include a Secret Detection job, and the results are included in the merge request -widget. +Pipelines now include a Secret Detection job. ## Responding to a leaked secret diff --git a/doc/user/application_security/secret_detection/post_processing.md b/doc/user/application_security/secret_detection/post_processing.md index 8dbe459d4af..9c74467bce5 100644 --- a/doc/user/application_security/secret_detection/post_processing.md +++ b/doc/user/application_security/secret_detection/post_processing.md @@ -6,7 +6,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Secret Detection post-processing and revocation **(FREE SAAS)** -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4639) in GitLab 13.6. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4639) in GitLab 13.6. +> - [Disabled by default for GitLab personal access tokens](https://gitlab.com/gitlab-org/gitlab/-/issues/371658) in GitLab 15.6 [with a flag](../../../administration/feature_flags.md) named `gitlab_pat_auto_revocation`. Available to GitLab.com only. + +FLAG: +By default, auto revocation of GitLab personal access tokens is not available. To opt-in on GitLab.com +during the [Beta period](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga), please +[let us know by completing this form](https://docs.google.com/forms/d/e/1FAIpQLSdRbFhvA5jvI-Rt_Qnl1PQ1znOXKK8m6lRtmM0uva4upetKvQ/viewform). GitLab supports running post-processing hooks after detecting a secret. These hooks can perform actions, like notifying the cloud service that issued the secret. @@ -16,7 +22,7 @@ The cloud provider can then confirm the credentials and take remediation actions - Reissuing a secret. - Notifying the creator of the secret. -GitLab SaaS supports post-processing for Amazon Web Services (AWS). +GitLab SaaS supports post-processing for [GitLab personal access tokens](../../profile/personal_access_tokens.md) and Amazon Web Services (AWS). Post-processing workflows vary by supported cloud providers. Post-processing is limited to a project's default branch. The epic diff --git a/doc/user/application_security/terminology/index.md b/doc/user/application_security/terminology/index.md index f156d60be8f..2666d91c5aa 100644 --- a/doc/user/application_security/terminology/index.md +++ b/doc/user/application_security/terminology/index.md @@ -83,11 +83,6 @@ once. A finding that doesn't exist but is incorrectly reported as existing. -### Feedback - -Feedback the user provides about a finding. Types of feedback include dismissal, creating an issue, -or creating a merge request. - ### Finding An asset that has the potential to be vulnerable, identified in a project by an analyzer. Assets @@ -96,6 +91,11 @@ applications, and infrastructure. Findings are all potential vulnerability items scanners identify in MRs/feature branches. Only after merging to default does a finding become a [vulnerability](#vulnerability). +You can interact with vulnerability findings in two ways. + +1. You can open an issue or merge request for the vulnerability finding. +1. You can dismiss the vulnerability finding. Dismissing the finding hides it from the default views. + ### Grouping A flexible and non-destructive way to visually organize vulnerabilities in groups when there are multiple findings @@ -164,15 +164,15 @@ table.package-managers-and-types ul { <tbody> <tr> <td>gem</td> - <td><a href="https://bundler.io/">bundler</a></td> + <td><a href="https://bundler.io/">Bundler</a></td> </tr> <tr> - <td>packagist</td> - <td><a href="https://getcomposer.org/">composer</a></td> + <td>Packagist</td> + <td><a href="https://getcomposer.org/">Composer</a></td> </tr> <tr> - <td>conan</td> - <td><a href="https://conan.io/">conan</a></td> + <td>Conan</td> + <td><a href="https://conan.io/">Conan</a></td> </tr> <tr> <td>go</td> @@ -180,10 +180,10 @@ table.package-managers-and-types ul { </tr> <tr> <td rowspan="3">maven</td> - <td><a href="https://gradle.org/">gradle</a></td> + <td><a href="https://gradle.org/">Gradle</a></td> </tr> <tr> - <td><a href="https://maven.apache.org/">maven</a></td> + <td><a href="https://maven.apache.org/">Maven</a></td> </tr> <tr> <td><a href="https://www.scala-sbt.org">sbt</a></td> @@ -196,12 +196,12 @@ table.package-managers-and-types ul { <td><a href="https://classic.yarnpkg.com/en">yarn</a></td> </tr> <tr> - <td>nuget</td> - <td><a href="https://www.nuget.org/">nuget</a></td> + <td>NuGet</td> + <td><a href="https://www.nuget.org/">NuGet</a></td> </tr> <tr> - <td rowspan="4">pypi</td> - <td><a href="https://setuptools.pypa.io/en/latest/">setuptools</a></td> + <td rowspan="4">PyPI</td> + <td><a href="https://setuptools.pypa.io/en/latest/">Setuptools</a></td> </tr> <tr> <td><a href="https://pip.pypa.io/en/stable">pip</a></td> diff --git a/doc/user/application_security/vulnerabilities/index.md b/doc/user/application_security/vulnerabilities/index.md index 9ddb1bb51e2..e86f9ff4673 100644 --- a/doc/user/application_security/vulnerabilities/index.md +++ b/doc/user/application_security/vulnerabilities/index.md @@ -59,6 +59,8 @@ To change a vulnerability's status from its Vulnerability Page: 1. From the **Status** dropdown list select a status, then select **Change status**. 1. Optionally, at the bottom of the page, add a comment to the log entry. +The Actions log records each status change along with which user changed the status and the time of the change. + ## Creating an issue for a vulnerability From a vulnerability's page you can create an issue to track all action taken to resolve or diff --git a/doc/user/application_security/vulnerability_report/index.md b/doc/user/application_security/vulnerability_report/index.md index 2b78dde4f63..dd919889b4d 100644 --- a/doc/user/application_security/vulnerability_report/index.md +++ b/doc/user/application_security/vulnerability_report/index.md @@ -260,7 +260,7 @@ To add a new vulnerability finding from your project level Vulnerability Report 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Security & Compliance > Vulnerability Report**. -1. Select **Submit Vulnerability**. +1. Select **Submit vulnerability**. 1. Complete the fields and submit the form. You will be brought to the newly created vulnerability's detail page. Manually created records appear in the diff --git a/doc/user/application_security/vulnerability_report/pipeline.md b/doc/user/application_security/vulnerability_report/pipeline.md index 9e20e4f6f78..57c18cb045e 100644 --- a/doc/user/application_security/vulnerability_report/pipeline.md +++ b/doc/user/application_security/vulnerability_report/pipeline.md @@ -79,6 +79,23 @@ incorporated once the pipeline finishes. | Resolved | No | Needs triage (Detected) | | N/A (i.e.: new vulnerability) | No | Needs triage (Detected) | +## Retention period for vulnerabilities + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/351524) in GitLab 15.5. + +GitLab has the following retention policies for vulnerabilities on non-default branches. Vulnerabilities are no longer available: + +- When the related CI job artifact expires. +- 90 days after the pipeline is created, even if the related CI job artifacts are locked. + +To view vulnerabilities, either: + +- Run a new pipeline. +- Download the related CI job artifacts if they are available. + +NOTE: +This does not apply for the vulnerabilities existing on the default branch. + ## Deduplication process When a pipeline contains jobs that produce multiple security reports of the same type, it is possible that the same diff --git a/doc/user/asciidoc.md b/doc/user/asciidoc.md index 3ceecf88c5e..6b642363dd6 100644 --- a/doc/user/asciidoc.md +++ b/doc/user/asciidoc.md @@ -9,6 +9,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w GitLab uses the [Asciidoctor](https://asciidoctor.org) gem to convert AsciiDoc content to HTML5. Consult the [Asciidoctor User Manual](https://asciidoctor.org/docs/user-manual/) for a complete Asciidoctor reference. +You can use AsciiDoc in the following areas: + +- Wiki pages +- AsciiDoc documents (`.adoc` or `.asciidoc`) inside repositories + ## Syntax Here's a brief reference of the most commonly used AsciiDoc syntax. diff --git a/doc/user/clusters/agent/ci_cd_workflow.md b/doc/user/clusters/agent/ci_cd_workflow.md index 454be3c53c7..2a66549f9cb 100644 --- a/doc/user/clusters/agent/ci_cd_workflow.md +++ b/doc/user/clusters/agent/ci_cd_workflow.md @@ -60,6 +60,7 @@ Authorization configuration can take one or two minutes to propagate. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327850) in GitLab 14.4. > - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/346566) to remove hierarchy restrictions in GitLab 15.6. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/356831) to allow authorizing projects in a user namespace in GitLab 15.7. To authorize the agent to access the GitLab project where you keep Kubernetes manifests: @@ -73,7 +74,7 @@ To authorize the agent to access the GitLab project where you keep Kubernetes ma - id: path/to/project ``` - - Authorized projects must have the same root group as the agent's configuration project. + - Authorized projects must have the same root group or user namespace as the agent's configuration project. - You can install additional agents into the same cluster to accommodate additional hierarchies. - You can authorize up to 100 projects. @@ -286,6 +287,35 @@ The identity can be specified with the following keys: See the [official Kubernetes documentation for details](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation). +## Restrict project and group access to specific environments **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343885) in GitLab 15.7. + +By default, if your agent is [available to a project](#authorize-the-agent), all of the project's CI/CD jobs can use that agent. + +To restrict access to the agent to only jobs with specific environments, add `environments` to `ci_access.projects` or `ci_access.groups`. For example: + + ```yaml + ci_access: + projects: + - id: path/to/project-1 + - id: path/to/project-2 + environments: + - staging + - review/* + groups: + - id: path/to/group-1 + environments: + - production + ``` + +In this example: + +- All CI/CD jobs under `project-1` can access the agent. +- CI/CD jobs under `project-2` with `staging` or `review/*` environments can access the agent. + - `*` is a wildcard, so `review/*` matches all environments under `review`. +- CI/CD jobs for projects under `group-1` with `production` environments can access the agent. + ## Related topics - [Self-paced classroom workshop](https://gitlab-for-eks.awsworkshop.io) (Uses AWS EKS, but you can use for other Kubernetes clusters) diff --git a/doc/user/clusters/agent/gitops.md b/doc/user/clusters/agent/gitops.md index c0d4a5e088f..bd7dfb3abee 100644 --- a/doc/user/clusters/agent/gitops.md +++ b/doc/user/clusters/agent/gitops.md @@ -9,6 +9,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259669) in GitLab 13.7. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332227) in GitLab 14.0, the `resource_inclusions` and `resource_exclusions` attributes were removed and `reconcile_timeout`, `dry_run_strategy`, `prune`, `prune_timeout`, `prune_propagation_policy`, and `inventory_policy` attributes were added. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/346567) from GitLab Premium to GitLab Free in 15.3. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/346585) to make the `id` attribute optional in GitLab 15.7. +> - Specifying a branch, tag, or commit reference to fetch the Kubernetes manifest files [introduced](https://gitlab.com/groups/gitlab-org/-/epics/4516) in GitLab 15.7. With GitOps, you can manage containerized clusters and applications from a Git repository that: @@ -64,6 +66,10 @@ The following snippet shows an example of the possible keys and values for the G gitops: manifest_projects: - id: gitlab-org/cluster-integration/gitlab-agent + ref: # either `branch`, `tag` or `commit` can be specified + branch: production + # commit: <mysha> + # tag: v1.0 default_namespace: my-ns paths: # Read all YAML files from this directory. @@ -83,7 +89,11 @@ gitops: | Keyword | Description | |--|--| | `manifest_projects` | Projects where your Kubernetes manifests are stored. The agent monitors the files in the repositories in these projects. When manifest files change, the agent deploys the changes to the cluster. | -| `id` | Required. Path to a Git repository that has Kubernetes manifests in YAML or JSON format. No authentication mechanisms are currently supported. | +| `id` | Path to a Git repository that has Kubernetes manifests in YAML or JSON format. No authentication mechanisms are supported. Default is the agent configuration repository. | +| `ref` | Optional. Git reference in the configured Git repository to fetch the Kubernetes manifest files from. If not specified or empty, the default branch is used. If specified, it must contain either `branch`, `tag`, or `commit`. | +| `ref.branch` | Branch name in the configured Git repository to fetch the Kubernetes manifest files from. | +| `ref.tag` | Tag name in the configured Git repository to fetch the Kubernetes manifest files from. | +| `ref.commit` | Commit SHA in the configured Git repository to fetch the Kubernetes manifest files from. | | `default_namespace` | Namespace to use if not set explicitly in object manifest. Also used for inventory `ConfigMap` objects. | | `paths` | Repository paths to scan for manifest files. Directories with names that start with a dot `(.)` are ignored. | | `paths[].glob` | Required. See [doublestar](https://github.com/bmatcuk/doublestar#about) and [the match function](https://pkg.go.dev/github.com/bmatcuk/doublestar/v2#Match) for globbing rules. | diff --git a/doc/user/clusters/agent/gitops/helm.md b/doc/user/clusters/agent/gitops/helm.md index 0ec87376636..b9a59d37f5d 100644 --- a/doc/user/clusters/agent/gitops/helm.md +++ b/doc/user/clusters/agent/gitops/helm.md @@ -6,7 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Using Helm charts to update a Kubernetes cluster (Alpha) **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/371019) in GitLab 15.4. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/371019) in GitLab 15.4. +> - Specifying a branch, tag, or commit reference to fetch the Kubernetes manifest files [introduced](https://gitlab.com/groups/gitlab-org/-/epics/4516) in GitLab 15.7. You can deploy Helm charts to your Kubernetes cluster and keep the resources in your cluster in sync with your charts and values. To do this, you use the pull-based GitOps features of the agent for @@ -51,6 +52,8 @@ gitops: source: project: id: my-group/my-project-with-chart + ref: + branch: production path: dir-in-project/with/charts namespace: my-ns max_history: 1 @@ -68,6 +71,10 @@ gitops: | `max_history` | Optional. Maximum number of release [revisions to store in the cluster](https://helm.sh/docs/helm/helm_history/). | | `source` | Required. From where the chart should get installed. Only supports project sources. | | `source.project.id` | Required. ID of the project where Helm chart is committed. Authentication is not supported. | +| `source.project.ref` | Optional. Git reference in the configured Git repository to fetch the Chart from. If not specified or empty, the default branch is used. If specified, it must contain either `branch`, `tag`, or `commit`. | +| `source.project.ref.branch` | Branch name in the configured Git repository to fetch the Chart from. | +| `source.project.ref.tag` | Tag name in the configured Git repository to fetch the Chart from. | +| `source.project.ref.commit` | Commit SHA in the configured Git repository to fetch the Chart from. | | `source.project.path` | Optional. Path of the chart in the project repository. Root of the repository is used by default. Should be the directory with the `Chart.yaml` file. | ## Custom values diff --git a/doc/user/clusters/agent/install/index.md b/doc/user/clusters/agent/install/index.md index 2030052e3b0..62767f1dfd9 100644 --- a/doc/user/clusters/agent/install/index.md +++ b/doc/user/clusters/agent/install/index.md @@ -123,7 +123,7 @@ To install the agent on your cluster using Helm: 1. In your computer, open a terminal and [connect to your cluster](https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/). 1. Run the command you copied when you [registered your agent with GitLab](#register-the-agent-with-gitlab). -Optionally, you can [customize the Helm installation](#customize-the-helm-installation). +Optionally, you can [customize the Helm installation](#customize-the-helm-installation). If you install the agent on a production system, you should customize the Helm installation to skip creating the service account. ##### Customize the Helm installation diff --git a/doc/user/clusters/agent/vulnerabilities.md b/doc/user/clusters/agent/vulnerabilities.md index dcb3276deb5..d9a9981d211 100644 --- a/doc/user/clusters/agent/vulnerabilities.md +++ b/doc/user/clusters/agent/vulnerabilities.md @@ -12,17 +12,19 @@ info: To determine the technical writer assigned to the Stage/Group associated w To view cluster vulnerabilities, you can view the [vulnerability report](../../application_security/vulnerabilities/index.md). You can also configure your agent so the vulnerabilities are displayed with other agent information in GitLab. -## Enable operational container scanning **(ULTIMATE)** +## Enable operational container scanning -You can use operational container scanning -to scan container images in your cluster for security vulnerabilities. +You can use operational container scanning to scan container images in your cluster for security vulnerabilities. You +can enable the scanner to run on a cadence as configured via the agent, or setup scan execution policies within a +project that houses the agent. NOTE: In GitLab 15.0 and later, you do not need to install Starboard operator in the Kubernetes cluster. -To begin scanning all resources in your cluster, add a `container_scanning` -configuration block to your agent configuration with a `cadence` field -containing a CRON expression for when the scans will be run. +### Enable via agent configuration + +To enable scanning of all images within your Kubernetes cluster via the agent configuration, add a `container_scanning` configuration block to your agent +configuration with a `cadence` field containing a [CRON expression](https://docs.oracle.com/cd/E12058_01/doc/doc.1014/e12030/cron_expressions.htm) for when the scans will be run. ```yaml container_scanning: @@ -34,29 +36,67 @@ The `cadence` field is required. GitLab supports the following types of CRON syn - A daily cadence of once per hour at a specified hour, for example: `0 18 * * *` - A weekly cadence of once per week on a specified day and at a specified hour, for example: `0 13 * * 0` -It is possible that other elements of the CRON syntax will work in the cadence field, however, GitLab does not officially test or support them. +NOTE: +Other elements of the [CRON syntax](https://docs.oracle.com/cd/E12058_01/doc/doc.1014/e12030/cron_expressions.htm) may work in the cadence field if supported by the [cron](https://github.com/robfig/cron) we are using in our implementation, however, GitLab does not officially test or support them. + +NOTE: +The CRON expression is evaluated in [UTC](https://www.timeanddate.com/worldclock/timezone/utc) using the system-time of the Kubernetes-agent pod. By default, operational container scanning will attempt to scan the workloads in all -namespaces for vulnerabilities. The `vulnerability_report` block has a `namespaces` +namespaces for vulnerabilities. You can set the `vulnerability_report` block with the `namespaces` field which can be used to restrict which namespaces are scanned. For example, -if you would like to scan only the `development`, `staging`, and `production` -namespaces, you can use this configuration: +if you would like to scan only the `default`, `kube-system` namespaces, you can use this configuration: ```yaml container_scanning: cadence: '0 0 * * *' vulnerability_report: namespaces: - - development - - staging - - production + - default + - kube-system ``` -## View cluster vulnerabilities +## Enable via scan execution policies + +To enable scanning of all images within your Kubernetes cluster via scan execution policies, we can use the +[scan execution policy editor](../../application_security/policies/scan-execution-policies.md#scan-execution-policy-editor) +in order to create a new schedule rule. -Prerequisite: +NOTE: +The Kubernetes agent must be running in your cluster in order to scan running container images + +Here is an example of a policy which enables operational container scanning within the cluster the Kubernetes agent is attached to: + +```yaml +- name: Enforce Container Scanning in cluster connected through my-gitlab-agent for default and kube-system namespaces + enabled: true + rules: + - type: schedule + cadence: '0 10 * * *' + agents: + <agent-name>: + namespaces: + - 'default' + - 'kube-system' + actions: + - scan: container_scanning +``` -- You must have at least the Developer role. +The keys for a schedule rule are: + +- cadence (required): a [CRON expression](https://docs.oracle.com/cd/E12058_01/doc/doc.1014/e12030/cron_expressions.htm) for when the scans will be run +- agents:<agent-name> (required): The name of the agent to use for scanning +- agents:<agent-name>:namespaces (optional): The Kubernetes namespaces to scan. If omitted, all namespaces will be scanned + +NOTE: +Other elements of the [CRON syntax](https://docs.oracle.com/cd/E12058_01/doc/doc.1014/e12030/cron_expressions.htm) may work in the cadence field if supported by the [cron](https://github.com/robfig/cron) we are using in our implementation, however, GitLab does not officially test or support them. + +NOTE: +The CRON expression is evaluated in [UTC](https://www.timeanddate.com/worldclock/timezone/utc) using the system-time of the Kubernetes-agent pod. + +You can view the complete schema within the [scan execution policy documentation](../../application_security/policies/scan-execution-policies.md#scan-execution-policies-schema). + +## View cluster vulnerabilities To view vulnerability information in GitLab: @@ -68,3 +108,6 @@ To view vulnerability information in GitLab:  This information can also be found under [operational vulnerabilities](../../../user/application_security/vulnerability_report/index.md#operational-vulnerabilities). + +NOTE: +You must have at least the Developer role. diff --git a/doc/user/clusters/management_project_template.md b/doc/user/clusters/management_project_template.md index cbe577b9b74..a8d874ed608 100644 --- a/doc/user/clusters/management_project_template.md +++ b/doc/user/clusters/management_project_template.md @@ -102,7 +102,7 @@ The [built-in supported applications](https://gitlab.com/gitlab-org/project-temp - [Cert-manager](../infrastructure/clusters/manage/management_project_applications/certmanager.md) - [GitLab Runner](../infrastructure/clusters/manage/management_project_applications/runner.md) - [Ingress](../infrastructure/clusters/manage/management_project_applications/ingress.md) -- [Prometheus](../infrastructure/clusters/manage/management_project_applications/prometheus.md) +- [Prometheus](../../operations/metrics/index.md) - [Vault](../infrastructure/clusters/manage/management_project_applications/vault.md) Each application has an `applications/{app}/values.yaml` file. diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index fb5ce37c563..34434ef046a 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -177,7 +177,7 @@ directory of your project. Depending on your language, you may need to specify the path to the individual projects of a monorepo using the `LICENSE_FINDER_CLI_OPTS` variable. Passing in the project paths can significantly speed up builds over using the `--recursive` -license_finder option. +License Finder option. ```yaml include: @@ -660,7 +660,7 @@ The process for importing Docker images into a local offline Docker registry dep process by which external resources can be imported or temporarily accessed. Note that these scanners are [updated periodically](../../application_security/index.md#vulnerability-scanner-maintenance) with new definitions, so consider if you are able to make periodic updates yourself. -For details on saving and transporting Docker images as a file, see Docker's documentation on +For details on saving and transporting Docker images as a file, see the Docker documentation on [`docker save`](https://docs.docker.com/engine/reference/commandline/save/), [`docker load`](https://docs.docker.com/engine/reference/commandline/load/), [`docker export`](https://docs.docker.com/engine/reference/commandline/export/), and [`docker import`](https://docs.docker.com/engine/reference/commandline/import/). diff --git a/doc/user/free_user_limit.md b/doc/user/free_user_limit.md index 35777847947..4f569098d4d 100644 --- a/doc/user/free_user_limit.md +++ b/doc/user/free_user_limit.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Free user limit **(FREE SAAS)** -From October 19, 2022, a five-user limit will apply to top-level [namespaces](namespace/index.md) with private visibility on GitLab SaaS. These limits will roll out gradually, and impacted users will be notified in GitLab.com at least 60 days before the limit is applied. +A five-user limit applies to top-level [namespaces](namespace/index.md) with private visibility on GitLab SaaS. This limit is being rolled out gradually, and impacted users will be notified in GitLab.com at least 60 days before the limit is applied. When the five-user limit is applied, top-level private namespaces exceeding the user limit are placed in a read-only state. These namespaces cannot write new data to repositories, Git Large File Storage (LFS), packages, or registries. diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index d3d50ee1a8f..9bb1c4e968c 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -189,7 +189,7 @@ the default value [is the same as for self-managed instances](../admin_area/sett |-------------------------------|--------------------| | [Repository size including LFS](../admin_area/settings/account_and_limit_settings.md#repository-size-limit) | 10 GB | | [Maximum import size](../project/settings/import_export.md#maximum-import-file-size) | 5 GB | -| Maximum attachment size | 10 MB | +| Maximum attachment size | 100 MB | If you are near or over the repository size limit, you can either [reduce your repository size with Git](../project/repository/reducing_the_repo_size_using_git.md) @@ -314,7 +314,7 @@ The list of GitLab.com specific settings (and their defaults) is as follows: | `max_wal_senders` | 32 | 0 | | `max_wal_size` | 5GB | 1GB | | `shared_buffers` | 112896MB | Based on how much memory is available | -| `shared_preload_libraries` | pg_stat_statements | empty | +| `shared_preload_libraries` | `pg_stat_statements` | empty | | `shmall` | 30146560 | Based on the server's capabilities | | `shmmax` | 123480309760 | Based on the server's capabilities | | `wal_buffers` | 16MB | -1 | @@ -445,11 +445,18 @@ If more than the maximum number of allowed connections occur concurrently, they are dropped and users get [an `ssh_exchange_identification` error](../../topics/git/troubleshooting_git.md#ssh_exchange_identification-error). -### Import/export +### Group and project import by uploading export files -To help avoid abuse, project and group imports, exports, and export downloads -are rate limited. See [Project import/export rate limits](../../user/project/settings/import_export.md#rate-limits) and [Group import/export rate limits](../../user/group/settings/import_export.md#rate-limits) -for details. +To help avoid abuse, the following are rate limited: + +- Project and group imports. +- Group and project exports that use files. +- Export downloads. + +For more information, see: + +- [Project import/export rate limits](../../user/project/settings/import_export.md#rate-limits). +- [Group import/export rate limits](../../user/group/import/index.md#rate-limits). ### Non-configurable limits diff --git a/doc/user/group/access_and_permissions.md b/doc/user/group/access_and_permissions.md index 13a1fd31ee4..a7358db54df 100644 --- a/doc/user/group/access_and_permissions.md +++ b/doc/user/group/access_and_permissions.md @@ -228,7 +228,10 @@ projects in a group, allowing tighter control over project membership. For example, if you want to lock the group for an [Audit Event](../../administration/audit_events.md), you can guarantee that project membership cannot be modified during the audit. -You can still invite groups or to add members to groups, implicitly giving members access to projects in the **locked** group. +If group membership lock is enabled, the group owner can still: + +- Invite groups or add members to groups to give them access to projects in the **locked** group. +- Change the role of group members. The setting does not cascade. Projects in subgroups observe the subgroup configuration, ignoring the parent group. @@ -239,8 +242,10 @@ To prevent members from being added to projects in a group: 1. Under **Membership**, select **Users cannot be added to projects in this group**. 1. Select **Save changes**. -All users who previously had permissions can no longer add members to a group. -API requests to add a new user to a project are not possible. +After you lock the membership for a group: + +- All users who previously had permissions can no longer add members to a group. +- API requests to add a new user to a project are not possible. ## Manage group memberships via LDAP **(PREMIUM SELF)** diff --git a/doc/user/group/compliance_frameworks.md b/doc/user/group/compliance_frameworks.md index 7bd545003db..0e976cec866 100644 --- a/doc/user/group/compliance_frameworks.md +++ b/doc/user/group/compliance_frameworks.md @@ -21,7 +21,7 @@ Group owners can create, edit, and delete compliance frameworks: 1. Expand the **Compliance frameworks** section. 1. Create, edit, or delete compliance frameworks. -## Set a default compliance framework +## Default compliance frameworks > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/375036) in GitLab 15.6. @@ -29,6 +29,20 @@ Group owners can set a default compliance framework. The default framework is ap that are created within that group. It does not affect the framework applied to the existing projects. The default framework cannot be deleted. +A compliance framework that is set to default has a **default** label. + +### Set and remove as default + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/375038) in GitLab 15.7. + +Group owners can set a compliance framework as default (or remove the setting): + +1. On the top bar, select **Main menu > Groups > View all groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Compliance frameworks** section and locate the compliance framework to set (or remove) as default. +1. Select the vertical ellipsis (**{ellipsis_v}**) for the compliance frame and then select **Set default** (or + **Remove default**). + ### Example GraphQL mutations for setting a default compliance framework Creating a new compliance framework and setting it as the default framework for the group. @@ -191,7 +205,7 @@ When creating such an MR against a project with CF pipelines, the above snippet This is because in the context of the target project, `$CI_COMMIT_REF_NAME` evaluates to a non-existing branch name. To get the correct context, use `$CI_MERGE_REQUEST_SOURCE_PROJECT_PATH` instead of `$CI_PROJECT_PATH`. -This variable is only availabe in +This variable is only available in [merge request pipelines](../../ci/pipelines/merge_request_pipelines.md). For example, for a configuration that supports both merge request pipelines originating in project forks and branch pipelines, diff --git a/doc/user/group/contribution_analytics/index.md b/doc/user/group/contribution_analytics/index.md index b1efd2e9251..ddf468e39b0 100644 --- a/doc/user/group/contribution_analytics/index.md +++ b/doc/user/group/contribution_analytics/index.md @@ -24,8 +24,7 @@ To view Contribution Analytics: ## Using Contribution Analytics -There are three main bar graphs that illustrate the number of contributions per group -member for the following: +Three bar graphs illustrate the number of contributions made by each group member: - Push events - Merge requests diff --git a/doc/user/group/devops_adoption/index.md b/doc/user/group/devops_adoption/index.md index 67263f15f06..a81b61c50ce 100644 --- a/doc/user/group/devops_adoption/index.md +++ b/doc/user/group/devops_adoption/index.md @@ -12,7 +12,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - DAST and SAST metrics [added](https://gitlab.com/gitlab-org/gitlab/-/issues/328033) in GitLab 14.1. > - Fuzz Testing metrics [added](https://gitlab.com/gitlab-org/gitlab/-/issues/330398) in GitLab 14.2. > - Dependency Scanning metrics [added](https://gitlab.com/gitlab-org/gitlab/-/issues/328034) in GitLab 14.2. -> - Multiselect [added](https://gitlab.com/gitlab-org/gitlab/-/issues/333586) in GitLab 14.2. +> - Multi-select [added](https://gitlab.com/gitlab-org/gitlab/-/issues/333586) in GitLab 14.2. > - Overview table [added](https://gitlab.com/gitlab-org/gitlab/-/issues/335638) in GitLab 14.3. > - Adoption over time chart [added](https://gitlab.com/gitlab-org/gitlab/-/issues/337561) in GitLab 14.4. diff --git a/doc/user/group/epics/manage_epics.md b/doc/user/group/epics/manage_epics.md index 61aa5c5fe02..8e7b6fd82ad 100644 --- a/doc/user/group/epics/manage_epics.md +++ b/doc/user/group/epics/manage_epics.md @@ -494,7 +494,8 @@ The maximum number of direct child epics is 100. ### Child epics from other groups -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8502) in GitLab 15.6 [with a flag](../../../administration/feature_flags.md) named `child_epics_from_different_hierarchies`. Disabled by default. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8502) in GitLab 15.6 [with a flag](../../../administration/feature_flags.md) named `child_epics_from_different_hierarchies`. Disabled by default. +> - Minimum required role for the group [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/382503) from Reporter to Guest in GitLab 15.7. FLAG: On self-managed GitLab, by default this feature is not available. To make it available per group, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `child_epics_from_different_hierarchies`. @@ -504,16 +505,18 @@ You can add a child epic that belongs to a group that is different from the pare Prerequisites: -- You must have at least the Reporter role for both the child and parent epics' groups. +- You must have at least the Guest role for both the child and parent epics' groups. - Multi-level child epics must be available for both the child and parent epics' groups. To add a child epic from another group, paste the epic's URL when [adding an existing epic](#add-a-child-epic-to-an-epic). ### Add a child epic to an epic +> Minimum required role for the group [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/382503) from Reporter to Guest in GitLab 15.7. + Prerequisites: -- You must have at least the Reporter role for the parent epic's group. +- You must have at least the Guest role for the parent epic's group. To add a new epic as child epic: @@ -534,7 +537,8 @@ To add an existing epic as child epic: ### Move child epics between epics -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33039) in GitLab 13.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33039) in GitLab 13.0. +> - Minimum required role for the group [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/382503) from Reporter to Guest in GitLab 15.7. New child epics appear at the top of the list in the **Epics and Issues** tab. You can move child epics from one epic to another. @@ -543,7 +547,7 @@ Issues and child epics cannot be intermingled. Prerequisites: -- You must have at least the Reporter role for the parent epic's group. +- You must have at least the Guest role for the parent epic's group. To move child epics to another epic: @@ -552,14 +556,15 @@ To move child epics to another epic: ### Reorder child epics assigned to an epic -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9367) in GitLab 12.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9367) in GitLab 12.5. +> - Minimum required role for the group [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/382503) from Reporter to Guest in GitLab 15.7. New child epics appear at the top of the list in the **Epics and Issues** tab. You can reorder the list of child epics. Prerequisites: -- You must have at least the Reporter role for the parent epic's group. +- You must have at least the Guest role for the parent epic's group. To reorder child epics assigned to an epic: @@ -568,9 +573,11 @@ To reorder child epics assigned to an epic: ### Remove a child epic from a parent epic +> Minimum required role for the group [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/382503) from Reporter to Guest in GitLab 15.7. + Prerequisites: -- You must have at least the Reporter role for the parent epic's group. +- You must have at least the Guest role for the parent epic's group. To remove a child epic from a parent epic: diff --git a/doc/user/group/import/index.md b/doc/user/group/import/index.md index 49515dd8a11..9a671ff6679 100644 --- a/doc/user/group/import/index.md +++ b/doc/user/group/import/index.md @@ -4,7 +4,23 @@ group: Import info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Migrating groups with GitLab Migration **(FREE)** +# Migrating GitLab groups **(FREE)** + +You can migrate GitLab groups: + +- From self-managed GitLab to GitLab.com. +- From GitLab.com to self-managed GitLab. +- From one self-managed GitLab instance to another. +- Between groups in the same GitLab instance. + +You can migrate groups in two ways: + +- By direct transfer (recommended). +- By uploading an export file. + +If you migrate from GitLab.com to self-managed GitLab, an administrator can create users on the self-managed GitLab instance. + +## Migrate groups by direct transfer (recommended) > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/249160) in GitLab 13.7 for group resources [with a flag](../../feature_flags.md) named `bulk_import`. Disabled by default. > - Group items [enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/338985) in GitLab 14.3. @@ -18,78 +34,103 @@ On self-managed GitLab, by default [migrating project items](#migrated-project-i this feature, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `bulk_import_projects`. On GitLab.com, migration of both groups and projects is available. -Users with the Owner role on a top-level group can use GitLab Migration to migrate the group to: +Prerequisites: + +- Network connection between instances or GitLab.com. Must support HTTPS. +- Owner role on the top-level group to migrate. + +You can import top-level groups to: - Another top-level group. - The subgroup of any existing top-level group. - Another GitLab instance, including GitLab.com. -Migrating groups using GitLab Migration is not the same as [migrating groups using file exports](../settings/import_export.md). -Importing and exporting groups using file exports requires you to export a group to a file and then import that file in -another GitLab instance. Migrating groups using GitLab Migration automates this step. +You can migrate: + +- By direct transfer using either the UI or the [API](../../../api/bulk_imports.md). +- Many groups at once. -## Import your groups into GitLab +When migrating a top-level group to GitLab.com, all its subgroups and projects are migrated too. -When you migrate a group, you connect to your GitLab instance and then choose -groups to import. Not all the data is migrated. See: +Not all group and project resources are imported. See list of migrated resources below: - [Migrated group items](#migrated-group-items). - [Migrated project items](#migrated-project-items). -Leave feedback about group migration in [the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/284495). +### Preparation + +GitLab maps users and their contributions correctly provided: + +- Users already exist on the target GitLab instance. +- Users have a public email on the source GitLab instance that matches their primary email on the target GitLab instance. +- Users' primary email addresses on the target GitLab instance are confirmed. Most users receives an email asking them to confirm their email address. +- When using an OmniAuth provider like SAML, GitLab and SAML accounts of users on GitLab must be linked. All users on the target GitLab instance must sign in + and verify their account on the target GitLab instance. -NOTE: You might need to reconfigure your firewall to prevent blocking the connection on the self-managed instance. -### Connect to the remote GitLab instance +If you use [SAML SSO for GitLab.com groups](../../group/saml_sso/index.md), +contributing users must have [linked their SAML identity to their GitLab.com account](../../group/saml_sso/index.md#linking-saml-to-your-existing-gitlabcom-account). -Before you begin, ensure that the target GitLab instance can communicate with the source over HTTPS -(HTTP is not supported). You might need to reconfigure your firewall to prevent blocking the connection on the self-managed -instance. +When migrating to GitLab.com, you must create users manually unless [SCIM](../../group/saml_sso/scim_setup.md) is used. Creating users with the API is only +available to self-managed instances because it requires administrator access. -Then create the group you want to import into, and connect: +### Connect to the source GitLab instance -1. Create a new group or subgroup: +Create the group you want to import to and connect the source: - - On the top bar, select `+` and then **New group**. - - Or, on an existing group's page, in the top right, select **New subgroup**. +1. Create either: -1. Select **Import group**. -1. Enter the source URL of your GitLab instance. + - A new group. On the top bar, select **{plus-square}**, then **New group**, and select **Import group**. + - A new subgroup. On existing group's page, either: + - Select **New subgroup**. + - On the top bar, Select **{plus-square}** and then **New subgroup**. Then on the left sidebar, select the **import an existing group** link. +1. Enter the URL of your source GitLab instance. 1. Generate or copy a [personal access token](../../../user/profile/personal_access_tokens.md) - with the `api` and `read_repository` scopes on your remote GitLab instance. -1. Enter the [personal access token](../../../user/profile/personal_access_tokens.md) for your remote GitLab instance. + with the `api` scope on your source GitLab instance. Both `api` and `read_repository` scopes are required when migrating from GitLab 15.0 and earlier. +1. Enter the [personal access token](../../../user/profile/personal_access_tokens.md) for your source GitLab instance. 1. Select **Connect instance**. ### Select the groups to import -After you have authorized access to the GitLab instance, you are redirected to the GitLab Group -Migration importer page. The remote groups you have the Owner role for are listed. +After you have authorized access to the source GitLab instance, you are redirected to the GitLab group +importer page. The top-level groups on the connected source instance you have the Owner role for are listed. -1. By default, the proposed group namespaces match the names as they exist in remote instance, but based on your permissions, you can choose to edit these names before you proceed to import any of them. +1. By default, the proposed group namespaces match the names as they exist in source instance, but based on your permissions, you can choose to edit these names before you proceed to import any of them. 1. Next to the groups you want to import, select **Import**. 1. The **Status** column shows the import status of each group. If you leave the page open, it updates in real-time. 1. After a group has been imported, select its GitLab path to open its GitLab URL.  -## Automate group and project import **(PREMIUM)** +### Group import history -For information on automating user, group, and project import API calls, see -[Automate group and project import](../../project/import/index.md#automate-group-and-project-import). +You can view all groups migrated by you by direct transfer listed on the group import history page. This list includes: + +- Paths of source groups. +- Paths of destination groups. +- Start date of each import. +- Status of each import. +- Error details if any errors occurred. -## Migrated group items +To view group import history: + +1. Sign in to GitLab. +1. On the top bar, select **Create new…** (**{plus-square}**). +1. Select **New group**. +1. Select **Import group**. +1. Select **History** in the upper right corner. +1. If there are any errors for a particular import, you can see them by selecting **Details**. + +### Migrated group items The [`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/group/import_export.yml) -file for groups lists many of the items migrated when migrating groups using group migration. View this file in the branch +file for groups lists many of the items imported when migrating groups by direct transfer. View this file in the branch for your version of GitLab to see the list of items relevant to you. For example, [`import_export.yml` on the `14-10-stable-ee` branch](https://gitlab.com/gitlab-org/gitlab/-/blob/14-10-stable-ee/lib/gitlab/import_export/group/import_export.yml). -Migrating projects with file exports uses the same export and import mechanisms as creating projects from templates at the [group](../custom_project_templates.md) and -[instance](../../admin_area/custom_project_templates.md) levels. Therefore, the list of exported items is the same. - -Items that are migrated to the target instance include: +Group items that are migrated to the target instance include: - Badges ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292431) in 13.11) - Board Lists @@ -114,24 +155,24 @@ Items that are migrated to the target instance include: Any other items are **not** migrated. -## Migrated project items +### Migrated project items > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4 [with a flag](../../feature_flags.md) named `bulk_import_projects`. Disabled by default. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/339941) in GitLab 15.6. FLAG: -On self-managed GitLab, migrating project resources are not available by default. To make them available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `bulk_import_projects`. On GitLab.com, migration of project resources is available. +On self-managed GitLab, migrating project resources when migrating groups is not available by default. To make it available ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `bulk_import_projects`. On GitLab.com, groups are migrated with all their projects by default. The [`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/project/import_export.yml) -file for projects lists many of the items migrated when migrating projects using group migration. View this file in the branch +file for projects lists many of the items imported when migrating projects using group migration. View this file in the branch for your version of GitLab to see the list of items relevant to you. For example, [`import_export.yml` on the `14-10-stable-ee` branch](https://gitlab.com/gitlab-org/gitlab/-/blob/14-10-stable-ee/lib/gitlab/import_export/project/import_export.yml). -Migrating projects with file exports uses the same export and import mechanisms as creating projects from templates at the [group](../../group/custom_project_templates.md) and -[instance](../../admin_area/custom_project_templates.md) levels. Therefore, the list of exported items is the same. +Project items that are migrated to the target instance include: - Projects ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4) - Auto DevOps ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339410) in GitLab 14.6) + - Badges ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75029) in GitLab 14.6) - Branches (including protected branches) ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339414) in GitLab 14.7) - CI Pipelines ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339407) in GitLab 14.6) - Designs ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339421) in GitLab 15.1) @@ -141,6 +182,8 @@ Migrating projects with file exports uses the same export and import mechanisms - Issue resource milestone events ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) in GitLab 15.4) - Issue resource iteration events ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) in GitLab 15.4) - Merge request URL references ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267947) in GitLab 15.6) + - Time Tracking ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267946) in GitLab 14.4) + - Issue boards ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71661) in GitLab 14.4) - Labels ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339419) in GitLab 14.4) - LFS Objects ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339405) in GitLab 14.8) - Members ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/341886) in GitLab 14.8) @@ -151,18 +194,33 @@ Migrating projects with file exports uses the same export and import mechanisms - Merge request resource state events ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) in GitLab 15.4) - Merge request resource milestone events ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) in GitLab 15.4) - Issue URL references ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267947) in GitLab 15.6) - - Migrate Push Rules ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339403) in GitLab 14.6) - - Pull Requests (including external pull requests) ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339409) in GitLab 14.5) + - Time Tracking ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339403) in GitLab 14.5) + - Push Rules ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339403) in GitLab 14.6) + - Milestones ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339417) in GitLab 14.5) + - External Pull Requests ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339409) in GitLab 14.5) - Pipeline History ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339412) in GitLab 14.6) - Pipeline Schedules ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339408) in GitLab 14.8) + - Project Features ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74722) in GitLab 14.6) - Releases ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339422) in GitLab 15.1) - Release Evidences ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360567) in GitLab 15.1) - Repositories ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4) - Snippets ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343438) in GitLab 14.6) + - Settings ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339416) in GitLab 14.6) + - Avatar ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75249) in GitLab 14.6) + - Container Expiration Policy ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75653) in GitLab 14.6) + - Project Properties ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75898) in GitLab 14.6) + - Service Desk ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75653) in GitLab 14.6) - Uploads ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339401) in GitLab 14.5) - Wikis ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345923) in GitLab 14.6) -## Troubleshooting Group Migration +Items excluded from migration, because they contain sensitive information: + +- Pipeline Triggers. + +Migrating projects with file exports uses the same export and import mechanisms as creating projects from templates at the [group](../../group/custom_project_templates.md) and +[instance](../../admin_area/custom_project_templates.md) levels. Therefore, the list of exported items is the same. + +### Troubleshooting In a [rails console session](../../../administration/operations/rails_console.md#starting-a-rails-console-session), you can find the failure or error messages for the group import attempt using: @@ -184,7 +242,10 @@ entities.map(&:failures).flatten entities.where(status: [-1]).pluck(:destination_name, :destination_namespace, :status) ``` -### Stale imports +You can also see all migrated entities with any failures related to them using an +[API endpoint](../../../api/bulk_imports.md#list-all-gitlab-migrations-entities). + +#### Stale imports > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352985) in GitLab 14.10. @@ -199,10 +260,12 @@ import = BulkImports::Entity.where(namespace_id: Group.id).map(&:bulk_import) import.status #=> 3 means that the import timed out. ``` -### Error: `404 Group Not Found` +#### Error: `404 Group Not Found` + +If you attempt to import a group that has a path comprised of only numbers (for example, `5000`), GitLab attempts to +find the group by ID instead of the path. This causes a `404 Group Not Found` error in GitLab 15.4 and earlier. -If you attempt to import a group that has a path comprised of only numbers (for example, `5000`), GitLab attempts to find the group by ID instead of the -path. This causes a `404 Group Not Found` error. To solve this, the source group path must be changed to include a non-numerical character using either: +To solve this, you must change the source group path to include a non-numerical character using either: - The GitLab UI: @@ -212,3 +275,170 @@ path. This causes a `404 Group Not Found` error. To solve this, the source group 1. Under **Change group URL**, change the group URL to include non-numeric characters. - The [Groups API](../../../api/groups.md#update-group). + +### Provide feedback + +Please leave your feedback about migrating groups by direct transfer in +[the feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/284495). + +## Migrate groups by uploading an export file (deprecated) + +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2888) in GitLab 13.0 as an experimental feature. May change in future releases. +> - [Deprecated](https://gitlab.com/groups/gitlab-org/-/epics/4619) in GitLab 14.6. + +WARNING: +This feature was [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/4619) in GitLab 14.6 and replaced by +[migrating groups by direct transfer](#migrate-groups-by-direct-transfer-recommended). To follow progress on a solution for +[offline environments](../../application_security/offline_deployments/index.md), see +[the relevant epic](https://gitlab.com/groups/gitlab-org/-/epics/8985). + +Prerequisites: + +- Owner role on the group to migrate. + +Using file exports, you can: + +- Export any group to a file and upload that file to another GitLab instance or to another location on the same instance. +- Use either the GitLab UI or the [API](../../../api/group_import_export.md). +- Migrate groups one by one, then export and import each project for the groups one by one. + +GitLab maps user contributions correctly when an admin access token is used to perform the import. GitLab does not map +user contributions correctly when you are importing from a self-managed instance to GitLab.com. Correct mapping of user +contributions when importing from a self-managed instance to GitLab.com can be preserved with paid involvement of +Professional Services team. + +Note the following: + +- Exports are stored in a temporary directory and are deleted every 24 hours by a specific worker. +- To preserve group-level relationships from imported projects, export and import groups first so that projects can + be imported into the desired group structure. +- Imported groups are given a `private` visibility level, unless imported into a parent group. +- If imported into a parent group, a subgroup inherits the same level of visibility unless otherwise restricted. +- You can export groups from the [Community Edition to the Enterprise Edition](https://about.gitlab.com/install/ce-or-ee/) + and vice versa. The Enterprise Edition retains some group data that isn't part of the Community Edition. If you're + exporting a group from the Enterprise Edition to the Community Edition, you may lose this data. For more information, + see [downgrading from EE to CE](../../../index.md). + +### Exported contents + +The [`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/group/import_export.yml) +file for groups lists items exported and imported when migrating groups using file exports. View this file in the branch +for your version of GitLab to see the list of items relevant to you. For example, +[`import_export.yml` on the `14-10-stable-ee` branch](https://gitlab.com/gitlab-org/gitlab/-/blob/14-10-stable-ee/lib/gitlab/import_export/group/import_export.yml). + +Group items that are exported include: + +- Milestones +- Labels +- Boards and Board Lists +- Badges +- Subgroups (including all the aforementioned data) +- Epics + - Epic resource state events ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) in GitLab 15.4) +- Events +- [Wikis](../../project/wiki/group.md) + ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53247) in GitLab 13.9) +- Iterations cadences ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95372) in 15.4) + +Items that are **not** exported include: + +- Projects +- Runner tokens +- SAML discovery tokens + +### Preparation + +- To preserve the member list and their respective permissions on imported groups, review the users in these groups. Make +sure these users exist before importing the desired groups. +- Users must set a public email in the source GitLab instance that matches one of their verified emails in the target GitLab instance. + +### Enable export for a group + +Prerequisite: + +- You must have the Owner role for the group. + +To enable import and export for a group: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand **Visibility and access controls**. +1. In the **Import sources** section, select the checkboxes for the sources you want. + +### Export a group + +Prerequisites: + +- You must have the Owner role for the group. + +To export the contents of a group: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. In the **Advanced** section, select **Export Group**. +1. After the export is generated, you should receive an email with a link to the [exported contents](#exported-contents) + in a compressed tar archive, with contents in NDJSON format. +1. Alternatively, you can download the export from the UI: + + 1. Return to your group's **Settings > General** page. + 1. In the **Advanced** section, select **Download export**. + You can also generate a new file by selecting **Regenerate export**. + +You can also export a group [using the API](../../../api/group_import_export.md). + +### Import the group + +1. Create a new group: + - On the top bar, select **Create new…** (**{plus-square}**) and then **New group**. + - On an existing group's page, select the **New subgroup** button. +1. Select **Import group**. +1. Enter your group name. +1. Accept or modify the associated group URL. +1. Select **Choose file**. +1. Select the file that you exported in the [Export a group](#export-a-group) section. +1. To begin importing, select **Import group**. + +Your newly imported group page appears after the operation completes. + +NOTE: +The maximum import file size can be set by the administrator, default is `0` (unlimited). +As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the +[Application settings API](../../../api/settings.md#change-application-settings) or the +[Admin Area](../../admin_area/settings/account_and_limit_settings.md). +Default [modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50MB to 0 in GitLab 13.8. + +### Rate limits + +To help avoid abuse, by default, users are rate limited to: + +| Request Type | Limit | +| ---------------- | ---------------------------------------- | +| Export | 6 groups per minute | +| Download export | 1 download per group per minute | +| Import | 6 groups per minute | + +### Version history + +#### 14.0+ + +In GitLab 14.0, the JSON format is no longer supported for project and group exports. To allow for a +transitional period, you can still import any JSON exports. The new format for imports and exports +is NDJSON. + +#### 13.0+ + +GitLab can import bundles that were exported from a different GitLab deployment. +This ability is limited to two previous GitLab [minor](../../../policy/maintenance.md#versioning) +releases, which is similar to our process for [Security Releases](../../../policy/maintenance.md#security-releases). + +For example: + +| Current version | Can import bundles exported from | +|-----------------|----------------------------------| +| 13.0 | 13.0, 12.10, 12.9 | +| 13.1 | 13.1, 13.0, 12.10 | + +## Automate group and project import **(PREMIUM)** + +For information on automating user, group, and project import API calls, see +[Automate group and project import](../../project/import/index.md#automate-group-and-project-import). diff --git a/doc/user/group/manage.md b/doc/user/group/manage.md index a63e6c6dd7f..414b80d0f1d 100644 --- a/doc/user/group/manage.md +++ b/doc/user/group/manage.md @@ -78,7 +78,7 @@ If you don't want to wait, you can remove a group immediately. Prerequisites: -- You must have at least the Owner role for a group. +- You must have the Owner role for a group. - You have [marked the group for deletion](#remove-a-group). To immediately remove a group marked for deletion: @@ -172,7 +172,7 @@ Prerequisite: 1. On the left sidebar, select **Group information > Members**. 1. Select **Invite members**. 1. Fill in the fields. - - The role applies to all projects in the group. [Learn more about permissions](../permissions.md). + - The role applies to all projects in the group. For more information, see [permissions](../permissions.md). - On the **Access expiration date**, the user can no longer access projects in the group. 1. Select **Invite**. @@ -530,10 +530,10 @@ in a subgroup has access to the templates for that subgroup and any immediate parent groups. To learn how to create templates for issues and merge requests, see -[Description templates](../project/description_templates.md). +[description templates](../project/description_templates.md). Define project templates at a group level by setting a group as the template source. -[Learn more about group-level project templates](custom_project_templates.md). +For more information, see [group-level project templates](custom_project_templates.md). ### Enable group file template **(PREMIUM)** diff --git a/doc/user/group/saml_sso/group_sync.md b/doc/user/group/saml_sso/group_sync.md index 001c73b6979..80d145fc6bb 100644 --- a/doc/user/group/saml_sso/group_sync.md +++ b/doc/user/group/saml_sso/group_sync.md @@ -10,9 +10,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363084) for self-managed instances in GitLab 15.1. WARNING: -Changing Group Sync configuration can remove users from the mapped GitLab group. +Adding or changing Group Sync configuration can remove users from the mapped GitLab group. Removal happens if there is any mismatch between the group names and the list of `groups` in the SAML response. -If changes must be made, ensure either the SAML response includes the `groups` attribute +Before making changes, ensure either the SAML response includes the `groups` attribute and the `AttributeValue` value matches the **SAML Group Name** in GitLab, or that all groups are removed from GitLab to disable Group Sync. @@ -21,17 +21,29 @@ For a demo of Group Sync using Azure, see [Demo: SAML Group Sync](https://youtu. ## Configure SAML Group Sync +NOTE: +You must include the SAML configuration block on all Sidekiq nodes in addition to Rails application nodes if you: + +- Use SAML Group Sync. +- Have multiple GitLab nodes, for example in a distributed or highly available architecture. + +WARNING: +To prevent users being accidentally removed from the GitLab group, follow these instructions closely before +enabling Group Sync in GitLab. + To configure SAML Group Sync: -- For GitLab self-managed: - 1. Configure the [SAML OmniAuth Provider](../../../integration/saml.md). - 1. Ensure your SAML identity provider sends an attribute statement with the same name as the value of the `groups_attribute` setting. -- For GitLab.com: - 1. See [SAML SSO for GitLab.com groups](index.md). - 1. Ensure your SAML identity provider sends an attribute statement named `Groups` or `groups`. +1. Configure the identity Provider: + - For self-managed GitLab, see the [SAML OmniAuth Provider documentation](../../../integration/saml.md). + - For GitLab.com, see the [SAML SSO for GitLab.com groups documentation](index.md). + +1. Capture [a SAML response](troubleshooting.md#saml-debugging-tools) during the sign-in process to confirm your SAML identity provider sends an attribute statement: + - For self-managed GitLab, with the same name as the value of the `groups_attribute` setting. + - For GitLab.com, named `Groups` or `groups`. NOTE: -The value for `Groups` or `groups` in the SAML response can be either the group name or the group ID. +The value for `Groups` or `groups` in the SAML response may be either the group name or an ID. +For example, Azure AD sends the Azure Group Object ID instead of the name. Use the ID value when configuring [SAML Group Links](#configure-saml-group-links). ```xml <saml:AttributeStatement> @@ -55,7 +67,7 @@ a SAML identity provider group name to a GitLab role. This can be done for a top To link the SAML groups: -1. In **SAML Group Name**, enter the value of the relevant `saml:AttributeValue`. +1. In **SAML Group Name**, enter the value of the relevant `saml:AttributeValue`. The value entered here must exactly match the value sent in the SAML response. For some IdPs, this may be a group ID or object ID (Azure AD) instead of a friendly group name. 1. Choose the role in **Access Level**. 1. Select **Save**. 1. Repeat to add additional group links if required. @@ -177,4 +189,4 @@ Because of a [known issue with Azure AD](https://support.esri.com/en/technical-a in the user's SAML assertion. To work around this issue, allow more than 150 group IDs to be sent in SAML token using configuration steps in the -[Azure AD documentation](https://support.esri.com/en/technical-article/000022190). +[Azure AD documentation](https://support.esri.com/en/technical-article/000022190). diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index 1c5e7ff0115..bd10560e138 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -53,7 +53,6 @@ GitLab.com uses the SAML NameID to identify users. The NameID element: also case-insensitive, which can result in users being unable to sign in. The relevant field name and recommended value for supported providers are in the [provider specific notes](#providers). -appropriate corresponding field. WARNING: Once users have signed into GitLab using the SSO SAML setup, changing the `NameID` breaks the configuration and potentially locks users out of the GitLab group. @@ -72,7 +71,7 @@ must be specified as an attribute named `email` or `mail`. You can configure the following attributes with GitLab.com Group SAML: - `username` or `nickname`. We recommend you configure only one of these. -- The [attributes available](../../../integration/saml.md#assertions) to self-managed GitLab instances. +- The [attributes available](../../../integration/saml.md#configure-assertions) to self-managed GitLab instances. ### Metadata configuration @@ -98,7 +97,7 @@ After you set up your identity provider to work with GitLab, you must configure  NOTE: -The certificate [fingerprint algorithm](../../../integration/saml.md#notes-on-configuring-your-identity-provider) must be in SHA1. When configuring the identity provider (such as [Google Workspace](#google-workspace-setup-notes)), use a secure signature algorithm. +The certificate [fingerprint algorithm](../../../integration/saml.md#configure-saml-on-your-idp) must be in SHA1. When configuring the identity provider (such as [Google Workspace](#google-workspace-setup-notes)), use a secure signature algorithm. ### Additional configuration information @@ -124,7 +123,7 @@ It can also help to compare the XML response from your provider with our [exampl > - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/215155) in GitLab 15.5 [with a flag](../../../administration/feature_flags.md) named `transparent_sso_enforcement` to include transparent enforcement even when SSO enforcement is not enabled. Disabled on GitLab.com. FLAG: -On self-managed GitLab, transparent SSO enforcement is unavailable. On GitLab.com, transparent SSO enforcement is unavailable and can be configured by GitLab.com administrators only. +On self-managed GitLab, transparent SSO enforcement is unavailable. On GitLab.com, see the [Transparent SSO rollout](https://gitlab.com/gitlab-org/gitlab/-/issues/375788) issue for the current status. SSO is enforced when users access groups and projects in the organization's group hierarchy. Users can view other groups and projects without SSO sign in. @@ -178,13 +177,24 @@ When SSO is enforced, users are not immediately revoked. If the user: - Has an active session, they can continue accessing the group for up to 24 hours until the identity provider session times out. +### Selectively enable and disable transparent SSO enforcement + +There are two feature flags associated with this feature to allow precise control. If a customer has a problem with transparent SSO on GitLab.com, GitLab can help troubleshoot and override the feature flag as necessary. + +**`transparent_sso_enforcement`:** This feature flag should only be enabled or disabled by the Authentication and Authorization group +or in the case of a serious and widespread issue affecting many groups or users. See [issue 375788](https://gitlab.com/gitlab-org/gitlab/-/issues/375788) for the current GitLab.com rollout status. + +**`transparent_sso_enforcement_override`:** When the `transparent_sso_enforcement` feature flag is enabled, support or production teams can +turn off transparent SSO by enabling this feature flag for a specific customer group. **Enabling** this feature flag +disables transparent SSO enforcement. + ## Providers The SAML standard means that you can use a wide range of identity providers with GitLab. Your identity provider might have relevant documentation. It can be generic SAML documentation or specifically targeted for GitLab. When [configuring your identity provider](#configure-your-identity-provider), consider the notes below for specific providers to help avoid common issues and as a guide for terminology used. -For providers not listed below, you can refer to the [instance SAML notes on configuring an identity provider](../../../integration/saml.md#notes-on-configuring-your-identity-provider) +For providers not listed below, you can refer to the [instance SAML notes on configuring an identity provider](../../../integration/saml.md#configure-saml-on-your-idp) for additional guidance on information your identity provider may require. GitLab provides the following information for guidance only. @@ -338,10 +348,14 @@ When a user tries to sign in with Group SSO, GitLab attempts to find or create a ### Linking SAML to your existing GitLab.com account +> **Remember me** checkbox [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/121569) in GitLab 15.7. + To link SAML to your existing GitLab.com account: 1. Sign in to your GitLab.com account. [Reset your password](https://gitlab.com/users/password/new) if necessary. 1. Locate and visit the **GitLab single sign-on URL** for the group you're signing in to. A group owner can find this on the group's **Settings > SAML SSO** page. If the sign-in URL is configured, users can connect to the GitLab app from the identity provider. +1. Optional. Select the **Remember me** checkbox to stay signed in to GitLab for 2 weeks. You may still be asked to re-authenticate with your SAML provider + more frequently. 1. Select **Authorize**. 1. Enter your credentials on the identity provider if prompted. 1. You are then redirected back to GitLab.com and should now have access to the group. In the future, you can use SAML to sign in to GitLab.com. diff --git a/doc/user/group/saml_sso/troubleshooting.md b/doc/user/group/saml_sso/troubleshooting.md index 7dafd2c5075..f8075e62ecc 100644 --- a/doc/user/group/saml_sso/troubleshooting.md +++ b/doc/user/group/saml_sso/troubleshooting.md @@ -52,7 +52,7 @@ You can use one of the following to troubleshoot SAML: - A [quick start guide to start a Docker container](../../../administration/troubleshooting/test_environments.md#saml) with a plug and play SAML 2.0 identity provider if you only require a SAML provider. - A local environment by - [enabling SAML for groups on a self-managed instance](../../../integration/saml.md#configuring-group-saml-on-a-self-managed-gitlab-instance). + [enabling SAML for groups on a self-managed instance](../../../integration/saml.md#group-saml-on-a-self-managed-gitlab-instance). ## Verify configuration @@ -96,7 +96,14 @@ In these cases, use one of the [SAML debugging tools](#saml-debugging-tools), or a group owner can get a copy of the SAML response from when they select the "Verify SAML Configuration" button on the group SSO Settings page. -Use a base64 decoder to see a human-readable version of the SAML response. +Use a base64 decoder to see a human-readable version of the SAML response. To avoid pasting the SAML response online to decode it, you can use your +browser's console in the developers tools: + +```javascript +atob(decodeURI("<paste_SAML_response_here>")) +``` + +You should get the SAML response in XML format as output. ## Configuration errors @@ -138,7 +145,7 @@ Make sure this information is provided. Another issue that can result in this error is when the correct information is being sent by the identity provider, but the attributes don't match the names in the OmniAuth `info` hash. In this case, you must set `attribute_statements` in the SAML configuration to -[map the attribute names in your SAML Response to the corresponding OmniAuth `info` hash names](../../../integration/saml.md#attribute_statements). +[map the attribute names in your SAML Response to the corresponding OmniAuth `info` hash names](../../../integration/saml.md#map-saml-response-attribute-names). ## User sign in banner error messages @@ -221,7 +228,7 @@ If all users are receiving a `404` when attempting to sign in using SAML, confir [there is an active subscription](../../../subscriptions/gitlab_com/index.md#view-your-gitlab-saas-subscription) being used in this SAML SSO namespace. If you receive a `404` during setup when using "verify configuration", make sure you have used the correct -[SHA-1 generated fingerprint](../../../integration/saml.md#notes-on-configuring-your-identity-provider). +[SHA-1 generated fingerprint](../../../integration/saml.md#configure-saml-on-your-idp). If a user is trying to sign in for the first time and the GitLab single sign-on URL has not [been configured](index.md#configure-your-identity-provider), they may see a 404. As outlined in the [user access section](index.md#linking-saml-to-your-existing-gitlabcom-account), a group Owner needs to provide the URL to users. @@ -262,3 +269,15 @@ Pay particular attention to the following 403 errors: - `app_not_configured` - `app_not_configured_for_user` + +## SAML Name ID and email address do not match your user account **(PREMIUM SAAS)** + +If users encounter the error `SAML Name ID and email address do not match your user account. Contact an administrator.` +this means: + +- The NameID value sent by SAML does not match the existing SAML identity `extern_uid` value. +- Either the SAML response did not include an email address or the email address did not match the user's GitLab email address. + +A GitLab group Owner can use the [SAML API](../../../api/saml.md) to update the user's SAML `extern_uid`. +The `extern_uid` value must match the Name ID value sent by the SAML identity provider (IdP). Depending on the IdP configuration +this may be a generated unique ID, an email address, or other value. diff --git a/doc/user/group/settings/group_access_tokens.md b/doc/user/group/settings/group_access_tokens.md index 158e1654c6e..6e0caa633eb 100644 --- a/doc/user/group/settings/group_access_tokens.md +++ b/doc/user/group/settings/group_access_tokens.md @@ -68,6 +68,11 @@ To create a group access token: A group access token is displayed. Save the group access token somewhere safe. After you leave or refresh the page, you can't view it again. +WARNING: +Group access tokens are treated as [internal users](../../../development/internal_users.md). +If an internal user creates a group access token, that token is able to access all +groups that have visibility level set to [Internal](../../public_access.md). + ## Create a group access token using Rails console GitLab 14.6 and earlier doesn't support creating group access tokens using the UI diff --git a/doc/user/group/settings/import_export.md b/doc/user/group/settings/import_export.md index cec17688902..ff64a7dcd54 100644 --- a/doc/user/group/settings/import_export.md +++ b/doc/user/group/settings/import_export.md @@ -1,164 +1,11 @@ --- -stage: Manage -group: Import -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../import/index.md' +remove_date: '2023-03-08' --- -# Migrating groups using file exports (deprecated) **(FREE)** +This document was moved to [another location](../import/index.md). -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2888) in GitLab 13.0 as an experimental feature. May change in future releases. -> - [Deprecated](https://gitlab.com/groups/gitlab-org/-/epics/4619) in GitLab 14.6. - -WARNING: -This feature was [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/4619) in GitLab 14.6 and replaced by -[a different migration method](../import/index.md). To follow progress on a solution for -[offline environments](../../application_security/offline_deployments/index.md), see -[the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/363406). - -You can export groups, with all their related data, from one GitLab instance to another. You can also: - -- [Migrate groups](../import/index.md) using the preferred method. -- [Migrate projects using file exports](../../project/settings/import_export.md). - -## Enable export for a group - -Prerequisite: - -- You must have the Owner role for the group. - -To enable import and export for a group: - -1. On the top bar, select **Main menu > Admin**. -1. On the left sidebar, select **Settings > General**. -1. Expand **Visibility and access controls**. -1. In the **Import sources** section, select the checkboxes for the sources you want. - -## Important Notes - -Note the following: - -- Exports are stored in a temporary directory and are deleted every 24 hours by a specific worker. -- To preserve group-level relationships from imported projects, run the Group Import/Export first, to allow projects to -be imported into the desired group structure. -- Imported groups are given a `private` visibility level, unless imported into a parent group. -- If imported into a parent group, a subgroup inherits the same level of visibility unless otherwise restricted. -- To preserve the member list and their respective permissions on imported groups, review the users in these groups. Make -sure these users exist before importing the desired groups. -- Users must set a public email in the source GitLab instance that matches one of their verified emails in the target GitLab instance. - -### Exported contents - -The [`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/group/import_export.yml) -file for groups lists many of the items exported and imported when migrating groups using file exports. View this file in the branch -for your version of GitLab to see the list of items relevant to you. For example, -[`import_export.yml` on the `14-10-stable-ee` branch](https://gitlab.com/gitlab-org/gitlab/-/blob/14-10-stable-ee/lib/gitlab/import_export/group/import_export.yml). - -Migrating projects with file exports uses the same export and import mechanisms as creating projects from templates at the [group](../custom_project_templates.md) and -[instance](../../admin_area/custom_project_templates.md) levels. Therefore, the list of exported items is the same. - -Items that are exported include: - -- Milestones -- Labels -- Boards and Board Lists -- Badges -- Subgroups (including all the aforementioned data) -- Epics - - Epic resource state events ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) in GitLab 15.4) -- Events -- [Wikis](../../project/wiki/group.md) - ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53247) in GitLab 13.9) -- Iterations cadences ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95372) in 15.4) - -Items that are **not** exported include: - -- Projects -- Runner tokens -- SAML discovery tokens - -NOTE: -For more details on the specific data persisted in a group export, see the -[`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/group/import_export.yml) file. - -## Export a group - -Prerequisites: - -- You must have the Owner role for the group. - -To export the contents of a group: - -1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Settings > General**. -1. In the **Advanced** section, select **Export Group**. -1. After the export is generated, you should receive an email with a link to the [exported contents](#exported-contents) - in a compressed tar archive, with contents in NDJSON format. -1. Alternatively, you can download the export from the UI: - - 1. Return to your group's **Settings > General** page. - 1. In the **Advanced** section, select **Download export**. - You can also generate a new file by selecting **Regenerate export**. - -NOTE: -The maximum import file size can be set by the Administrator, default is `0` (unlimited). -As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](../../../api/settings.md#change-application-settings) or the [Admin Area](../../admin_area/settings/account_and_limit_settings.md). Default [modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50MB to 0 in GitLab 13.8. - -You can also use the [group import/export API](../../../api/group_import_export.md). - -### Between CE and EE - -You can export groups from the [Community Edition to the Enterprise Edition](https://about.gitlab.com/install/ce-or-ee/) and vice versa. - -The Enterprise Edition retains some group data that isn't part of the Community Edition. If you're exporting a group from the Enterprise Edition to the Community Edition, you may lose this data. For more information, see [downgrading from EE to CE](../../../index.md). - -## Import the group - -1. Create a new group: - - On the top bar, select **New** (**{plus}**) and then **New group**. - - On an existing group's page, select the **New subgroup** button. -1. Select **Import group**. -1. Enter your group name. -1. Accept or modify the associated group URL. -1. Select **Choose file**. -1. Select the file that you exported in the [Export a group](#export-a-group) section. -1. To begin importing, select **Import group**. - -Your newly imported group page appears after the operation completes. - -## Automate group and project import **(PREMIUM)** - -For information on automating user, group, and project import API calls, see -[Automate group and project import](../../project/import/index.md#automate-group-and-project-import). - -## Version history - -### 14.0+ - -In GitLab 14.0, the JSON format is no longer supported for project and group exports. To allow for a -transitional period, you can still import any JSON exports. The new format for imports and exports -is NDJSON. - -### 13.0+ - -GitLab can import bundles that were exported from a different GitLab deployment. -This ability is limited to two previous GitLab [minor](../../../policy/maintenance.md#versioning) -releases, which is similar to our process for [Security Releases](../../../policy/maintenance.md#security-releases). - -For example: - -| Current version | Can import bundles exported from | -|-----------------|----------------------------------| -| 13.0 | 13.0, 12.10, 12.9 | -| 13.1 | 13.1, 13.0, 12.10 | - -## Rate Limits - -To help avoid abuse, by default, users are rate limited to: - -| Request Type | Limit | -| ---------------- | ---------------------------------------- | -| Export | 6 groups per minute | -| Download export | 1 download per group per minute | -| Import | 6 groups per minute | - -GitLab.com may have [different settings](../../gitlab_com/index.md#importexport) from the defaults. +<!-- This redirect file can be deleted after <2023-03-08>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/group/value_stream_analytics/index.md b/doc/user/group/value_stream_analytics/index.md index 980618ac3ae..1c02ca59e3d 100644 --- a/doc/user/group/value_stream_analytics/index.md +++ b/doc/user/group/value_stream_analytics/index.md @@ -223,7 +223,7 @@ Value stream analytics records the following times for each stage: - **Review**: 14:00 to 19:00: 5 hrs - **Staging**: 19:00 to 19:30: 30 minutes -There are some additional considerations for this example: +Keep in mind the following observations related to this example: - This example demonstrates that it doesn't matter if your first commit doesn't mention the issue number, you can do this later in any commit diff --git a/doc/user/img/markdown_logo.png b/doc/user/img/markdown_logo.png Binary files differindex 5184851b6cf..b5b08738106 100644 --- a/doc/user/img/markdown_logo.png +++ b/doc/user/img/markdown_logo.png diff --git a/doc/user/infrastructure/clusters/connect/new_gke_cluster.md b/doc/user/infrastructure/clusters/connect/new_gke_cluster.md index 6b8e2003b8d..7e6a0495d90 100644 --- a/doc/user/infrastructure/clusters/connect/new_gke_cluster.md +++ b/doc/user/infrastructure/clusters/connect/new_gke_cluster.md @@ -94,7 +94,7 @@ Use CI/CD environment variables to configure your project. 1. On the left sidebar, select **Settings > CI/CD**. 1. Expand **Variables**. 1. Set the variable `BASE64_GOOGLE_CREDENTIALS` to the `base64` encoded JSON file you just created. -1. Set the variable `TF_VAR_gcp_project` to your GCP's `project` name. +1. Set the variable `TF_VAR_gcp_project` to your GCP `project` name. 1. Set the variable `TF_VAR_agent_token` to the agent token displayed in the previous task. 1. Set the variable `TF_VAR_kas_address` to the agent server address displayed in the previous task. diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/prometheus.md b/doc/user/infrastructure/clusters/manage/management_project_applications/prometheus.md deleted file mode 100644 index 64d325dedc6..00000000000 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/prometheus.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../../../../../operations/metrics/index.md' -remove_date: '2022-11-03' ---- - -This document was moved to [another location](../../../../../operations/metrics/index.md). - -<!-- This redirect file can be deleted after <2022-11-03>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/sentry.md b/doc/user/infrastructure/clusters/manage/management_project_applications/sentry.md deleted file mode 100644 index f42e9c83120..00000000000 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/sentry.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../../../../../operations/error_tracking.md' -remove_date: '2022-11-03' ---- - -This document was moved to [another location](../../../../../operations/error_tracking.md). - -<!-- This redirect file can be deleted after <2022-11-03>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/infrastructure/iac/index.md b/doc/user/infrastructure/iac/index.md index 866b16652fa..f9891934067 100644 --- a/doc/user/infrastructure/iac/index.md +++ b/doc/user/infrastructure/iac/index.md @@ -72,10 +72,12 @@ To use a Terraform template: include: # To fetch the latest template, use: - template: Terraform.latest.gitlab-ci.yml + # To fetch the advanced latest template, use: + - template: Terraform/Base.latest.gitlab-ci.yml # To fetch the stable template, use: + - template: Terraform.gitlab-ci.yml + # To fetch the advanced stable template, use: - template: Terraform/Base.gitlab-ci.yml - # To fetch the advanced template, use: - - template: Terraform/Base.latest.gitlab-ci.yml ``` 1. Add the variables as described below: @@ -91,6 +93,10 @@ To use a Terraform template: 1. Optional. Override in your `.gitlab-ci.yml` file the attributes present in the template you fetched to customize your configuration. +### Terraform template recipes + +For GitLab-curated template recipes, see [Terraform template recipes](terraform_template_recipes.md). + ## Related topics - View [the images that contain the `gitlab-terraform` shell script](https://gitlab.com/gitlab-org/terraform-images). diff --git a/doc/user/infrastructure/iac/terraform_state.md b/doc/user/infrastructure/iac/terraform_state.md index a690cc78121..fc86210ed56 100644 --- a/doc/user/infrastructure/iac/terraform_state.md +++ b/doc/user/infrastructure/iac/terraform_state.md @@ -6,7 +6,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab-managed Terraform state **(FREE)** -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2673) in GitLab 13.0. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2673) in GitLab 13.0. +> - Support for state names that contain periods introduced in GitLab 15.7 [with a flag](../../../administration/feature_flags.md) named `allow_dots_on_tf_state_names`. Disabled by default. [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106861) in GitLab 15.7. + +FLAG: +On self-managed GitLab, by default support for state names that contain periods is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `allow_dots_on_tf_state_names`. On GitLab.com, support for state names that contain periods is available. Requests for state files might generate HTTP 404 errors after enabling this feature. For more information, see [Troubleshooting the Terraform integration with GitLab](troubleshooting.md#state-not-found-if-the-state-name-contains-a-period). Terraform uses state files to store details about your infrastructure configuration. With Terraform remote [backends](https://www.terraform.io/language/settings/backends/configuration), diff --git a/doc/user/infrastructure/iac/terraform_template_recipes.md b/doc/user/infrastructure/iac/terraform_template_recipes.md new file mode 100644 index 00000000000..ab2c8c1c48a --- /dev/null +++ b/doc/user/infrastructure/iac/terraform_template_recipes.md @@ -0,0 +1,183 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Terraform template recipes **(FREE)** + +You can customize your Terraform integration by adding the recipes on +this page to your pipeline. + +If you'd like to share your own Terraform configuration, consider +[contributing a recipe](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/user/infrastructure/iac/terraform_template_recipes.md) +to this page. + +## Enable a `terraform destroy` job + +Add the following snippet to your `.gitlab-ci.yml`: + +```yaml +include: + - template: Terraform.latest.gitlab-ci.yml + +destroy: + extends: .terraform:destroy +``` + +The `destroy` job is part of the `cleanup` stage. Like the `deploy` +job, the `destroy` job is always `manual` and is not tied to the +default branch. + +## Run a custom `terraform` command in a job + +To define a job that runs a custom `terraform` command, the +`gitlab-terraform` wrapper can be used in any job: + +```yaml +include: + - template: Terraform.latest.gitlab-ci.yml + +state-list: + stage: validate # you can use any stage, just make sure to define it + script: gitlab-terraform state list +``` + +The `gitlab-terraform` command sets up a `terraform` command and runs +it with the given arguments. + +To run this job in the Terraform state-specific [resource group](../../../ci/resource_groups/index.md), +assign the job with `resource_group`: + +```yaml +include: + - template: Terraform.latest.gitlab-ci.yml + +state-list: + stage: validate # you can use any stage, just make sure to define it + resource_group: ${TF_STATE_NAME} + script: gitlab-terraform state list +``` + +## Add custom debug tools to jobs + +The default image used by Terraform template jobs contains only minimal tooling. +However, you might want to add additional tools for debugging. + +To add an additional tool: + +1. Install the tool in the `before_script` of a job or pipeline. +1. Use the tool in the `script` or `after_script` block. + - If you use the `script` block, be sure to re-add the template job commands. + +For example, the following snippet installs `bash` and `jq` in the `before_script` for all +jobs in the pipeline: + +```yaml +include: + - template: Terraform.latest.gitlab-ci.yml + +default: + before_script: apk add --update bash jq +``` + +To add it to only the `build` and `deploy` jobs, add it to those jobs directly: + +```yaml +include: + - template: Terraform.latest.gitlab-ci.yml + +build: + before_script: apk add --update bash jq + +deploy: + before_script: apk add --update bash jq +``` + +## Add custom container images + +For debug tools and simple installations, you should +[add a custom debug tool to your job](#add-custom-debug-tools-to-jobs). +If your tool is complex or benefits from caching, +you can create a custom container image based on the +[`gitlab-terraform`](https://gitlab.com/gitlab-org/terraform-images) images. +You can use your custom image in subsequent Terraform jobs. + +To define a custom container image: + +1. Define a new `Dockerfile` with custom tooling. For example, install `bash` and `jq` in `.gitlab/ci/Dockerfile`: + + ```dockerfile + FROM registry.gitlab.com/gitlab-org/terraform-images/stable:latest + + RUN apk add --update bash jq + ``` + +1. In a new job, define a `prepare` stage that builds the image whenever the `Dockerfile` changes. + - The built image is pushed to the [GitLab Container Registry](../../packages/container_registry). A tag is applied to indicate whether the image was built from a merge request or from the default branch. +1. Use your image in your Terraform jobs, such as `build` and `deploy`. + - You can combine your image with specialized `before_script` configurations to perform setup commands, like to generate inputs for Terraform. + +For example, a fully functioning pipeline configuration might look like: + +```yaml +include: + - template: Terraform.latest.gitlab-ci.yml + +variables: + IMAGE_TAG: latest + +workflow: + rules: + - if: $CI_MERGE_REQUEST_IID + changes: + - .gitlab/ci/Dockerfile + variables: + IMAGE_TAG: ${CI_COMMIT_REF_SLUG} + - when: always + +stages: + - prepare + - validate + - test + - build + - deploy + - cleanup + +prepare:image: + needs: [] + stage: prepare + image: + name: gcr.io/kaniko-project/executor:v1.9.0-debug + entrypoint: [""] + rules: + # Tag with the commit SHA if we're in an MR + - if: $CI_MERGE_REQUEST_IID + changes: + - .gitlab/ci/Dockerfile + variables: + DOCKER_TAG: $CI_COMMIT_REF_SLUG + # If we're on our main branch, tag with "latest" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + changes: + - .gitlab/ci/Dockerfile + variables: + DOCKER_TAG: latest + before_script: + # Authenticate to the docker registry and dependency proxy + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json + script: + - /kaniko/executor + --context "${CI_PROJECT_DIR}/.gitlab/ci" + --cache=true + --dockerfile "${CI_PROJECT_DIR}/.gitlab/ci/Dockerfile" + --destination "${CI_REGISTRY_IMAGE}:${DOCKER_TAG}" + +build: + image: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} + +deploy: + image: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} +``` + +For an example repository, see the [GitLab Terraform template usage project](https://gitlab.com/gitlab-org/configure/examples/terraform-template-usage). diff --git a/doc/user/infrastructure/iac/troubleshooting.md b/doc/user/infrastructure/iac/troubleshooting.md index 3921c6a7dc8..ad1821fbe10 100644 --- a/doc/user/infrastructure/iac/troubleshooting.md +++ b/doc/user/infrastructure/iac/troubleshooting.md @@ -131,3 +131,41 @@ To permit a user with the Developer role to run destructive commands, you need a 1. Set the value of `TF_USERNAME` to the username of your project access token. 1. Set the value of `TF_PASSWORD` to the password of your project access token. 1. Optional. Protect the variables to make them only available in pipelines that run on protected branches or protected tags. + +### State not found if the state name contains a period + +GitLab 15.6 and earlier returned 404 errors if the state name contained a period and Terraform attempted +a state lock. + +You could work around this limitation by adding `-lock=false` to your Terraform commands. The GitLab backend +accepted the request, but internally stripped the period and any characters that followed from the state name. +For example, a state named `foo.bar` would be stored as `foo`. However, this workaround wasn't recommended, +and could even cause state name collisions. + +In GitLab 15.7 and later, [state names with periods are supported](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106861). If you use the `-lock=false` workaround and upgrade to GitLab 15.7 or later, +your jobs might fail. The failure is caused by the GitLab backend storing a new state with the full state name, which diverges from the existing state name. + +To fix the failing jobs, rename your state names to exclude the period and any characters that follow it. For example, +if you use the Terraform template: + +```yaml +include: + - template: Terraform.gitlab-ci.yml + +variables: + TF_STATE_NAME: foo +``` + +If your `TF_HTTP_ADDRESS`, `TF_HTTP_LOCK_ADDRESS` and `TF_HTTP_UNLOCK_ADDRESS` are set, be sure +to update the state names there. + +Alternatively, you can [migrate your terraform state](terraform_state.md#migrate-to-a-gitlab-managed-terraform-state). + +#### Self-managed GitLab instances + +By default, support for state names with periods is not enabled on self-managed GitLab. +You can enable it from the Rails console: + +```ruby +Feature.enable(:allow_dots_on_tf_state_names) +``` diff --git a/doc/user/markdown.md b/doc/user/markdown.md index b6f3ba1cfdd..17b91eb9483 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Source Code +stage: Plan +group: Project Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -211,7 +211,7 @@ You can use it to point out a :bug: or warn about :speak_no_evil: patches. And i If you're new to this, don't be :fearful:. You can join the emoji :family:. Just look up one of the supported codes. -Consult the [Emoji Cheat Sheet](https://www.emojicopy.com) for a list of all supported emoji codes. :thumbsup: +Consult the [Emoji Cheat Sheet](https://www.webfx.com/tools/emoji-cheat-sheet/) for a list of all supported emoji codes. :thumbsup: ``` Sometimes you want to <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/public/-/emojis/2/monkey.png" width="20px" height="20px" style="display:inline;margin:0;border: 0"> around a bit and add some <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/public/-/emojis/2/star2.png" width="20px" height="20px" style="display:inline;margin:0;border: 0"> to your <img src="https://gitlab.com/gitlab-org/gitlab-foss/raw/master/public/-/emojis/2/speech_balloon.png" width="20px" height="20px" style="display:inline;margin:0;border: 0">. Well we have a gift for you: @@ -374,12 +374,12 @@ a^2+b^2=c^2 #### LaTeX-compatible fencing -> Introduced in GitLab 15.4 [with a flag](../administration/feature_flags.md) named `markdown_dollar_math`. Disabled by default. +> Introduced in GitLab 15.4 [with a flag](../administration/feature_flags.md) named `markdown_dollar_math`. Disabled by default. Enabled on GitLab.com. [View this topic in GitLab](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/user/markdown.md#latex-compatible-fencing). FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, +On self-managed GitLab, by default this feature is not available. To make it available per group, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `markdown_dollar_math`. On GitLab.com, this feature is available. The feature is not ready for production use. @@ -1017,8 +1017,25 @@ Do not change to a reference style link.  -In the rare case where you must set a specific height or width for an image, -you can use the `img` HTML tag instead of Markdown and set its `height` and +#### Change the image dimensions + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/28118) in GitLab 15.7. + +You can control the width and height of an image by following the image with +an attribute list. +The value must an integer with a unit of either `px` (default) or `%`. + +For example + +```markdown +{width=100 height=100px} + +{width=75%} +``` + +{width=100 height=100px} + +You can also use the `img` HTML tag instead of Markdown and set its `height` and `width` parameters. #### Videos @@ -1283,7 +1300,7 @@ Some text to show that the reference links can follow later. Using header ID anchors: -- This line links to [a section on a different Markdown page, using a "#" and the header ID](permissions.md#project-features-permissions) +- This line links to [a section on a different Markdown page, using a "#" and the header ID](permissions.md#project-members-permissions) - This line links to [a different section on the same page, using a "#" and the header ID](#header-ids-and-links) Using references: @@ -1530,6 +1547,7 @@ Tables are not part of the core Markdown specification, but are part of GitLab F by pipes (`|`). - You **can** have blank cells. 1. Column widths are calculated dynamically based on the content of the cells. +1. To use the pipe character (`|`) in the text and not as table delimiter, you must escape it with a backslash (`\|`). Example: @@ -1581,7 +1599,8 @@ use `<br>` tags to force a cell to have multiple lines: | Item2 | This item has:<br>- Multiple items<br>- That we want listed separately | You can use HTML formatting in GitLab itself to add [task lists](#task-lists) with checkboxes, -but they do not render properly on `docs.gitlab.com`: +but they do not render properly on `docs.gitlab.com`. Note that these tasks will not save their +state when selected, like regular GitLab task lists. ```markdown | header 1 | header 2 | @@ -1590,6 +1609,31 @@ but they do not render properly on `docs.gitlab.com`: | cell 3 | <ul><li> - [ ] Task one </li><li> - [ ] Task two </li></ul> | ``` +To have fully functioning task lists in a table, create an HTML table with Markdown in the cells: + +```html +<table> +<thead> +<tr><th>header 1</th><th>header 2</th></tr> +</thead> +<tbody> +<tr> +<td>cell 1</td> +<td>cell 2</td> +</tr> +<tr> +<td>cell 3</td> +<td> + +- [ ] Task one +- [ ] Task two + +</td> +</tr> +</tbody> +</table> +``` + ##### Copy and paste from a spreadsheet > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27205) in GitLab 12.7. diff --git a/doc/user/namespace/index.md b/doc/user/namespace/index.md index 36e048868ad..7d26600cc38 100644 --- a/doc/user/namespace/index.md +++ b/doc/user/namespace/index.md @@ -9,20 +9,24 @@ info: To determine the technical writer assigned to the Stage/Group associated w In GitLab, a *namespace* provides one place to organize your related projects. Projects in one namespace are separate from projects in other namespaces, which means you can use the same name for projects in different namespaces. +## Types of namespaces + GitLab has two types of namespaces: -- A *Personal* namespace, which is based on your username and provided to you when you create your account. - - If you change your username, the project and namespace URLs in your account also change. Before you change your username, - read about [repository redirects](../project/repository/index.md#what-happens-when-a-repository-path-changes). +- A *personal* namespace, which is based on your username and provided to you when you create your account. - You cannot create subgroups in a personal namespace. - Groups in your namespace do not inherit your namespace permissions and group features. - - All the *Personal Projects* created will fall under the scope of this namespace. + - All the projects you create are under the scope of this namespace. + - If you change your username, the project and namespace URLs in your account also change. Before you change your username, + read about [repository redirects](../project/repository/index.md#what-happens-when-a-repository-path-changes). -- A *group* or *subgroup* namespace: +- A *group* or *subgroup* namespace, which is based on the group or subgroup name: - You can create multiple subgroups to manage multiple projects. - - You can change the URL of group and subgroup namespaces. - You can configure settings specifically for each subgroup and project in the namespace. - When you create a subgroup, it inherits some of the parent group settings. You can view these in the subgroup **Settings**. + - You can change the URL of group and subgroup namespaces. + +## Determine which type of namespace you're viewing To determine whether you're viewing a group or personal namespace, you can view the URL. For example: diff --git a/doc/user/operations_dashboard/index.md b/doc/user/operations_dashboard/index.md index 0a65ea0f64d..619d822adf2 100644 --- a/doc/user/operations_dashboard/index.md +++ b/doc/user/operations_dashboard/index.md @@ -1,6 +1,6 @@ --- -stage: none -group: unassigned +stage: Release +group: Release info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/packages/conan_repository/index.md b/doc/user/packages/conan_repository/index.md index 3d3fe35fd65..dd6605c2f01 100644 --- a/doc/user/packages/conan_repository/index.md +++ b/doc/user/packages/conan_repository/index.md @@ -270,8 +270,9 @@ Prerequisites: ``` NOTE: -If you try to install the package you just created in this tutorial, the package -already exists on your local computer, so this command has no effect. +If you try installing the package you created in this tutorial, the install command +will have no effect because the package already exists. +Delete `~/.conan/data` to clean up the packages stored in the cache. ## Remove a Conan package diff --git a/doc/user/packages/container_registry/index.md b/doc/user/packages/container_registry/index.md index 65e7918d827..4b4d6190dc2 100644 --- a/doc/user/packages/container_registry/index.md +++ b/doc/user/packages/container_registry/index.md @@ -6,11 +6,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Container Registry **(FREE)** -> - The group-level Container Registry was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23315) in GitLab 12.10. -> - Searching by image repository name was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31322) in GitLab 13.0. +> Searching by image repository name was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31322) in GitLab 13.0. NOTE: -If you pull container images from Docker Hub, you can also use the [GitLab Dependency Proxy](../dependency_proxy/index.md#use-the-dependency-proxy-for-docker-images) to avoid running into rate limits and speed up your pipelines. +If you pull container images from Docker Hub, you can use the [GitLab Dependency Proxy](../dependency_proxy/index.md#use-the-dependency-proxy-for-docker-images) +to avoid rate limits and speed up your pipelines. With the Docker Container Registry integrated into GitLab, every GitLab project can have its own space to store its Docker images. @@ -28,26 +28,26 @@ You can view the Container Registry for a project or group. 1. Go to your project or group. 1. Go to **Packages and registries > Container Registry**. -You can search, sort, filter, and [delete](#delete-images-from-within-gitlab) +You can search, sort, filter, and [delete](#delete-images-using-the-gitlab-ui) containers on this page. You can share a filtered view by copying the URL from your browser. Only members of the project or group can access a private project's Container Registry. +Images downloaded from a private registry may be [available to other users in a shared runner](https://docs.gitlab.com/runner/security/index.html#usage-of-private-docker-images-with-if-not-present-pull-policy). If a project is public, so is the Container Registry. ### View the tags of a specific image -You can view a list of tags associated with a given container image: +You can use the Container Registry **Tag Details** page to view a list of tags associated with a given container image: 1. Go to your project or group. 1. Go to **Packages and registries > Container Registry**. 1. Select the container image you are interested in. -This brings up the Container Registry **Tag Details** page. You can view details about each tag, -such as when it was published, how much storage it consumes, and the manifest and configuration -digests. +You can view details about each tag, such as when it was published, how much storage it consumes, +and the manifest and configuration digests. -You can search, sort (by tag name), filter, and [delete](#delete-images-from-within-gitlab) +You can search, sort (by tag name), filter, and [delete](#delete-images-using-the-gitlab-ui) tags on this page. You can share a filtered view by copying the URL from your browser. ## Use images from the Container Registry @@ -67,7 +67,7 @@ To download and run a container image hosted in the GitLab Container Registry: docker run [options] registry.example.com/group/project/image [arguments] ``` -[Authentication](#authenticate-with-the-container-registry) is needed to download images from private repository. +[Authentication](#authenticate-with-the-container-registry) is needed to download images from a private repository. For more information on running Docker containers, visit the [Docker documentation](https://docs.docker.com/get-started/). @@ -85,7 +85,7 @@ then your image must be named `gitlab.example.com/mynamespace/myproject` at a mi You can append additional names to the end of an image name, up to two levels deep. -For example, these are all valid image names for images within the project named `myproject`: +For example, these are all valid image names for images in the project named `myproject`: ```plaintext registry.example.com/mynamespace/myproject:some-tag @@ -192,12 +192,12 @@ You can configure your `.gitlab-ci.yml` file to build and push images to the Con - Before building, use `docker build --pull` to fetch changes to base images. It takes slightly longer, but it ensures your image is up-to-date. - Before each `docker run`, do an explicit `docker pull` to fetch - the image that was just built. This is especially important if you are + the image that was just built. This step is especially important if you are using multiple runners that cache images locally. If you use the Git SHA in your image tag, each job is unique and you should never have a stale image. However, it's still possible to have a - stale image if you re-build a given commit after a dependency has changed. + stale image if you rebuild a given commit after a dependency has changed. - Don't build directly to the `latest` tag because multiple jobs may be happening simultaneously. @@ -234,12 +234,12 @@ build: - docker push $IMAGE_TAG ``` -Here, `$CI_REGISTRY_IMAGE` would be resolved to the address of the registry tied -to this project. Since `$CI_COMMIT_REF_NAME` resolves to the branch or tag name, -and your branch name can contain forward slashes (for example, `feature/my-feature`), it is -safer to use `$CI_COMMIT_REF_SLUG` as the image tag. This is due to that image tags -cannot contain forward slashes. We also declare our own variable, `$IMAGE_TAG`, -combining the two to save us some typing in the `script` section. +In this example, `$CI_REGISTRY_IMAGE` resolves to the address of the registry tied +to this project. `$CI_COMMIT_REF_NAME` resolves to the branch or tag name, which +can contain forward slashes. Image tags can't contain forward slashes. Use +`$CI_COMMIT_REF_SLUG` as the image tag. You can declare the variable, `$IMAGE_TAG`, +combining `$CI_REGISTRY_IMAGE` and `$CI_REGISTRY_IMAGE` to save some typing in the +`script` section. Here's a more elaborate example that splits up the tasks into 4 pipeline stages, including two tests that run in parallel. The `build` is stored in the container @@ -385,17 +385,18 @@ unreferenced, administrators must run [garbage collection](../../../administrati On GitLab.com, the latest version of the Container Registry includes an automatic online garbage collector. For more information, see [this blog post](https://about.gitlab.com/blog/2021/10/25/gitlab-com-container-registry-update/). -This is an instance-wide feature, rolling out gradually to a subset of the user base, so some new image repositories created -from GitLab 14.5 onwards are served by this new version of the Container Registry. In this new -version of the Container Registry, layers that aren't referenced by any image manifest, and image -manifests that have no tags and aren't referenced by another manifest (such as multi-architecture -images), are automatically scheduled for deletion after 24 hours if left unreferenced. +The automatic online garbage collector is an instance-wide feature, rolling out gradually to a subset +of the user base. Some new image repositories created from GitLab 14.5 onward are served by this +new version of the Container Registry. In this new version of the Container Registry, layers that aren't +referenced by any image manifest, and image manifests that have no tags and aren't referenced by another +manifest (such as multi-architecture images), are automatically scheduled for deletion after 24 hours if +left unreferenced. -### Delete images from within GitLab +### Delete images using the GitLab UI -To delete images from within GitLab: +To delete images using the GitLab UI: -1. Navigate to your project's or group's **Packages and registries > Container Registry**. +1. Go to your project's or group's **Packages and registries > Container Registry**. 1. From the **Container Registry** page, you can select what you want to delete, by either: @@ -419,7 +420,7 @@ information, see the following endpoints: ### Delete images using GitLab CI/CD WARNING: -GitLab CI/CD doesn't provide a built-in way to remove your images, but this example +GitLab CI/CD doesn't provide a built-in way to remove your images. This example uses a third-party tool called [reg](https://github.com/genuinetools/reg) that talks to the GitLab Registry API. You are responsible for your own actions. For assistance with this tool, see @@ -455,21 +456,18 @@ build_image: - main delete_image: - image: docker:20.10.16 + before_script: + - curl --fail --show-error --location "https://github.com/genuinetools/reg/releases/download/v$REG_VERSION/reg-linux-amd64" --output ./reg + - echo "$REG_SHA256 ./reg" | sha256sum -c - + - chmod a+x ./reg + image: curlimages/curl:7.86.0 + script: + - ./reg rm -d --auth-url $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $IMAGE_TAG stage: clean - services: - - docker:20.10.16-dind variables: IMAGE_TAG: $CI_PROJECT_PATH:$CI_COMMIT_REF_SLUG REG_SHA256: ade837fc5224acd8c34732bf54a94f579b47851cc6a7fd5899a98386b782e228 REG_VERSION: 0.16.1 - before_script: - - apk add --no-cache curl - - curl --fail --show-error --location "https://github.com/genuinetools/reg/releases/download/v$REG_VERSION/reg-linux-amd64" --output /usr/local/bin/reg - - echo "$REG_SHA256 /usr/local/bin/reg" | sha256sum -c - - - chmod a+x /usr/local/bin/reg - script: - - /usr/local/bin/reg rm -d --auth-url $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $IMAGE_TAG only: - branches except: @@ -487,16 +485,14 @@ defined in the `delete_image` job. You can create a per-project [cleanup policy](reduce_container_registry_storage.md#cleanup-policy) to ensure older tags and images are regularly removed from the Container Registry. -## Limitations +## Known issues -- Moving or renaming existing Container Registry repositories is not supported - once you have pushed images, because the images are stored in a path that matches - the repository path. To move or rename a repository with a - Container Registry, you must delete all existing images. - Community suggestions to work around this limitation have been shared in - [issue 18383](https://gitlab.com/gitlab-org/gitlab/-/issues/18383#possible-workaround). -- Prior to GitLab 12.10, any tags that use the same image ID as the `latest` tag - are not deleted by the cleanup policy. +Moving or renaming existing Container Registry repositories is not supported +after you have pushed images. The images are stored in a path that matches +the repository path. To move or rename a repository with a +Container Registry, you must delete all existing images. +Community suggestions to work around this known issue have been shared in +[issue 18383](https://gitlab.com/gitlab-org/gitlab/-/issues/18383#possible-workaround). ## Disable the Container Registry for a project @@ -530,7 +526,7 @@ for more details about the permissions that this setting grants to users. is internal or private, the Container Registry is also internal or private. - **Only Project Members**: The Container Registry is visible only to project members with - Reporter role or higher. This is similar to the behavior of a private project with Container + Reporter role or higher. This visibility is similar to the behavior of a private project with Container Registry visibility set to **Everyone With Access**. 1. Select **Save changes**. @@ -541,7 +537,7 @@ The ability to view the Container Registry and pull images is controlled by the visibility permissions. You can change this through the [visibility setting on the UI](#change-visibility-of-the-container-registry) or the [API](../../../api/container_registry.md#change-the-visibility-of-the-container-registry). [Other permissions](../../permissions.md) -such as updating the Container Registry, pushing or deleting images, and so on are not affected by +such as updating the Container Registry and pushing or deleting images are not affected by this setting. However, disabling the Container Registry disables all Container Registry operations. | | | Anonymous<br/>(Everyone on internet) | Guest | Reporter, Developer, Maintainer, Owner | @@ -553,188 +549,3 @@ this setting. However, disabling the Container Registry disables all Container R | Private project with Container Registry visibility <br/> set to **Everyone With Access** (UI) or `enabled` (API) | View Container Registry <br/> and pull images | No | No | Yes | | Private project with Container Registry visibility <br/> set to **Only Project Members** (UI) or `private` (API) | View Container Registry <br/> and pull images | No | No | Yes | | Any project with Container Registry `disabled` | All operations on Container Registry | No | No | No | - -## Troubleshooting the GitLab Container Registry - -### Migrating OCI container images to GitLab Container Registry - -Migrating built container images to the GitLab registry is not a current feature. However, an [epic](https://gitlab.com/groups/gitlab-org/-/epics/5210) is open to track the work on this feature. - -Some third-party tools can help migrate container images, for example, [skopeo](https://github.com/containers/skopeo), which can [copy container images](https://github.com/containers/skopeo#copying-images) between various storage mechanisms. You can use skopeo to copy from container registries, container storage backends, local directories, and local OCI-layout directories to the GitLab Container Registry. - -### Docker connection error - -A Docker connection error can occur when there are special characters in either the group, -project or branch name. Special characters can include: - -- Leading underscore -- Trailing hyphen/dash - -To get around this, you can [change the group path](../../group/manage.md#change-a-groups-path), -[change the project path](../../project/settings/index.md#rename-a-repository) or change the branch -name. - -You may also get a `404 Not Found` or `Unknown Manifest` message if you are using -a Docker Engine version earlier than 17.12. Later versions of Docker Engine use -[the v2 API](https://docs.docker.com/registry/spec/manifest-v2-2/). - -The images in your GitLab Container Registry must also use the Docker v2 API. -For information on how to update your images, see the [Docker help](https://docs.docker.com/registry/spec/deprecated-schema-v1). - -### `Blob unknown to registry` error when pushing a manifest list - -When [pushing a Docker manifest list](https://docs.docker.com/engine/reference/commandline/manifest/#create-and-push-a-manifest-list) -to the GitLab Container Registry, you may receive the error -`manifest blob unknown: blob unknown to registry`. This is likely caused by having multiple images -with different architectures, spread out over several repositories instead of the same repository. - -For example, you may have two images, each representing an architecture: - -- The `amd64` platform -- The `arm64v8` platform - -To build a multi-arch image with these images, you must push them to the same repository as the -multi-arch image. - -To address the `Blob unknown to registry` error, include the architecture in the tag name of -individual images. For example, use `mygroup/myapp:1.0.0-amd64` and `mygroup/myapp:1.0.0-arm64v8`. -You can then tag the manifest list with `mygroup/myapp:1.0.0`. - -### Troubleshoot as a GitLab server administrator - -Troubleshooting the GitLab Container Registry, most of the times, requires -you to sign in to GitLab server with administrator access. - -[Read how to troubleshoot the Container Registry](../../../administration/packages/container_registry.md#troubleshooting). - -### Unable to change path or transfer a project - -If you try to change a project's path or transfer a project to a new namespace, -you may receive one of the following errors: - -- "Project cannot be transferred, because tags are present in its container registry." -- "Namespace cannot be moved because at least one project has tags in container registry." - -This issue occurs when the project has images in the Container Registry. -You must delete or move these images before you can change the path or transfer -the project. - -The following procedure uses these sample project names: - -- For the current project: `gitlab.example.com/org/build/sample_project/cr:v2.9.1` -- For the new project: `gitlab.example.com/new_org/build/new_sample_project/cr:v2.9.1` - -Use your own URLs to complete the following steps: - -1. Download the Docker images on your computer: - - ```shell - docker login gitlab.example.com - docker pull gitlab.example.com/org/build/sample_project/cr:v2.9.1 - ``` - - NOTE: - For container registry authentication, use either a - [personal access token](../../profile/personal_access_tokens.md) or a - [deploy token](../../project/deploy_tokens/index.md). - -1. Rename the images to match the new project name: - - ```shell - docker tag gitlab.example.com/org/build/sample_project/cr:v2.9.1 gitlab.example.com/new_org/build/new_sample_project/cr:v2.9.1 - ``` - -1. Delete the images in the old project by using the [UI](#delete-images) or [API](../../../api/packages.md#delete-a-project-package). - There may be a delay while the images are queued and deleted. -1. Change the path or transfer the project by going to **Settings > General** - and expanding **Advanced**. -1. Restore the images: - - ```shell - docker push gitlab.example.com/new_org/build/new_sample_project/cr:v2.9.1 - ``` - -Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/18383) for details. - -### Tags on S3 backend remain after successful deletion requests - -With S3 as your storage backend, tags may remain even though: - -- In the UI, you see that the tags are scheduled for deletion. -- In the API, you get an HTTP `200` response. -- The registry log shows a successful `Delete` request. - -An example `DELETE` request in the registry log: - -```shell -{"content_type":"","correlation_id":"01FQGNSKVMHQEAVE21KYTJN2P4","duration_ms":62,"host":"localhost:5000","level":"info","method":"DELETE","msg":"access","proto":"HTTP/1.1","referrer":"","remote_addr":"127.0.0.1:47498","remote_ip":"127.0.0.1","status":202,"system":"http","time":"2021-12-22T08:58:15Z","ttfb_ms":62,"uri":"/v2/<path to repo>/tags/reference/<tag_name>","user_agent":"GitLab/<version>","written_bytes":0} -``` - -There may be some errors not properly cached. Follow these steps to investigate further: - -1. In your configuration file, set the registry's log level to `debug`, and the S3 driver's log - level to `logdebugwithhttpbody`. For Omnibus, make these edits in the `gitlab.rb` file: - - ```shell - # Change the registry['log_level'] to debug - registry['log_level'] = 'debug' - - # Set log level for registry log from storage side - registry['storage'] = { - 's3' => { - 'bucket' => 'your-s3-bucket', - 'region' => 'your-s3-region' - }, - - 'loglevel' = "logdebugwithhttpbody" - } - ``` - - Then save and reconfigure GitLab: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -1. Attempt to delete one or more tags using the GitLab UI or API. - -1. Inspect the registry logs and look for a response from S3. Although the response could be - `200 OK`, the body might have the error `AccessDenied`. This indicates a permission problem from - the S3 side. - -1. Ensure your S3 configuration has the `deleteObject` permission scope. Here's an - [example role for an S3 bucket](../../../administration/object_storage.md#iam-permissions). - Once adjusted, trigger another tag deletion. You should be able to successfully delete tags. - -Follow [this issue](https://gitlab.com/gitlab-org/container-registry/-/issues/551) for details. - -### Tags temporarily cannot be marked for deletion - -GitLab is [migrating to the next generation of the Container Registry](https://gitlab.com/groups/gitlab-org/-/epics/5523). -During the migration, you may encounter difficulty deleting tags. -If you encounter an error, it's likely that your image repository is in the process of being migrated. -Wait a few minutes and try again. - -### `unauthorized: authentication required` when pushing large images - -When pushing large images, you might get an error like the following: - -```shell -docker push gitlab.example.com/myproject/docs:latest -The push refers to a repository [gitlab.example.com/myproject/docs] -630816f32edb: Preparing -530d5553aec8: Preparing -... -4b0bab9ff599: Waiting -d1c800db26c7: Waiting -42755cf4ee95: Waiting -unauthorized: authentication required -``` - -On self-managed GitLab instances, by default, tokens for the Container Registry expire every five minutes. -When pushing larger images, or images that take longer than five minutes to push, -you might encounter this error. On GitLab.com, the expiration time is 15 minutes. - -If you are using self-managed GitLab, you can ask an administrator to -[increase the token duration](../../../administration/packages/container_registry.md#increase-token-duration) -if necessary. diff --git a/doc/user/packages/container_registry/reduce_container_registry_data_transfer.md b/doc/user/packages/container_registry/reduce_container_registry_data_transfer.md index 74cbcba2ffc..0ce9635e05a 100644 --- a/doc/user/packages/container_registry/reduce_container_registry_data_transfer.md +++ b/doc/user/packages/container_registry/reduce_container_registry_data_transfer.md @@ -44,7 +44,7 @@ Consider using a smaller base image, such as [Alpine Linux](https://alpinelinux. An Alpine image is around 5MB, which is several times smaller than popular base images such as [Debian](https://hub.docker.com/_/debian). If your application is distributed as a self-contained static binary, such as for Go applications, -you can also consider using Docker's [scratch](https://hub.docker.com/_/scratch/) +you can also consider using the Docker [scratch](https://hub.docker.com/_/scratch/) base image. If you need to use a specific base image OS, look for `-slim` or `-minimal` variants, as this helps diff --git a/doc/user/packages/container_registry/reduce_container_registry_storage.md b/doc/user/packages/container_registry/reduce_container_registry_storage.md index 23d835ddf5f..cbf9af633ac 100644 --- a/doc/user/packages/container_registry/reduce_container_registry_storage.md +++ b/doc/user/packages/container_registry/reduce_container_registry_storage.md @@ -4,17 +4,18 @@ group: Container Registry info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Reduce Container Registry Storage **(FREE)** +# Reduce Container Registry storage **(FREE)** -Container registries become large over time without cleanup. When a large number of images or tags are added: +Container registries can grow in size over time if you don't manage your registry usage. For example, +if you add a large number of images or tags: -- Fetching the list of available tags or images becomes slower. +- Retrieving the list of available tags or images becomes slower. - They take up a large amount of storage space on the server. -We recommend deleting unnecessary images and tags and setting up a [cleanup policy](#cleanup-policy) +You should delete unnecessary images and tags and set up a [cleanup policy](#cleanup-policy) to automatically manage your container registry usage. -## Check Container Registry Storage Use +## Check Container Registry storage use The Usage Quotas page (**Settings > Usage Quotas > Storage**) displays storage usage for Packages. This page includes the Container Registry usage, which is only available on GitLab.com. @@ -23,10 +24,16 @@ metadata database. Support for improvements is proposed in epic [5523](https://g You cannot use the Container Registry in self-managed instances, but epic [5521](https://gitlab.com/groups/gitlab-org/-/epics/5521) proposes to change this behavior. Image layers stored in the Container Registry are deduplicated at the root namespace level. -If you tag the same image more than once in the same repository or across distinct -repositories under the same root namespace, it is only counted once. -If an image layer is shared across multiple images, in the same -container repository, project, group, or across different repositories, it is only counted once. + +An image is only counted once if: + +- You tag the same image more than once in the same repository. +- You tag the same image across distinct repositories under the same root namespace. + +An image layer is only counted once if: + +- You share the image layer across multiple images in the same container repository, project, or group. +- You share the image layer across different repositories. Only layers that are referenced by tagged images are accounted for. Untagged images and any layers referenced exclusively by them are subject to [online garbage collection](index.md#delete-images). @@ -50,7 +57,7 @@ To delete the underlying layers and images that aren't associated with any tags, ### Enable the cleanup policy -Cleanup policies can be run on all projects, with these exceptions: +You can run cleanup policies on all projects with these exceptions: - For self-managed GitLab instances, the project must have been created in GitLab 12.8 or later. However, an administrator can enable the cleanup policy @@ -63,7 +70,7 @@ Cleanup policies can be run on all projects, with these exceptions: ApplicationSetting.last.update(container_expiration_policies_enable_historic_entries: true) ``` - Enabling cleanup policies on all project can impact performance, especially if you + Enabling cleanup policies on all projects can impact performance, especially if you are using an [external registry](#use-with-external-container-registries). WARNING: @@ -72,34 +79,34 @@ GitLab.com that don't have a container image. ### How the cleanup policy works -The cleanup policy collects all tags in the Container Registry and excludes tags -until only the tags to be deleted remain. +The cleanup policy collects all tags in the Container Registry and excludes tags until the only +tags you want to delete remain. The cleanup policy searches for images based on the tag name. Support for full path matching is tracked in issue [281071](https://gitlab.com/gitlab-org/gitlab/-/issues/281071). The cleanup policy: 1. Collects all tags for a given repository in a list. -1. Excludes the tag named `latest` from the list. -1. Evaluates the `name_regex` (tags to expire), excluding non-matching names from the list. -1. Excludes from the list any tags matching the `name_regex_keep` value (tags to preserve). +1. Excludes the tag named `latest`. +1. Evaluates the `name_regex` (tags to expire), excluding non-matching names. +1. Excludes any tags matching the `name_regex_keep` value (tags to preserve). 1. Excludes any tags that do not have a manifest (not part of the options in the UI). 1. Orders the remaining tags by `created_date`. -1. Excludes from the list the N tags based on the `keep_n` value (Number of tags to retain). -1. Excludes from the list the tags more recent than the `older_than` value (Expiration interval). -1. Finally, the remaining tags in the list are deleted from the Container Registry. +1. Excludes the N tags based on the `keep_n` value (Number of tags to retain). +1. Excludes the tags more recent than the `older_than` value (Expiration interval). +1. Deletes the remaining tags in the list from the Container Registry. WARNING: On GitLab.com, the execution time for the cleanup policy is limited. Some tags may remain in the Container Registry after the policy runs. The next time the policy runs, the remaining tags are included. -It may take multiple runs for all tags to be deleted. +It may take multiple runs to delete all tags. WARNING: GitLab self-managed installations support third-party container registries that comply with the [Docker Registry HTTP API V2](https://docs.docker.com/registry/spec/api/) -specification. However, this specification does not include a tag delete operation. Therefore, when -interacting with third-party container registries, GitLab uses a workaround to delete tags. See the -[related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/15737) +specification. However, this specification does not include a tag delete operation. Therefore, GitLab uses a +workaround to delete tags when interacting with third-party container registries. Refer to +issue [15737](https://gitlab.com/gitlab-org/gitlab/-/issues/15737) for more information. Due to possible implementation variations, this workaround is not guaranteed to work with all third-party registries in the same predictable way. If you use the GitLab Container Registry, this workaround is not required because we implemented a special tag delete operation. In @@ -115,18 +122,18 @@ To create a cleanup policy in the UI: 1. Expand the **Clean up image tags** section. 1. Complete the fields. - | Field | Description | - |---------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------| - | **Toggle** | Turn the policy on or off. | - | **Run cleanup** | How often the policy should run. | - | **Keep the most recent** | How many tags to _always_ keep for each image. | - | **Keep tags matching** | The regex pattern that determines which tags to preserve. The `latest` tag is always preserved. For all tags, use `.*`. See other [regex pattern examples](#regex-pattern-examples). | - | **Remove tags older than** | Remove only tags older than X days. | - | **Remove tags matching** | The regex pattern that determines which tags to remove. This value cannot be blank. For all tags, use `.*`. See other [regex pattern examples](#regex-pattern-examples). | + | Field | Description | + |----------------------------|-------------------------------------------------| + | **Toggle** | Turn the policy on or off. | + | **Run cleanup** | How often the policy should run. | + | **Keep the most recent** | How many tags to _always_ keep for each image. | + | **Keep tags matching** | A regex pattern that determines which tags to preserve. The `latest` tag is always preserved. For all tags, use `.*`. See other [regex pattern examples](#regex-pattern-examples). | + | **Remove tags older than** | Remove only tags older than X days. | + | **Remove tags matching** | A regex pattern that determines which tags to remove. This value cannot be blank. For all tags, use `.*`. See other [regex pattern examples](#regex-pattern-examples). | 1. Select **Save**. -Depending on the interval you chose, the policy is scheduled to run. +The policy runs on the scheduled interval you selected. NOTE: If you edit the policy and select **Save** again, the interval is reset. @@ -135,7 +142,8 @@ If you edit the policy and select **Save** again, the interval is reset. Cleanup policies use regex patterns to determine which tags should be preserved or removed, both in the UI and the API. -Regex patterns are automatically surrounded with `\A` and `\Z` anchors. Do not include any `\A`, `\Z`, `^` or `$` token in the regex patterns as they are not necessary. +Regex patterns are automatically surrounded with `\A` and `\Z` anchors. Therefore, you do not need to include any +`\A`, `\Z`, `^` or `$` tokens in the regex patterns. Here are some examples of regex patterns you can use: @@ -180,17 +188,17 @@ Here are some examples of regex patterns you can use: Cleanup policies are executed as a background process. This process is complex, and depending on the number of tags to delete, the process can take time to finish. -To prevent server resource starvation, the following application settings are available: +You can use the following application settings to prevent server resource starvation: - `container_registry_expiration_policies_worker_capacity`: the maximum number of cleanup workers - running concurrently. This must be greater than or equal to `0`. We recommend starting with a low - number and increasing it after monitoring the resources used by the background workers. To remove + running concurrently. This value must be greater than or equal to `0`. You should start with a low + number and increase it after monitoring the resources used by the background workers. To remove all workers and not execute the cleanup policies, set this to `0`. The default value is `4`. - `container_registry_delete_tags_service_timeout`: the maximum time (in seconds) that the cleanup process can take to delete a batch of tags. The default value is `250`. - `container_registry_cleanup_tags_service_max_list_size`: the maximum number of tags that can be - deleted in a single execution. Additional tags must be deleted in another execution. We recommend - starting with a low number and increasing it after monitoring that container images are properly + deleted in a single execution. Additional tags must be deleted in another execution. You should + start with a low number and increase it after verifying that container images are properly deleted. The default value is `200`. - `container_registry_expiration_policies_caching`: enable or disable tag creation timestamp caching during execution of policies. Cached timestamps are stored in [Redis](../../../development/architecture.md#redis). @@ -213,7 +221,8 @@ You can set, update, and disable the cleanup policies using the GitLab API. Examples: -- Select all tags, keep at least 1 tag per image, clean up any tag older than 14 days, run once a month, preserve any images with the name `main` and the policy is enabled: +- Select all tags, keep at least 1 tag per image, clean up any tag older than 14 days, run once a month, preserve +any images with the name `main`, and the policy is enabled: ```shell curl --request PUT --header 'Content-Type: application/json;charset=UTF-8' --header "PRIVATE-TOKEN: <your_access_token>" \ @@ -251,14 +260,14 @@ See the API documentation for further details: [Edit project API](../../../api/p When using an [external container registry](../../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint), running a cleanup policy on a project may have some performance risks. -If a project runs a policy to remove thousands of tags +If a project runs a policy to remove thousands of tags, the GitLab background jobs may get backed up or fail completely. -For projects created before GitLab 12.8, we recommend you enable container cleanup policies +For projects created before GitLab 12.8, you should enable container cleanup policies only if the number of tags being cleaned up is minimal. ## More Container Registry storage reduction options -Here are some other options to reduce your project's use of Container Registry storage: +Here are some other options you can use to reduce the Container Registry storage used by your project: - Use the [GitLab UI](index.md#delete-images) to delete individual image tags or the entire repository containing all the tags. @@ -330,6 +339,10 @@ the tags. To create the list and delete the tags: 1. Remove any tags that you want to keep from the `list_o_tags.out` file. For example, you can use `sed` to parse the file and remove the tags. + ::Tabs + + :::TabTitle Linux + ```shell # Remove the `latest` tag from the file sed -i '/latest/d' list_o_tags.out @@ -344,12 +357,24 @@ the tags. To create the list and delete the tags: sed -i '/_v3$/d' list_o_tags.out ``` - If you are running macOS, you must add `.bak` to the commands. For example: + :::TabTitle macOS ```shell + # Remove the `latest` tag from the file sed -i .bak '/latest/d' list_o_tags.out + + # Remove the first N tags from the file + sed -i .bak '1,Nd' list_o_tags.out + + # Remove the tags starting with `Av` from the file + sed -i .bak '/^Av/d' list_o_tags.out + + # Remove the tags ending with `_v3` from the file + sed -i .bak '/_v3$/d' list_o_tags.out ``` + ::EndTabs + 1. Double-check the `list_o_tags.out` file to make sure it contains only the tags that you want to delete. diff --git a/doc/user/packages/container_registry/troubleshoot_container_registry.md b/doc/user/packages/container_registry/troubleshoot_container_registry.md new file mode 100644 index 00000000000..eac7e0fcacd --- /dev/null +++ b/doc/user/packages/container_registry/troubleshoot_container_registry.md @@ -0,0 +1,129 @@ +--- +stage: Package +group: Container Registry +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Troubleshooting the GitLab Container Registry + +You must sign in to GitLab with administrator rights to troubleshoot most issues with the GitLab Container Registry. + +You can find [additional troubleshooting information](../../../administration/packages/container_registry.md#troubleshooting) in the GitLab Container Registry administration documentation. + +## Migrating OCI container images to GitLab Container Registry + +Migrating container images to the GitLab registry is not supported, but [epic](https://gitlab.com/groups/gitlab-org/-/epics/5210) proposes to change this behavior. + +You can use third-party tools to migrate container images. For example, [skopeo](https://github.com/containers/skopeo), can [copy container images](https://github.com/containers/skopeo#copying-images) between various storage mechanisms. You can use skopeo to copy from container registries, container storage backends, local directories, and local OCI-layout directories to the GitLab Container Registry. + +## Docker connection error + +A Docker connection error can occur when there are special characters in either the group, +project, or branch name. Special characters include: + +- A leading underscore. +- A trailing hyphen or dash. + +To resolve this error, you can change the [group path](../../group/manage.md#change-a-groups-path), +the [project path](../../project/settings/index.md#rename-a-repository) or the branch name. + +You may get a `404 Not Found` or `Unknown Manifest` error message if you use +Docker Engine 17.11 or earlier. Current versions of Docker Engine use +the [v2 API](https://docs.docker.com/registry/spec/manifest-v2-2/). + +The images in your GitLab Container Registry must use the Docker v2 API. +For information on how to update version 1 images to version 2, see the [Docker documentation](https://docs.docker.com/registry/spec/deprecated-schema-v1). + +## `Blob unknown to registry` error when pushing a manifest list + +When [pushing a Docker manifest list](https://docs.docker.com/engine/reference/commandline/manifest/#create-and-push-a-manifest-list) +to the GitLab Container Registry, you may receive the error +`manifest blob unknown: blob unknown to registry`. This error is likely caused by having multiple images +with different architectures spread out over several repositories instead of the same repository. + +For example, you may have two images, each representing an architecture: + +- The `amd64` platform. +- The `arm64v8` platform. + +To build a multi-arch image with these images, you must push them to the same repository as the +multi-arch image. + +To address the `Blob unknown to registry` error, include the architecture in the tag name of +individual images. For example, use `mygroup/myapp:1.0.0-amd64` and `mygroup/myapp:1.0.0-arm64v8`. +You can then tag the manifest list with `mygroup/myapp:1.0.0`. + +## Unable to change project path or transfer a project + +If you try to change a project path or transfer a project to a new namespace, +you may receive one of the following errors: + +- Project cannot be transferred because tags are present in its container registry. +- Namespace cannot be moved because at least one project has tags in the container registry. + +This error occurs when the project has images in the Container Registry. +You must delete or move these images before you change the path or transfer +the project. + +The following procedure uses these sample project names: + +- For the current project: `gitlab.example.com/org/build/sample_project/cr:v2.9.1`. +- For the new project: `gitlab.example.com/new_org/build/new_sample_project/cr:v2.9.1`. + +1. Download the Docker images on your computer: + + ```shell + docker login gitlab.example.com + docker pull gitlab.example.com/org/build/sample_project/cr:v2.9.1 + ``` + + NOTE: + Use either a [personal access token](../../profile/personal_access_tokens.md) or a + [deploy token](../../project/deploy_tokens/index.md) to authenticate your user account. + +1. Rename the images to match the new project name: + + ```shell + docker tag gitlab.example.com/org/build/sample_project/cr:v2.9.1 gitlab.example.com/new_org/build/new_sample_project/cr:v2.9.1 + ``` + +1. Delete the images in the old project by using the [UI](index.md#delete-images) or [API](../../../api/packages.md#delete-a-project-package). + There may be a delay while the images are queued and deleted. +1. Change the path or transfer the project: + + 1. On the top bar, select **Main menu > Projects** and find your project. + 1. On the left sidebar, select **Settings > General**. + 1. Expand the **Advanced** section. + 1. In the **Change path** text box, edit the path. + 1. Select **Change path**. + +1. Restore the images: + + ```shell + docker push gitlab.example.com/new_org/build/new_sample_project/cr:v2.9.1 + ``` + +See this [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/18383) for details. + +## `unauthorized: authentication required` when pushing large images + +When pushing large images, you may see an authentication error like the following: + +```shell +docker push gitlab.example.com/myproject/docs:latest +The push refers to a repository [gitlab.example.com/myproject/docs] +630816f32edb: Preparing +530d5553aec8: Preparing +... +4b0bab9ff599: Waiting +d1c800db26c7: Waiting +42755cf4ee95: Waiting +unauthorized: authentication required +``` + +This error happens when your authentication token expires before the image push is complete. By default, tokens for +the Container Registry on self-managed GitLab instances expire every five minutes. On GitLab.com, the token expiration +time is set to 15 minutes. + +If you are using self-managed GitLab, you can ask an administrator to +[increase the token duration](../../../administration/packages/container_registry.md#increase-token-duration). diff --git a/doc/user/packages/generic_packages/index.md b/doc/user/packages/generic_packages/index.md index 563f35f2f4f..9b49f946984 100644 --- a/doc/user/packages/generic_packages/index.md +++ b/doc/user/packages/generic_packages/index.md @@ -118,7 +118,7 @@ API or the UI. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/293755) in GitLab 13.12. > - [Required permissions](https://gitlab.com/gitlab-org/gitlab/-/issues/350682) changed from developer to maintainer in GitLab 15.0. -To prevent users from publishing duplicate generic packages, you can use the [GraphQl API](../../../api/graphql/reference/index.md#packagesettings) +To prevent users from publishing duplicate generic packages, you can use the [GraphQL API](../../../api/graphql/reference/index.md#packagesettings) or the UI. In the UI: diff --git a/doc/user/packages/gradle_repository/index.md b/doc/user/packages/gradle_repository/index.md new file mode 100644 index 00000000000..4247c13297d --- /dev/null +++ b/doc/user/packages/gradle_repository/index.md @@ -0,0 +1,372 @@ +--- +stage: Package +group: Package Registry +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Maven packages in the Package Registry **(FREE)** + +Publish [Maven](https://maven.apache.org) artifacts in your project's Package Registry using Gradle. +Then, install the packages whenever you need to use them as a dependency. + +For documentation of the specific API endpoints that the Maven package manager +client uses, see the [Maven API documentation](../../../api/packages/maven.md). + +Learn how to build a [Gradle](../workflows/build_packages.md#gradle) package. + +## Publish to the GitLab Package Registry + +### Tokens + +You need a token to publish a package. Different tokens are available depending on what you're trying to +achieve. For more information, review the [guidance on tokens](../package_registry/index.md#authenticate-with-the-registry). + +- If your organization uses two-factor authentication (2FA), you must use a personal access token with the scope set to `api`. +- If you publish a package via CI/CD pipelines, you must use a CI job token. + +Create a token and save it to use later in the process. + +## Authenticate to the Package Registry with Gradle + +### Authenticate with a personal access token or deploy token in Gradle + +In [your `GRADLE_USER_HOME` directory](https://docs.gradle.org/current/userguide/directory_layout.html#dir:gradle_user_home), +create a file `gradle.properties` with the following content: + +```properties +gitLabPrivateToken=REPLACE_WITH_YOUR_TOKEN +``` + +Your token name depends on which token you use. + +| Token type | Token name | +| --------------------- | --------------- | +| Personal access token | `Private-Token` | +| Deploy token | `Deploy-Token` | + +Add a `repositories` section to your +[`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) +file: + +```groovy +repositories { + maven { + url "https://gitlab.example.com/api/v4/groups/<group>/-/packages/maven" + name "GitLab" + credentials(HttpHeaderCredentials) { + name = 'REPLACE_WITH_TOKEN_NAME' + value = gitLabPrivateToken + } + authentication { + header(HttpHeaderAuthentication) + } + } +} +``` + +Or add it to your `build.gradle.kts` file if you are using Kotlin DSL: + +```kotlin +repositories { + maven { + url = uri("https://gitlab.example.com/api/v4/groups/<group>/-/packages/maven") + name = "GitLab" + credentials(HttpHeaderCredentials::class) { + name = "REPLACE_WITH_TOKEN_NAME" + value = findProperty("gitLabPrivateToken") as String? + } + authentication { + create("header", HttpHeaderAuthentication::class) + } + } +} +``` + +### Authenticate with a CI job token in Gradle + +To authenticate with a CI job token, add a `repositories` section to your +[`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) +file: + +```groovy +repositories { + maven { + url "${CI_API_V4_URL}/groups/<group>/-/packages/maven" + name "GitLab" + credentials(HttpHeaderCredentials) { + name = 'Job-Token' + value = System.getenv("CI_JOB_TOKEN") + } + authentication { + header(HttpHeaderAuthentication) + } + } +} +``` + +Or add it to your `build.gradle.kts` file if you are using Kotlin DSL: + +```kotlin +repositories { + maven { + url = uri("$CI_API_V4_URL/groups/<group>/-/packages/maven") + name = "GitLab" + credentials(HttpHeaderCredentials::class) { + name = "Job-Token" + value = System.getenv("CI_JOB_TOKEN") + } + authentication { + create("header", HttpHeaderAuthentication::class) + } + } +} +``` + +### Naming convention + +You can use one of three API endpoints to install a Maven package. You must publish a package to a project, but note which endpoint +you use to install the package. The option you choose determines the settings you add to your `pom.xml` file for publishing. + +The three endpoints are: + +- **Project-level**: Use when you have a few Maven packages that are not in the same GitLab group. +- **Group-level**: Use when installing packages from many different projects in the same GitLab group. GitLab does not guarantee the uniqueness of package names in the group. You can have two projects with the same package name and package version. As a result, GitLab serves whichever one is more recent. +- **Instance-level**: Use when installing many packages from different GitLab groups or in their own namespace. + +**Only packages with the same path as the project** are exposed by the instance-level endpoint. + +| Project | Package | Instance-level endpoint available | +| ------------------- | -------------------------------- | --------------------------------- | +| `foo/bar` | `foo/bar/1.0-SNAPSHOT` | Yes | +| `gitlab-org/gitlab` | `foo/bar/1.0-SNAPSHOT` | No | +| `gitlab-org/gitlab` | `gitlab-org/gitlab/1.0-SNAPSHOT` | Yes | + +#### Endpoint URLs + +| Endpoint | Endpoint URL | Additional information | +| -------- | ------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------- | +| Project | `https://gitlab.example.com/api/v4/projects/<project_id>/packages/maven` | Replace `gitlab.example.com` with your domain name. Replace `<project_id>` with your project ID found on your project's homepage. | +| Group | `https://gitlab.example.com/api/v4/groups/<group_id>/-/packages/maven` | Replace `gitlab.example.com` with your domain name. Replace `<group_id>` with your group ID found on your group's homepage. | +| Instance | `https:///gitlab.example.com/api/v4/packages/maven` | Replace `gitlab.example.com` with your domain name. | + +In all cases, to publish a package, you need: + +- A project-specific URL in the `distributionManagement` section. +- A `repository` and `distributionManagement` section. + +### Edit the Groovy DSL or Kotlin DSL + +The Gradle Groovy DSL `repositories` section should look like this: + +```groovy +repositories { + maven { + url "<your_endpoint_url>" + name "GitLab" + } +} +``` + +In Kotlin DSL: + +```kotlin +repositories { + maven { + url = uri("<your_endpoint_url>") + name = "GitLab" + } +} +``` + +- Replace `<your_endpoint_url>` with the [endpoint](#endpoint-urls) you chose. + +## Publish using Gradle + +Your token name depends on which token you use. + +| Token type | Token name | +| --------------------- | --------------- | +| Personal access token | `Private-Token` | +| Deploy token | `Deploy-Token` | + +To publish a package by using Gradle: + +1. Add the Gradle plugin [`maven-publish`](https://docs.gradle.org/current/userguide/publishing_maven.html) to the plugins section: + + In Groovy DSL: + + ```groovy + plugins { + id 'java' + id 'maven-publish' + } + ``` + + In Kotlin DSL: + + ```kotlin + plugins { + java + `maven-publish` + } + ``` + +1. Add a `publishing` section: + + In Groovy DSL: + + ```groovy + publishing { + publications { + library(MavenPublication) { + from components.java + } + } + repositories { + maven { + url "https://gitlab.example.com/api/v4/projects/<PROJECT_ID>/packages/maven" + credentials(HttpHeaderCredentials) { + name = "REPLACE_WITH_TOKEN_NAME" + value = gitLabPrivateToken // the variable resides in $GRADLE_USER_HOME/gradle.properties + } + authentication { + header(HttpHeaderAuthentication) + } + } + } + } + ``` + + In Kotlin DSL: + + ```kotlin + publishing { + publications { + create<MavenPublication>("library") { + from(components["java"]) + } + } + repositories { + maven { + url = uri("https://gitlab.example.com/api/v4/projects/<PROJECT_ID>/packages/maven") + credentials(HttpHeaderCredentials::class) { + name = "REPLACE_WITH_TOKEN_NAME" + value = + findProperty("gitLabPrivateToken") as String? // the variable resides in $GRADLE_USER_HOME/gradle.properties + } + authentication { + create("header", HttpHeaderAuthentication::class) + } + } + } + } + ``` + +1. Replace `PROJECT_ID` with your project ID, which you can find on your project's home page. + +1. Run the publish task: + + ```shell + gradle publish + ``` + +Go to your project's **Packages and registries** page and view the published packages. + +## Install a package + +To install a package from the GitLab Package Registry, you must configure +the [remote and authenticate](#authenticate-to-the-package-registry-with-gradle). +After configuring the remote and authenticate, you can install a package from a project, group, or namespace. + +If multiple packages have the same name and version, when you install +a package, the most recently-published package is retrieved. + +Add a [dependency](https://docs.gradle.org/current/userguide/declaring_dependencies.html) to `build.gradle` in the dependencies section: + +```groovy +dependencies { + implementation 'com.mycompany.mydepartment:my-project:1.0-SNAPSHOT' +} +``` + +Or to `build.gradle.kts` if you are using Kotlin DSL: + +```kotlin +dependencies { + implementation("com.mycompany.mydepartment:my-project:1.0-SNAPSHOT") +} +``` + +## Helpful hints + +For the complete list of helpful hints, see the [Maven documentation](../maven_repository/index.md#helpful-hints). + +### Create Maven packages with GitLab CI/CD by using Gradle + +You can create a package each time the `main` branch +is updated. + +1. Authenticate with [a CI job token in Gradle](#authenticate-with-a-ci-job-token-in-gradle). + +1. Add a `deploy` job to your `.gitlab-ci.yml` file: + + ```yaml + deploy: + image: gradle:6.5-jdk11 + script: + - 'gradle publish' + only: + - main + ``` + +1. Commit files to your repository. + +When the pipeline is successful, the Maven package is created. + +### Publishing a package with the same name or version + +When you publish a package with the same name and version as an existing package, the new package +files are added to the existing package. You can still use the UI or API to access and view the +existing package's older assets. + +Consider using the Packages API or the UI to delete older package versions. + +### Do not allow duplicate Maven packages + +To prevent users from publishing duplicate Maven packages, you can use the [GraphQl API](../../../api/graphql/reference/index.md#packagesettings) or the UI. + +In the UI: + +1. For your group, go to **Settings > Packages and registries**. +1. Expand the **Package Registry** section. +1. Turn on the **Do not allow duplicates** toggle. +1. Optional. To allow some duplicate packages, in the **Exceptions** box, enter a regex pattern that matches the names and/or versions of packages you want to allow. + +Your changes are automatically saved. + +### Request forwarding to Maven Central + +FLAG: +By default, this feature is not available for self-managed. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `maven_central_request_forwarding`. +This feature is not available for SaaS users. + +When a Maven package is not found in the Package Registry, the request is forwarded +to [Maven Central](https://search.maven.org/). + +When the feature flag is enabled, administrators can disable this behavior in the +[Continuous Integration settings](../../admin_area/settings/continuous_integration.md). + +There are many ways to configure your Maven project to request packages +in Maven Central from GitLab. Maven repositories are queried in a +[specific order](https://maven.apache.org/guides/mini/guide-multiple-repositories.html#repository-order). +By default, maven-central is usually checked first through the +[Super POM](https://maven.apache.org/guides/introduction/introduction-to-the-pom.html#Super_POM), so +GitLab needs to be configured to be queried before maven-central. + +[Using GitLab as a mirror of the central proxy](../maven_repository/index.md#setting-gitlab-as-a-mirror-for-the-central-proxy) is one +way to force GitLab to be queried in place of maven-central. + +Maven forwarding is restricted to only the project level and +group level [endpoints](#naming-convention). The instance-level endpoint +has naming restrictions that prevent it from being used for packages that don't follow that convention and also +introduces too much security risk for supply-chain style attacks. diff --git a/doc/user/packages/helm_repository/index.md b/doc/user/packages/helm_repository/index.md index bba68494c2d..785ef344c8e 100644 --- a/doc/user/packages/helm_repository/index.md +++ b/doc/user/packages/helm_repository/index.md @@ -72,7 +72,7 @@ Once built, a chart can be uploaded to the desired channel with `curl` or `helm ### Release channels You can publish Helm charts to channels in GitLab. Channels are a method you can use to differentiate Helm chart repositories. -For example, you can use `stable` and `devel` as channels to allow users to add the `stable` repo while `devel` charts are isolated. +For example, you can use `stable` and `devel` as channels to allow users to add the `stable` repository while `devel` charts are isolated. ## Use CI/CD to publish a Helm package diff --git a/doc/user/packages/maven_repository/index.md b/doc/user/packages/maven_repository/index.md index 2aa71e111fb..c6c2f238564 100644 --- a/doc/user/packages/maven_repository/index.md +++ b/doc/user/packages/maven_repository/index.md @@ -4,9 +4,7 @@ group: Package Registry info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Maven packages in the Package Repository **(FREE)** - -> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) from GitLab Premium to GitLab Free in 13.3. +# Maven packages in the Package Registry **(FREE)** Publish [Maven](https://maven.apache.org) artifacts in your project's Package Registry. Then, install the packages whenever you need to use them as a dependency. @@ -14,75 +12,29 @@ Then, install the packages whenever you need to use them as a dependency. For documentation of the specific API endpoints that the Maven package manager client uses, see the [Maven API documentation](../../../api/packages/maven.md). -Learn how to build a [Maven](../workflows/build_packages.md#maven) or [Gradle](../workflows/build_packages.md#gradle) package. - -## Authenticate to the Package Registry with Maven - -To authenticate to the Package Registry, you need one of the following: +Learn how to build a [Maven](../workflows/build_packages.md#maven) package. -- A [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api`. -- A [deploy token](../../project/deploy_tokens/index.md) with the scope set to `read_package_registry`, `write_package_registry`, or both. -- A [CI_JOB_TOKEN](#authenticate-with-a-ci-job-token-in-maven). +## Publish to the GitLab Package Registry -### Authenticate with a personal access token in Maven - -To use a personal access token, add this section to your -[`settings.xml`](https://maven.apache.org/settings.html) file. +### Authenticate to the Package Registry -The `name` must be `Private-Token`. - -```xml -<settings> - <servers> - <server> - <id>gitlab-maven</id> - <configuration> - <httpHeaders> - <property> - <name>Private-Token</name> - <value>REPLACE_WITH_YOUR_PERSONAL_ACCESS_TOKEN</value> - </property> - </httpHeaders> - </configuration> - </server> - </servers> -</settings> -``` +You need an token to publish a package. There are different tokens available depending on what you're trying to achieve. For more information, review the [guidance on tokens](../package_registry/index.md#authenticate-with-the-registry). -### Authenticate with a deploy token in Maven +Create a token and save it to use later in the process. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213566) deploy token authentication in GitLab 13.0. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) from GitLab Premium to GitLab Free in 13.3. +### Edit the `settings.xml` -To use a deploy token, add this section to your +Add the following section to your [`settings.xml`](https://maven.apache.org/settings.html) file. -The `name` must be `Deploy-Token`. - -```xml -<settings> - <servers> - <server> - <id>gitlab-maven</id> - <configuration> - <httpHeaders> - <property> - <name>Deploy-Token</name> - <value>REPLACE_WITH_YOUR_DEPLOY_TOKEN</value> - </property> - </httpHeaders> - </configuration> - </server> - </servers> -</settings> -``` - -### Authenticate with a CI job token in Maven - -To authenticate with a CI job token, add this section to your -[`settings.xml`](https://maven.apache.org/settings.html) file. +NOTE: +The `<name>` field must be named to match the token you chose. -The `name` must be `Job-Token`. +| Token type | Name must be | Token | +| --------------------- | --------------- | ---------------------------------------------------------------------- | +| Personal access token | `Private-Token` | Paste token as-is, or define an environment variable to hold the token | +| Deploy token | `Deploy-Token` | Paste token as-is, or define an environment variable to hold the token | +| CI Job token | `Job-Token` | `${CI_JOB_TOKEN}` | ```xml <settings> @@ -92,8 +44,8 @@ The `name` must be `Job-Token`. <configuration> <httpHeaders> <property> - <name>Job-Token</name> - <value>${CI_JOB_TOKEN}</value> + <name>REPLACE_WITH_NAME</name> + <value>REPLACE_WITH_TOKEN</value> </property> </httpHeaders> </configuration> @@ -102,361 +54,70 @@ The `name` must be `Job-Token`. </settings> ``` -Read more about [how to create Maven packages using GitLab CI/CD](#create-maven-packages-with-gitlab-cicd). - -## Authenticate to the Package Registry with Gradle - -To authenticate to the Package Registry, you need either a personal access token or deploy token. - -- If you use a [personal access token](../../../user/profile/personal_access_tokens.md), set the scope to `api`. -- If you use a [deploy token](../../project/deploy_tokens/index.md), set the scope to `read_package_registry`, `write_package_registry`, or both. - -### Authenticate with a personal access token in Gradle - -In [your `GRADLE_USER_HOME` directory](https://docs.gradle.org/current/userguide/directory_layout.html#dir:gradle_user_home), -create a file `gradle.properties` with the following content: - -```properties -gitLabPrivateToken=REPLACE_WITH_YOUR_PERSONAL_ACCESS_TOKEN -``` - -Add a `repositories` section to your -[`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) -file: - -```groovy -repositories { - maven { - url "https://gitlab.example.com/api/v4/groups/<group>/-/packages/maven" - name "GitLab" - credentials(HttpHeaderCredentials) { - name = 'Private-Token' - value = gitLabPrivateToken - } - authentication { - header(HttpHeaderAuthentication) - } - } -} -``` - -Or add it to your `build.gradle.kts` file if you are using Kotlin DSL: - -```kotlin -repositories { - maven { - url = uri("https://gitlab.example.com/api/v4/groups/<group>/-/packages/maven") - name = "GitLab" - credentials(HttpHeaderCredentials::class) { - name = "Private-Token" - value = findProperty("gitLabPrivateToken") as String? - } - authentication { - create("header", HttpHeaderAuthentication::class) - } - } -} -``` +### Naming convention -### Authenticate with a deploy token in Gradle - -To authenticate with a deploy token, add a `repositories` section to your -[`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) -file: - -```groovy -repositories { - maven { - url "https://gitlab.example.com/api/v4/groups/<group>/-/packages/maven" - name "GitLab" - credentials(HttpHeaderCredentials) { - name = 'Deploy-Token' - value = '<deploy-token>' - } - authentication { - header(HttpHeaderAuthentication) - } - } -} -``` +You can use one of three endpoints to install a Maven package. You must publish a package to a project, but the endpoint you choose determines the settings you add to your `pom.xml` file for publishing. -Or add it to your `build.gradle.kts` file if you are using Kotlin DSL: - -```kotlin -repositories { - maven { - url = uri("https://gitlab.example.com/api/v4/groups/<group>/-/packages/maven") - name = "GitLab" - credentials(HttpHeaderCredentials::class) { - name = "Deploy-Token" - value = "<deploy-token>" - } - authentication { - create("header", HttpHeaderAuthentication::class) - } - } -} -``` +The three endpoints are: -### Authenticate with a CI job token in Gradle - -To authenticate with a CI job token, add a `repositories` section to your -[`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) -file: - -```groovy -repositories { - maven { - url "${CI_API_V4_URL}/groups/<group>/-/packages/maven" - name "GitLab" - credentials(HttpHeaderCredentials) { - name = 'Job-Token' - value = System.getenv("CI_JOB_TOKEN") - } - authentication { - header(HttpHeaderAuthentication) - } - } -} -``` +- **Project-level**: Use when you have a few Maven packages and they are not in the same GitLab group. +- **Group-level**: Use when you want to install packages from many different projects in the same GitLab group. GitLab does not guarantee the uniqueness of package names within the group. You can have two projects with the same package name and package version. As a result, GitLab serves whichever one is more recent. +- **Instance-level**: Use when you have many packages in different GitLab groups or in their own namespace. -Or add it to your `build.gradle.kts` file if you are using Kotlin DSL: - -```kotlin -repositories { - maven { - url = uri("$CI_API_V4_URL/groups/<group>/-/packages/maven") - name = "GitLab" - credentials(HttpHeaderCredentials::class) { - name = "Job-Token" - value = System.getenv("CI_JOB_TOKEN") - } - authentication { - create("header", HttpHeaderAuthentication::class) - } - } -} -``` +**Only packages that have the same path as the project** are exposed by the instance-level endpoint. -## Use the GitLab endpoint for Maven packages +| Project | Package | Instance-level endpoint available | +| ------------------- | -------------------------------- | --------------------------------- | +| `foo/bar` | `foo/bar/1.0-SNAPSHOT` | Yes | +| `gitlab-org/gitlab` | `foo/bar/1.0-SNAPSHOT` | No | +| `gitlab-org/gitlab` | `gitlab-org/gitlab/1.0-SNAPSHOT` | Yes | -To use the GitLab endpoint for Maven packages, choose an option: +#### Endpoint URLs -- **Project-level**: To publish Maven packages to a project, use a project-level endpoint. - To install Maven packages, use a project-level endpoint when you have few Maven packages - and they are not in the same GitLab group. -- **Group-level**: Use a group-level endpoint when you want to install packages from - many different projects in the same GitLab group. -- **Instance-level**: Use an instance-level endpoint when you want to install many - packages from different GitLab groups or in their own namespace. +| Endpoint | Endpoint URL for `pom.xml` | Additional information | +| -------- | ------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------- | +| Project | `https://gitlab.example.com/api/v4/projects/<project_id>/packages/maven` | Replace `gitlab.example.com` with your domain name. Replace `<project_id>` with your project ID, found on your project's homepage. | +| Group | `https://gitlab.example.com/api/v4/groups/<group_id>/-/packages/maven` | Replace `gitlab.example.com` with your domain name. Replace `<group_id>` with your group ID, found on your group's homepage. | +| Instance | `https:///gitlab.example.com/api/v4/packages/maven` | Replace `gitlab.example.com` with your domain name. | -The option you choose determines the settings you add to your `pom.xml` file. +### Edit the `pom.xml` for publishing -In all cases, to publish a package, you need: +No matter which endpoint you choose, you must have: - A project-specific URL in the `distributionManagement` section. - A `repository` and `distributionManagement` section. -### Project-level Maven endpoint - -The relevant `repository` section of your `pom.xml` -in Maven should look like this: +The relevant `repository` section of your `pom.xml` in Maven should look like this: ```xml <repositories> <repository> <id>gitlab-maven</id> - <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> + <url><your_endpoint_url></url> </repository> </repositories> <distributionManagement> <repository> <id>gitlab-maven</id> - <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> + <url>https://gitlab.example.com/api/v4/projects/<project_id>/packages/maven</url> </repository> <snapshotRepository> <id>gitlab-maven</id> - <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> + <url>https://gitlab.example.com/api/v4/projects/<project_id>/packages/maven</url> </snapshotRepository> </distributionManagement> ``` -The corresponding section in Gradle Groovy DSL would be: - -```groovy -repositories { - maven { - url "https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven" - name "GitLab" - } -} -``` - -In Kotlin DSL: - -```kotlin -repositories { - maven { - url = uri("https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven") - name = "GitLab" - } -} -``` - -- The `id` is what you [defined in `settings.xml`](#authenticate-to-the-package-registry-with-maven). -- The `PROJECT_ID` is your project ID, which you can view on your project's home page. -- Replace `gitlab.example.com` with your domain name. -- For retrieving artifacts, use either the - [URL-encoded](../../../api/index.md#namespaced-path-encoding) path of the project - (like `group%2Fproject`) or the project's ID (like `42`). However, only the - project's ID can be used for publishing. - -### Group-level Maven endpoint - -> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) from GitLab Premium to GitLab Free in 13.3. - -If you rely on many packages, it might be inefficient to include the `repository` section -with a unique URL for each package. Instead, you can use the group-level endpoint for -all the Maven packages stored within one GitLab group. Only packages you have access to -are available for download. - -The group-level endpoint works with any package names, so you -have more flexibility in naming, compared to the [instance-level endpoint](#instance-level-maven-endpoint). -However, GitLab does not guarantee the uniqueness of package names within -the group. You can have two projects with the same package name and package -version. As a result, GitLab serves whichever one is more recent. - -This example shows the relevant `repository` section of your `pom.xml` file. -You still need a project-specific URL for publishing a package in -the `distributionManagement` section: - -```xml -<repositories> - <repository> - <id>gitlab-maven</id> - <url>https://gitlab.example.com/api/v4/groups/GROUP_ID/-/packages/maven</url> - </repository> -</repositories> -<distributionManagement> - <repository> - <id>gitlab-maven</id> - <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> - </repository> - <snapshotRepository> - <id>gitlab-maven</id> - <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> - </snapshotRepository> -</distributionManagement> -``` - -For Gradle, the corresponding `repositories` section in Groovy DSL would look like: - -```groovy -repositories { - maven { - url "https://gitlab.example.com/api/v4/groups/GROUP_ID/-/packages/maven" - name "GitLab" - } -} -``` - -In Kotlin DSL: - -```kotlin -repositories { - maven { - url = uri("https://gitlab.example.com/api/v4/groups/GROUP_ID/-/packages/maven") - name = "GitLab" - } -} -``` - -- For the `id`, use what you [defined in `settings.xml`](#authenticate-to-the-package-registry-with-maven). -- For `GROUP_ID`, use your group ID, which you can view on your group's home page. -- For `PROJECT_ID`, use your project ID, which you can view on your project's home page. +- The `id` is what you [defined in `settings.xml`](#edit-the-settingsxml). +- The `<your_endpoint_url>` depends on which [endpoint](#endpoint-urls) you choose. - Replace `gitlab.example.com` with your domain name. -- For retrieving artifacts, use either the - [URL-encoded](../../../api/index.md#namespaced-path-encoding) path of the group - (like `group%2Fsubgroup`) or the group's ID (like `12`). - -### Instance-level Maven endpoint - -> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) from GitLab Premium to GitLab Free in 13.3. - -If you rely on many packages, it might be inefficient to include the `repository` section -with a unique URL for each package. Instead, you can use the instance-level endpoint for -all Maven packages stored in GitLab. All packages you have access to are available -for download. - -**Only packages that have the same path as the project** are exposed by -the instance-level endpoint. - -| Project | Package | Instance-level endpoint available | -| ------------------- | -------------------------------- | --------------------------------- | -| `foo/bar` | `foo/bar/1.0-SNAPSHOT` | Yes | -| `gitlab-org/gitlab` | `foo/bar/1.0-SNAPSHOT` | No | -| `gitlab-org/gitlab` | `gitlab-org/gitlab/1.0-SNAPSHOT` | Yes | - -This example shows how relevant `repository` section of your `pom.xml`. -You still need a project-specific URL in the `distributionManagement` section. - -```xml -<repositories> - <repository> - <id>gitlab-maven</id> - <url>https://gitlab.example.com/api/v4/packages/maven</url> - </repository> -</repositories> -<distributionManagement> - <repository> - <id>gitlab-maven</id> - <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> - </repository> - <snapshotRepository> - <id>gitlab-maven</id> - <url>https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven</url> - </snapshotRepository> -</distributionManagement> -``` - -The corresponding repositories section in Gradle Groovy DSL would look like: - -```groovy -repositories { - maven { - url "https://gitlab.example.com/api/v4/packages/maven" - name "GitLab" - } -} -``` - -In Kotlin DSL: - -```kotlin -repositories { - maven { - url = uri("https://gitlab.example.com/api/v4/packages/maven") - name = "GitLab" - } -} -``` - -- The `id` is what you [defined in `settings.xml`](#authenticate-to-the-package-registry-with-maven). -- The `PROJECT_ID` is your project ID, which you can view on your project's home page. -- Replace `gitlab.example.com` with your domain name. -- For retrieving artifacts, use either the - [URL-encoded](../../../api/index.md#namespaced-path-encoding) path of the project - (like `group%2Fproject`) or the project's ID (like `42`). However, only the - project's ID can be used for publishing. ## Publish a package -After you have set up the [remote and authentication](#authenticate-to-the-package-registry-with-maven) -and [configured your project](#use-the-gitlab-endpoint-for-maven-packages), +After you have set up the [authentication](#authenticate-to-the-package-registry) +and [chosen an endpoint for publishing](#naming-convention), publish a Maven package to your project. -### Publish by using Maven - To publish a package by using Maven: ```shell @@ -474,122 +135,13 @@ If the deploy is successful, the build success message should be displayed: The message should also show that the package was published to the correct location: ```shell -Uploading to gitlab-maven: https://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.jar +Uploading to gitlab-maven: https://example.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.jar ``` -### Publish by using Gradle - -To publish a package by using Gradle: - -1. Add the Gradle plugin [`maven-publish`](https://docs.gradle.org/current/userguide/publishing_maven.html) to the plugins section: - - In Groovy DSL: - - ```groovy - plugins { - id 'java' - id 'maven-publish' - } - ``` - - In Kotlin DSL: - - ```kotlin - plugins { - java - `maven-publish` - } - ``` - -1. Add a `publishing` section: - - In Groovy DSL: - - ```groovy - publishing { - publications { - library(MavenPublication) { - from components.java - } - } - repositories { - maven { - url "https://gitlab.example.com/api/v4/projects/<PROJECT_ID>/packages/maven" - credentials(HttpHeaderCredentials) { - name = "Private-Token" - value = gitLabPrivateToken // the variable resides in $GRADLE_USER_HOME/gradle.properties - } - authentication { - header(HttpHeaderAuthentication) - } - } - } - } - ``` - - In Kotlin DSL: - - ```kotlin - publishing { - publications { - create<MavenPublication>("library") { - from(components["java"]) - } - } - repositories { - maven { - url = uri("https://gitlab.example.com/api/v4/projects/<PROJECT_ID>/packages/maven") - credentials(HttpHeaderCredentials::class) { - name = "Private-Token" - value = - findProperty("gitLabPrivateToken") as String? // the variable resides in $GRADLE_USER_HOME/gradle.properties - } - authentication { - create("header", HttpHeaderAuthentication::class) - } - } - } - } - ``` - -1. Replace `PROJECT_ID` with your project ID, which can be found on your project's home page. - -1. Run the publish task: - - ```shell - gradle publish - ``` - -Now navigate to your project's **Packages and registries** page and view the published artifacts. - -### Publishing a package with the same name or version - -When you publish a package with the same name and version as an existing package, the new package -files are added to the existing package. You can still use the UI or API to access and view the -existing package's older files. - -To delete these older package versions, consider using the Packages API or the UI. - -#### Do not allow duplicate Maven packages - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/296895) in GitLab 13.9. -> - [Required permissions](https://gitlab.com/gitlab-org/gitlab/-/issues/350682) changed from developer to maintainer in GitLab 15.0. - -To prevent users from publishing duplicate Maven packages, you can use the [GraphQl API](../../../api/graphql/reference/index.md#packagesettings) or the UI. - -In the UI: - -1. For your group, go to **Settings > Packages and registries**. -1. Expand the **Package Registry** section. -1. Turn on the **Reject duplicates** toggle. -1. Optional. To allow some duplicate packages, in the **Exceptions** box, enter a regex pattern that matches the names and/or versions of packages you want to allow. - -Your changes are automatically saved. - ## Install a package To install a package from the GitLab Package Registry, you must configure -the [remote and authenticate](#authenticate-to-the-package-registry-with-maven). +the [remote and authenticate](#authenticate-to-the-package-registry). When this is completed, you can install a package from a project, group, or namespace. @@ -633,8 +185,8 @@ You can install packages by using the Maven `dependency:get` [command](https://m mvn dependency:get -Dartifact=com.nickkipling.app:nick-test-app:1.1-SNAPSHOT -DremoteRepositories=gitlab-maven::::<gitlab endpoint url> -s <path to settings.xml> ``` - - `<gitlab endpoint url>` is the URL of the GitLab [endpoint](#use-the-gitlab-endpoint-for-maven-packages). - - `<path to settings.xml>` is the path to the `settings.xml` file that contains the [authentication details](#authenticate-to-the-package-registry-with-maven). + - `<gitlab endpoint url>` is the URL of the GitLab [endpoint](#endpoint-urls). + - `<path to settings.xml>` is the path to the `settings.xml` file that contains the [authentication details](#edit-the-settingsxml). NOTE: The repository IDs in the command(`gitlab-maven`) and the `settings.xml` file must match. @@ -645,34 +197,34 @@ The message should show that the package is downloading from the Package Registr Downloading from gitlab-maven: http://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.pom ``` -NOTE: -In the GitLab UI, on the Package Registry page for Maven, you can view and copy these commands. +## Helpful hints -### Use Gradle +### Publishing a package with the same name or version -Add a [dependency](https://docs.gradle.org/current/userguide/declaring_dependencies.html) to `build.gradle` in the dependencies section: +When you publish a package with the same name and version as an existing package, the new package +files are added to the existing package. You can still use the UI or API to access and view the +existing package's older assets. -```groovy -dependencies { - implementation 'com.mycompany.mydepartment:my-project:1.0-SNAPSHOT' -} -``` +To delete older package versions, consider using the Packages API or the UI. -Or to `build.gradle.kts` if you are using Kotlin DSL: +### Do not allow duplicate Maven packages -```kotlin -dependencies { - implementation("com.mycompany.mydepartment:my-project:1.0-SNAPSHOT") -} -``` +To prevent users from publishing duplicate Maven packages, you can use the [GraphQl API](../../../api/graphql/reference/index.md#packagesettings) or the UI. -### Request forwarding to Maven Central +In the UI: + +1. For your group, go to **Settings > Packages and registries**. +1. Expand the **Package Registry** section. +1. Turn on the **Do not allow duplicates** toggle. +1. Optional. To allow some duplicate packages, in the **Exceptions** box, enter a regex pattern that matches the names and/or versions of packages you want to allow. + +Your changes are automatically saved. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362657) behind a [feature flag](../../feature_flags.md), disabled by default in GitLab 15.4 +### Request forwarding to Maven Central FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `maven_central_request_forwarding`. -On GitLab.com, this feature is not available. +By default this feature is not available for self-managed. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `maven_central_request_forwarding`. +This feature is not available for SaaS users. When a Maven package is not found in the Package Registry, the request is forwarded to [Maven Central](https://search.maven.org/). @@ -690,8 +242,8 @@ GitLab needs to be configured to be queried before maven-central. [Using GitLab as a mirror of the central proxy](#setting-gitlab-as-a-mirror-for-the-central-proxy) is one way to force GitLab to be queried in place of maven-central. -Maven forwarding is restricted to only the [project level](#project-level-maven-endpoint) and -[group level](#group-level-maven-endpoint) endpoints. The [instance level endpoint](#instance-level-maven-endpoint) +Maven forwarding is restricted to only the project level and +group level [endpoints](#naming-convention). The instance level endpoint has naming restrictions that prevent it from being used for packages that don't follow that convention and also introduces too much security risk for supply-chain style attacks. @@ -710,7 +262,7 @@ section to your `settings.xml`: <httpHeaders> <property> <name>Private-Token</name> - <value>{personal_access_token}</value> + <value><personal_access_token></value> </property> </httpHeaders> </configuration> @@ -720,25 +272,19 @@ section to your `settings.xml`: <mirror> <id>central-proxy</id> <name>GitLab proxy of central repo</name> - <url>https://gitlab.example.com/api/v4/projects/{project_id}/packages/maven</url> + <url>https://gitlab.example.com/api/v4/projects/<project_id>/packages/maven</url> <mirrorOf>central</mirrorOf> </mirror> </mirrors> </settings> ``` -## Remove a package - -For your project, go to **Packages and registries > Package Registry**. - -To remove a package, select the red trash icon or, from the package details, the **Delete** button. - -## Create Maven packages with GitLab CI/CD +### Create Maven packages with GitLab CI/CD After you have configured your repository to use the Package Repository for Maven, you can configure GitLab CI/CD to build new packages automatically. -### Create Maven packages with GitLab CI/CD by using Maven +### Create Maven packages with GitLab CI/CD using Maven You can create a new package each time the `main` branch is updated. @@ -808,37 +354,51 @@ user's home location. In this example: - The user is `root`, because the job runs in a Docker container. - Maven uses the configured CI/CD variables. -### Create Maven packages with GitLab CI/CD by using Gradle +### Version validation -You can create a package each time the `main` branch -is updated. +The version string is validated by using the following regex. -1. Authenticate with [a CI job token in Gradle](#authenticate-with-a-ci-job-token-in-gradle). +```ruby +\A(?!.*\.\.)[\w+.-]+\z +``` -1. Add a `deploy` job to your `.gitlab-ci.yml` file: +You can experiment with the regex and try your version strings on [this regular expression editor](https://rubular.com/r/rrLQqUXjfKEoL6). - ```yaml - deploy: - image: gradle:6.5-jdk11 - script: - - 'gradle publish' - only: - - main - ``` +### Useful Maven command-line options -1. Commit files to your repository. +There are some [Maven command-line options](https://maven.apache.org/ref/current/maven-embedder/cli.html) +that you can use when performing tasks with GitLab CI/CD. -When the pipeline is successful, the package is created. +- File transfer progress can make the CI logs hard to read. + Option `-ntp,--no-transfer-progress` was added in + [3.6.1](https://maven.apache.org/docs/3.6.1/release-notes.html#User_visible_Changes). + Alternatively, look at `-B,--batch-mode` + [or lower level logging changes.](https://stackoverflow.com/questions/21638697/disable-maven-download-progress-indication) -### Version validation +- Specify where to find the `pom.xml` file (`-f,--file`): -The version string is validated by using the following regex. + ```yaml + package: + script: + - 'mvn --no-transfer-progress -f helloworld/pom.xml package' + ``` -```ruby -\A(?!.*\.\.)[\w+.-]+\z -``` +- Specify where to find the user settings (`-s,--settings`) instead of + [the default location](https://maven.apache.org/settings.html). There's also a `-gs,--global-settings` option: + + ```yaml + package: + script: + - 'mvn -s settings/ci.xml package' + ``` -You can play around with the regex and try your version strings on [this regular expression editor](https://rubular.com/r/rrLQqUXjfKEoL6). +### Supported CLI commands + +The GitLab Maven repository supports the following Maven CLI commands: + +- `mvn deploy`: Publish your package to the Package Registry. +- `mvn install`: Install packages specified in your Maven project. +- `mvn dependency:get`: Install a specific package. ## Troubleshooting @@ -870,34 +430,6 @@ mvn deploy \ WARNING: When you set these options, all network requests are logged and a large amount of output is generated. -### Useful Maven command-line options - -There are some [Maven command-line options](https://maven.apache.org/ref/current/maven-embedder/cli.html) -that you can use when performing tasks with GitLab CI/CD. - -- File transfer progress can make the CI logs hard to read. - Option `-ntp,--no-transfer-progress` was added in - [3.6.1](https://maven.apache.org/docs/3.6.1/release-notes.html#User_visible_Changes). - Alternatively, look at `-B,--batch-mode` - [or lower level logging changes.](https://stackoverflow.com/questions/21638697/disable-maven-download-progress-indication) - -- Specify where to find the `pom.xml` file (`-f,--file`): - - ```yaml - package: - script: - - 'mvn --no-transfer-progress -f helloworld/pom.xml package' - ``` - -- Specify where to find the user settings (`-s,--settings`) instead of - [the default location](https://maven.apache.org/settings.html). There's also a `-gs,--global-settings` option: - - ```yaml - package: - script: - - 'mvn -s settings/ci.xml package' - ``` - ### Verify your Maven settings If you encounter issues within CI/CD that relate to the `settings.xml` file, try adding @@ -916,11 +448,3 @@ package: - 'mvn help:system' - 'mvn package' ``` - -## Supported CLI commands - -The GitLab Maven repository supports the following Maven CLI commands: - -- `mvn deploy`: Publish your package to the Package Registry. -- `mvn install`: Install packages specified in your Maven project. -- `mvn dependency:get`: Install a specific package. diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index 5d2efc52ba9..c62999100c1 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -6,250 +6,97 @@ info: To determine the technical writer assigned to the Stage/Group associated w # npm packages in the Package Registry **(FREE)** -> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) from GitLab Premium to GitLab Free in 13.3. - -Publish npm packages in your project's Package Registry. Then install the -packages whenever you need to use them as a dependency. - -Only [scoped](https://docs.npmjs.com/misc/scope/) packages are supported. - -For documentation of the specific API endpoints that the npm package manager -client uses, see the [npm API documentation](../../../api/packages/npm.md). - -WARNING: -Never hardcode GitLab tokens (or any tokens) directly in `.npmrc` files or any other files that can -be committed to a repository. +For documentation of the specific API endpoints that the npm package manager client uses, see the [npm API documentation](../../../api/packages/npm.md). Learn how to build an [npm](../workflows/build_packages.md#npm) or [yarn](../workflows/build_packages.md#yarn) package. -## Use the GitLab endpoint for npm packages - -To use the GitLab endpoint for npm packages, choose an option: +Watch a [video demo](https://youtu.be/yvLxtkvsFDA) of how to publish npm packages to the GitLab Package Registry. -- **Project-level**: Use when you have few npm packages and they are not in - the same GitLab group. The [package naming convention](#package-naming-convention) is not enforced at this level. - Instead, you should use a [scope](https://docs.npmjs.com/cli/v6/using-npm/scope/) for your package. - When you use a scope, the registry URL is [updated](#authenticate-to-the-package-registry) only for that scope. -- **Instance-level**: Use when you have many npm packages in different - GitLab groups or in their own namespace. Be sure to comply with the [package naming convention](#package-naming-convention). +## Publish to GitLab Package Registry -Some features such as [publishing](#publish-an-npm-package) a package is only available on the project-level endpoint. +### Authentication to the Package Registry -## Authenticate to the Package Registry +You need an token to publish a package. There are different tokens available depending on what you're trying to achieve. For more information, review the [guidance on tokens](../../../user/packages/package_registry/index.md#authenticate-with-the-registry). -You must authenticate with the Package Registry when the project -is private. Public projects do not require authentication. +- If your organization uses two factor authentication (2FA), you must use a personal access token with the scope set to `api`. +- If you are publishing a package via CI/CD pipelines, you must use a CI job token. -To authenticate, use one of the following: +Create a token and save it to use later in the process. -- A [personal access token](../../../user/profile/personal_access_tokens.md) - (required for two-factor authentication (2FA)), with the scope set to `api`. -- A [deploy token](../../project/deploy_tokens/index.md), with the scope set to `read_package_registry`, `write_package_registry`, or both. -- It's not recommended, but you can use [OAuth tokens](../../../api/oauth2.md#resource-owner-password-credentials-flow). - Standard OAuth tokens cannot authenticate to the GitLab npm Registry. You must use a personal access token with OAuth headers. -- A [CI job token](#authenticate-with-a-ci-job-token). -- Your npm package name must be in the format of [`@scope/package-name`](#package-naming-convention). - It must match exactly, including the case. +### Naming convention -### Authenticate with a personal access token or deploy token +Depending on how the package will be installed, you may need to adhere to the naming convention. -To authenticate with the Package Registry, you need a [personal access token](../../profile/personal_access_tokens.md) or [deploy token](../../project/deploy_tokens/index.md). +You can use one of two API endpoints to install packages: -#### Project-level npm endpoint +- **Instance-level**: Use when you have many npm packages in different GitLab groups or in their own namespace. +- **Project-level**: Use when you have few npm packages and they are not in the same GitLab group. -To use the [project-level](#use-the-gitlab-endpoint-for-npm-packages) npm endpoint, set your npm configuration: +If you plan to install a package through the [project level](#install-from-the-project-level), then you do not have to adhere to the naming convention. -```shell -# Set URL for your scoped packages. -# For example package with name `@foo/bar` will use this URL for download -npm config set @foo:registry https://gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/ - -# Add the token for the scoped packages URL. Replace <your_project_id> -# with the project where your package is located. -npm config set -- '//gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/:_authToken' "<your_token>" -``` - -- `<your_project_id>` is your project ID, found on the project's home page. -- `<your_token>` is your personal access token or deploy token. -- Replace `gitlab.example.com` with your domain name. +If you plan to install a package through the [instance level](#install-from-the-instance-level), then you must name your package with a [scope](https://docs.npmjs.com/misc/scope/). Scoped packages begin with a `@` have the format of `@owner/package-name`. You can set up the scope for your package in the `.npmrc` file and by using the `publishConfig` option in the `package.json`. -You should now be able to publish and install npm packages in your project. +- The value used for the `@scope` is the root of the project that will host the packages and not the root + of the project with the source code of the package itself. The scope should be lowercase. +- The package name can be anything you want -If you encounter an error with [Yarn](https://classic.yarnpkg.com/en/), view -[troubleshooting steps](#troubleshooting). +| Project URL | Package Registry in | Scope | Full package name | +| ------------------------------------------------------- | ------------------- | --------- | ---------------------- | +| `https://gitlab.com/my-org/engineering-group/analytics` | Analytics | `@my-org` | `@my-org/package-name` | -#### Instance-level npm endpoint - -NOTE: -Note: Using `CI_JOB_TOKEN` to install npm packages with dependencies in another project will give you 404 errors. You can use a [personal access token](../../profile/personal_access_tokens.md) as a workaround. [GitLab-#352962](https://gitlab.com/gitlab-org/gitlab/-/issues/352962) proposes a fix to this bug. - -To use the [instance-level](#use-the-gitlab-endpoint-for-npm-packages) npm endpoint, set your npm configuration: +Make sure that the name of your package in the `package.json` file matches this convention: ```shell -# Set URL for your scoped packages. -# For example package with name `@foo/bar` will use this URL for download -npm config set @foo:registry https://gitlab.example.com/api/v4/packages/npm/ - -# Add the token for the scoped packages URL. This will allow you to download -# `@foo/` packages from private projects. -npm config set -- '//gitlab.example.com/api/v4/packages/npm/:_authToken' "<your_token>" -``` - -- `<your_token>` is your personal access token or deploy token. -- Replace `gitlab.example.com` with your domain name. - -You should now be able to install npm packages in your project. - -If you encounter an error with [Yarn](https://classic.yarnpkg.com/en/), view -[troubleshooting steps](#troubleshooting). - -### Authenticate with a CI job token - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9104) in GitLab 12.5. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) from GitLab Premium to GitLab Free in 13.3. - -If you're using npm with GitLab CI/CD, a CI job token can be used instead of a personal access token or deploy token. -The token inherits the permissions of the user that generates the pipeline. - -#### Project-level npm endpoint - -To use the [project-level](#use-the-gitlab-endpoint-for-npm-packages) npm endpoint, add a corresponding section to your `.npmrc` file: - -```ini -@foo:registry=https://gitlab.example.com/api/v4/projects/${CI_PROJECT_ID}/packages/npm/ -//gitlab.example.com/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN} +"name": "@my-org/package-name" ``` -#### Instance-level npm endpoint - -To use the [instance-level](#use-the-gitlab-endpoint-for-npm-packages) npm endpoint, add a corresponding section to your `.npmrc` file: - -```ini -@foo:registry=https://gitlab.example.com/api/v4/packages/npm/ -//gitlab.example.com/api/v4/packages/npm/:_authToken=${CI_JOB_TOKEN} -``` +## Publishing a package via the command line -#### Use variables to avoid hard-coding auth token values +### Authenticating via the `.npmrc` -To avoid hard-coding the `authToken` value, you may use a variable in its place: +Create or edit the `.npmrc` file in the same directory as your `package.json`. Include the following lines in the `.npmrc` file: ```shell -npm config set -- '//gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/:_authToken' "${NPM_TOKEN}" -npm config set -- '//gitlab.example.com/api/v4/packages/npm/:_authToken' "${NPM_TOKEN}" +@scope:registry=https://your_domain_name/api/v4/projects/your_project_id/packages/npm/ +//your_domain_name/api/v4/projects/your_project_id/packages/npm/:_authToken="${NPM_TOKEN}" ``` -Then, you can run `npm publish` either locally or by using GitLab CI/CD. +- Replace `@scope` with the [root level group](#naming-convention) of the project you're publishing to the package to. +- Replace `your_domain_name` with your domain name, for example, `gitlab.com`. +- Replace `your_project_id` is your project ID, found on the project's home page. +- `"${NPM_TOKEN}"` will be associated with the token you created later in the process. -- **Locally:** Export `NPM_TOKEN` before publishing: - - ```shell - NPM_TOKEN=<your_token> npm publish - ``` - -- **GitLab CI/CD:** Set an `NPM_TOKEN` [CI/CD variable](../../../ci/variables/index.md) - under your project's **Settings > CI/CD > Variables**. +WARNING: +Never hardcode GitLab tokens (or any tokens) directly in `.npmrc` files or any other files that can +be committed to a repository. -## Working with private registries +### Publishing a package via the command line -When working with private repositories, you may want to configure additional settings to ensure a secure communication channel: +Associate your [token](#authentication-to-the-package-registry) with the `"${NPM_TOKEN}"` in the `.npmrc`. Replace `your_token` with a deploy token, group access token, project access token, or personal access token. ```shell -# Force npm to always require authentication when accessing the registry, even for GET requests. -npm config set always-auth true -``` - -## Package naming convention - -When you use the [instance-level endpoint](#use-the-gitlab-endpoint-for-npm-packages), only the packages with names in the format of `@scope/package-name` are available. - -- The `@scope` is the root namespace of the GitLab project. To follow npm's convention, it should be - lowercase. However, the GitLab package registry allows for uppercase. Before GitLab 13.10, the - `@scope` had to be a case-sensitive match of the GitLab project's root namespace. This was - problematic because the npm public registry does not allow uppercase letters. GitLab 13.10 relaxes - this requirement and translates uppercase in the GitLab `@scope` to lowercase for npm. For - example, a package `@MyScope/package-name` in GitLab becomes `@myscope/package-name` for npm. -- The `package-name` can be whatever you want. - -NOTE: -The value used for the `@scope` is the root of the project that will end up hosting the packages and not the root -of the project with the source code of the package itself. For example, assume your package source code is located -at `source-code-group/package-code` and deployed to a package registry inside `registries-group/registry-project`. -In this case, the `@scope` needs to be `@registries-group` and not `@source-code-group`. - -For example, if your project is `https://gitlab.example.com/my-org/engineering-group/team-amazing/analytics`, -the root namespace is `my-org`. When you publish a package, it must have `my-org` as the scope. - -| Project | Package | Supported | -| ------------------- | -------------------- | --------- | -| `my-org/bar` | `@my-org/bar` | Yes | -| `my-org/bar/baz` | `@my-org/baz` | Yes | -| `My-Org/Bar/baz` | `@my-org/Baz` | Yes | -| `My-Org/Bar/baz` | `@My-Org/Baz` | Yes | -| `my-org/bar/buz` | `@my-org/anything` | Yes | -| `gitlab-org/gitlab` | `@gitlab-org/gitlab` | Yes | -| `gitlab-org/gitlab` | `@foo/bar` | No | - -In GitLab, this regex validates all package names from all package managers: - -```plaintext -/\A\@?(([\w\-\.\+]*)\/)*([\w\-\.]+)@?(([\w\-\.\+]*)\/)*([\w\-\.]*)\z/ +NPM_TOKEN=your_token npm publish ``` -This regex allows almost all of the characters that npm allows, with a few exceptions (for example, `~` is not allowed). - -The regex also allows for capital letters, while npm does not. +Your package should now publish to the Package Registry. -## Limitations +## Publishing a package via a CI/CD pipeline -When you update the path of a user or group, or transfer a subgroup or project, -you must remove any npm packages first. You cannot update the root namespace -of a project with npm packages. Make sure you update your `.npmrc` files to follow -the naming convention and run `npm publish` if necessary. +### Authenticating via the `.npmrc` -## Publish an npm package - -Prerequisites: - -- [Authenticate](#authenticate-to-the-package-registry) to the Package Registry. -- Set a [project-level npm endpoint](#use-the-gitlab-endpoint-for-npm-packages). - -To upload an npm package to your project, run this command: +Create or edit the `.npmrc` file in the same directory as your `package.json` in a GitLab project. Include the following lines in the `.npmrc` file: ```shell -npm publish +@scope:registry=https://your_domain_name/api/v4/projects/your_project_id/packages/npm/ +//your_domain_name/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN} ``` -To view the package, go to your project's **Packages and registries**. +- Replace `@scope` with the [root level group](#naming-convention) of the project you're publishing to the package to. +- The `${CI_PROJECT_ID}` and `${CI_JOB_TOKEN}` are [predefined variables](../../../ci/variables/predefined_variables.md) that are available in the pipeline and do not need to be replaced. -You can also define `"publishConfig"` for your project in `package.json`. For example: +### Publishing a package via a CI/CD pipeline -```json -{ - "publishConfig": { - "@foo:registry": " https://gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/" - } -} -``` - -This forces the package to publish only to the specified registry. - -If you try to publish a package [with a name that already exists](#publishing-packages-with-the-same-name-or-version) within -a given scope, you get a `403 Forbidden!` error. - -## Publish an npm package by using CI/CD - -Prerequisites: - -- [Authenticate](#authenticate-to-the-package-registry) to the Package Registry. -- Set a [project-level npm endpoint](#use-the-gitlab-endpoint-for-npm-packages). -- Your npm package name must be in the format of [`@scope/package-name`](#package-naming-convention). - It must match exactly, including the case. This is different than the - npm naming convention, but it is required to work with the GitLab Package Registry. - -To work with npm commands within [GitLab CI/CD](../../../ci/index.md), you can use -`CI_JOB_TOKEN` in place of the personal access token or deploy token in your commands. - -An example `.gitlab-ci.yml` file for publishing npm packages: +In the GitLab project that houses your `.npmrc` and `package.json`, edit or create a `.gitlab-ci.yml` file. For example: ```yaml image: node:latest @@ -262,143 +109,105 @@ deploy: script: - echo "//${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN}">.npmrc - npm publish - environment: production ``` -See the -[Publish npm packages to the GitLab Package Registry using semantic-release](../../../ci/examples/semantic-release.md) -step-by-step guide and demo project for a complete example. - -## Configure the GitLab npm registry with Yarn 2 - -You can get started with Yarn 2 by following the [Yarn documentation](https://yarnpkg.com/getting-started/install/). +Your package should now publish to the Package Registry when the pipeline runs. -To publish and install with the project-level npm endpoint, set the following configuration in -`.yarnrc.yml`: +## Install a package -```yaml -npmScopes: - foo: - npmRegistryServer: 'https://gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/' - npmPublishRegistry: 'https://gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/' - -npmRegistries: - //gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/: - npmAlwaysAuth: true - npmAuthToken: '<your_token>' -``` +If multiple packages have the same name and version, when you install a package, the most recently-published package is retrieved. -For the instance-level npm endpoint, use this Yarn 2 configuration in `.yarnrc.yml`: +You can install a package from a GitLab project or instance: -```yaml -npmScopes: - foo: - npmRegistryServer: 'https://gitlab.example.com/api/v4/packages/npm/' - -npmRegistries: - //gitlab.example.com/api/v4/packages/npm/: - npmAlwaysAuth: true - npmAuthToken: '<your_token>' -``` +- **Instance-level**: Use when you have many npm packages in different GitLab groups or in their own namespace. +- **Project-level**: Use when you have few npm packages and they are not in the same GitLab group. -In this configuration: +### Install from the instance level -- Replace `<your_token>` with your personal access token or deploy token. -- Replace `<your_project_id>` with your project's ID, which you can find on the project's home page. -- Replace `gitlab.example.com` with your domain name. -- Your scope is `foo`, without `@`. +WARNING: +In order to install a package from the instance level, the package must have been published following the scoped [naming convention](#naming-convention). -## Publishing packages with the same name or version +1. Authenticate to the Package Registry -You cannot publish a package if a package of the same name and version already exists. -You must delete the existing package first. + If you would like to install a package from a private project, you will need to authenticate to the Package Registry. Skip this step if the project is not private. -This rule has a different impact depending on the package name: + ```shell + npm config set -- //your_domain_name/api/v4/packages/npm/:_authToken=your_token + ``` -- For packages following the [naming convention](#package-naming-convention), you can't publish a - package with a duplicate name and version to the root namespace. -- For packages not following the [naming convention](#package-naming-convention), you can't publish - a package with a duplicate name and version to the project you target with the upload. + - Replace `your_domain_name` with your domain name, for example, `gitlab.com`. + - Replace `your_token` with a deploy token, group access token, project access token, or personal access token. -This aligns with npmjs.org's behavior. However, npmjs.org does not ever let you publish -the same version more than once, even if it has been deleted. +1. Set the registry -## `package.json` limitations + ```shell + npm config set @scope:registry https://your_domain_name.com/api/v4/packages/npm/ + ``` -You can't publish a package if its `package.json` file exceeds 20,000 characters. + - Replace `@scope` with the [root level group](#naming-convention) of the project you're installing to the package from. + - Replace `your_domain_name` with your domain name, for example `gitlab.com`. + - Replace `your_token` with a deploy token, group access token, project access token, or personal access token. -## Install a package +1. Install the package -npm packages are commonly-installed by using the `npm` or `yarn` commands -in a JavaScript project. You can install a package from the scope of a project or instance. + ```shell + npm install @scope/my-package + ``` -If multiple packages have the same name and version, when you install a package, the most recently-published package is retrieved. +### Install from the project level -1. Set the URL for scoped packages. +1. Authenticate to the Package Registry - For [instance-level endpoints](#use-the-gitlab-endpoint-for-npm-packages) run: + If you would like to install a package from a private project, you will need to authenticate to the Package Registry. Skip this step if the project is not private. ```shell - npm config set @foo:registry https://gitlab.example.com/api/v4/packages/npm/ + npm config set -- //your_domain_name/api/v4/projects/your_project_id/packages/npm/:_authToken=your_token ``` - - Replace `@foo` with your scope. - - Replace `gitlab.example.com` with your domain name. + - Replace `your_domain_name` with your domain name, for example, `gitlab.com`. + - Replace `your_project_id` is your project ID, found on the project's home page. + - Replace `your_token` with a deploy token, group access token, project access token, or personal access token. - For [project-level endpoints](#use-the-gitlab-endpoint-for-npm-packages) run: +1. Set the registry ```shell - npm config set @foo:registry https://gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/ + npm config set @scope:registry=https://your_domain_name/api/v4/projects/your_project_id/packages/npm/ ``` - - Replace `@foo` with your scope. - - Replace `gitlab.example.com` with your domain name. - - Replace `<your_project_id>` with your project ID, found on the project's home page. + - Replace `@scope` with the [root level group](#naming-convention) of the project you're installing to the package from. + - Replace `your_domain_name` with your domain name, for example, `gitlab.com`. + - Replace `your_project_id` is your project ID, found on the project's home page. -1. Ensure [authentication](#authenticate-to-the-package-registry) is configured. - -1. To install a package in your project, run: +1. Install the package ```shell - npm install @my-scope/my-package + npm install @scope/my-package ``` - Or if you're using Yarn: +## Helpful hints - ```shell - yarn add @my-scope/my-package - ``` +### Package forwarding to npmjs.com -In [GitLab 12.9 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/55344), -when an npm package is not found in the Package Registry, the request is forwarded to [npmjs.com](https://www.npmjs.com/). +When an npm package is not found in the Package Registry, the request is forwarded to [npmjs.com](https://www.npmjs.com/). Administrators can disable this behavior in the [Continuous Integration settings](../../admin_area/settings/continuous_integration.md). +Group owners can disable this behavior in the group Packages and Registries settings. + ### Install npm packages from other organizations You can route package requests to organizations and users outside of GitLab. -To do this, add lines to your `.npmrc` file. Replace `my-org` with the namespace or group that owns your project's repository, +To do this, add lines to your `.npmrc` file. Replace `@my-other-org` with the namespace or group that owns your project's repository, and use your organization's URL. The name is case-sensitive and must match the name of your group or namespace exactly. -Use environment variables to set up your tokens: `export MY_TOKEN="<your token>"`. - ```shell -@foo:registry=https://gitlab.example.com/api/v4/packages/npm/ -//gitlab.example.com/api/v4/packages/npm/:_authToken=${MY_TOKEN} -//gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/:_authToken=${MY_TOKEN} - -@my-other-org:registry=https://gitlab.example.com/api/v4/packages/npm/ -//gitlab.example.com/api/v4/packages/npm/:_authToken=${MY_TOKEN} -//gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/:_authToken=${MY_TOKEN} +@scope:registry=https://my_domain_name.com/api/v4/packages/npm/ +@my-other-org:registry=https://my_domain_name.example.com/api/v4/packages/npm/ ``` ### npm metadata -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11867) in GitLab 12.6. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) from GitLab Premium to GitLab Free in 13.3. -> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/330929) in GitLab 14.5. - The GitLab Package Registry exposes the following attributes to the npm client. These are similar to the [abbreviated metadata format](https://github.com/npm/registry/blob/9e368cf6aaca608da5b2c378c0d53f475298b916/docs/responses/package-metadata.md#abbreviated-metadata-format): @@ -417,10 +226,7 @@ These are similar to the [abbreviated metadata format](https://github.com/npm/re - `engines` - `_hasShrinkwrap` -## Add npm distribution tags - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9425) in GitLab 12.8. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) from GitLab Premium to GitLab Free in 13.3. +### Add npm distribution tags You can add [distribution tags](https://docs.npmjs.com/cli/dist-tag/) to newly-published packages. Tags are optional and can be assigned to only one package at a time. @@ -443,87 +249,46 @@ View [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/258835) for deta Due to a bug in npm 6.9.0, deleting distribution tags fails. Make sure your npm version is 6.9.1 or later. -## Troubleshooting - -When troubleshooting npm issues, first run the same command with the `--verbose` flag to confirm -what registry you are hitting. - -To improve performance, npm caches files related to a package. Note that npm doesn't remove data by -itself. The cache grows as new packages are installed. If you encounter issues, clear the cache with -this command: - -```shell -npm cache clean --force -``` - -### Error running Yarn with the Package Registry for npm registry - -If you are using [Yarn](https://classic.yarnpkg.com/en/) with the npm registry, you may get -an error message like: - -```shell -yarn install v1.15.2 -warning package.json: No license field -info No lockfile found. -warning XXX: No license field -[1/4] 🔍 Resolving packages... -[2/4] 🚚 Fetching packages... -error An unexpected error occurred: "https://gitlab.example.com/api/v4/projects/XXX/packages/npm/XXX/XXX/-/XXX/XXX-X.X.X.tgz: Request failed \"404 Not Found\"". -info If you think this is a bug, please open a bug report with the information provided in "/Users/XXX/gitlab-migration/module-util/yarn-error.log". -info Visit https://classic.yarnpkg.com/en/docs/cli/install for documentation about this command -``` - -In this case, try adding this to your `.npmrc` file (and replace `<your_token>` -with your personal access token or deploy token): +### Supported CLI commands -```plaintext -//gitlab.example.com/api/v4/projects/:_authToken=<your_token> -``` +The GitLab npm repository supports the following commands for the npm CLI (`npm`) and yarn CLI +(`yarn`): -You can also use `yarn config` instead of `npm config` when setting your auth-token dynamically: +- `npm install`: Install npm packages. +- `npm publish`: Publish an npm package to the registry. +- `npm dist-tag add`: Add a dist-tag to an npm package. +- `npm dist-tag ls`: List dist-tags for a package. +- `npm dist-tag rm`: Delete a dist-tag. +- `npm ci`: Install npm packages directly from your `package-lock.json` file. +- `npm view`: Show package metadata. -```shell -yarn config set '//gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/:_authToken' "<your_token>" -yarn config set '//gitlab.example.com/api/v4/packages/npm/:_authToken' "<your_token>" -``` +## Troubleshooting ### `npm publish` targets default npm registry (`registry.npmjs.org`) Ensure that your package scope is set consistently in your `package.json` and `.npmrc` files. -For example, if your project name in GitLab is `foo/my-package`, then your `package.json` file +For example, if your project name in GitLab is `@scope/my-package`, then your `package.json` file should look like: ```json { - "name": "@foo/my-package", - "version": "1.0.0", - "description": "Example package for GitLab npm registry" + "name": "@scope/my-package" } ``` And the `.npmrc` file should look like: -```ini -//gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/:_authToken=<your_token> -//gitlab.example.com/api/v4/packages/npm/:_authToken=<your_token> -@foo:registry=https://gitlab.example.com/api/v4/packages/npm/ -``` - -### `npm install` returns `Error: Failed to replace env in config: ${npm_TOKEN}` - -You do not need a token to run `npm install` unless your project is private. The token is only required to publish. If the `.npmrc` file was checked in with a reference to `$npm_TOKEN`, you can remove it. If you prefer to leave the reference in, you must set a value prior to running `npm install` or set the value by using [GitLab CI/CD variables](../../../ci/variables/index.md): - ```shell -NPM_TOKEN=<your_token> npm install +@scope:registry=https://your_domain_name/api/v4/projects/your_project_id/packages/npm/ +//your_domain_name/api/v4/projects/your_project_id/packages/npm/:_authToken="${NPM_TOKEN}" ``` ### `npm install` returns `npm ERR! 403 Forbidden` If you get this error, ensure that: -- The Package Registry is enabled in your project settings. Although the Package Registry is enabled - by default, it's possible to [disable it](../package_registry/index.md#disable-the-package-registry). +- The Package Registry is enabled in your project settings. Although the Package Registry is enabled by default, it's possible to [disable it](../package_registry/index.md#disable-the-package-registry). - Your token is not expired and has appropriate permissions. - A package with the same name or version doesn't already exist within the given scope. - The scoped packages URL includes a trailing slash: @@ -534,30 +299,25 @@ If you get this error, ensure that: If you get this error, one of the following problems could be causing it. -#### Package name does not meet the naming convention +### Package name does not meet the naming convention -Your package name may not meet the -[`@scope/package-name` package naming convention](#package-naming-convention). +Your package name may not meet the [`@scope/package-name` package naming convention](#naming-convention). -Ensure the name meets the convention exactly, including the case. -Then try to publish again. +Ensure the name meets the convention exactly, including the case. Then try to publish again. -#### Package already exists +### Package already exists -Your package has already been published to another project in the same -root namespace and therefore cannot be published again using the same name. +Your package has already been published to another project in the same root namespace and therefore cannot be published again using the same name. -This is also true even if the prior published package shares the same name, -but not the version. +This is also true even if the prior published package shares the same name, but not the version. -#### Package JSON file is too large +### Package JSON file is too large -Make sure that your `package.json` file does not [exceed `20,000` characters](#packagejson-limitations). +Make sure that your `package.json` file does not exceed `20,000` characters. ### `npm publish` returns `npm ERR! 500 Internal Server Error - PUT` -This is a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/238950) in GitLab -13.3.x and later. The error in the logs will appear as: +This is a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/238950) in GitLab 13.3.x and later. The error in the logs will appear as: ```plaintext >NoMethodError - undefined method `preferred_language' for #<Rack::Response @@ -572,22 +332,7 @@ This might be accompanied by another error: This is usually a permissions issue with either: - `'packages_storage_path'` default `/var/opt/gitlab/gitlab-rails/shared/packages/`. -- The remote bucket if [object storage](../../../administration/packages/index.md#using-object-storage) +- The remote bucket if [object storage](../../../administration/packages/index.md#use-object-storage) is used. In the latter case, ensure the bucket exists and GitLab has write access to it. - -## Supported CLI commands - -The GitLab npm repository supports the following commands for the npm CLI (`npm`) and yarn CLI -(`yarn`): - -- `npm install`: Install npm packages. -- `npm publish`: Publish an npm package to the registry. -- `npm dist-tag add`: Add a dist-tag to an npm package. -- `npm dist-tag ls`: List dist-tags for a package. -- `npm dist-tag rm`: Delete a dist-tag. -- `npm ci`: Install npm packages directly from your `package-lock.json` file. -- `npm view`: Show package metadata. -- `yarn add`: Install an npm package. -- `yarn update`: Update your dependencies. diff --git a/doc/user/packages/nuget_repository/img/visual_studio_adding_nuget_source.png b/doc/user/packages/nuget_repository/img/visual_studio_adding_nuget_source.png Binary files differdeleted file mode 100644 index 7397403f4bf..00000000000 --- a/doc/user/packages/nuget_repository/img/visual_studio_adding_nuget_source.png +++ /dev/null diff --git a/doc/user/packages/nuget_repository/img/visual_studio_nuget_source_added.png b/doc/user/packages/nuget_repository/img/visual_studio_nuget_source_added.png Binary files differdeleted file mode 100644 index 8c14a14e304..00000000000 --- a/doc/user/packages/nuget_repository/img/visual_studio_nuget_source_added.png +++ /dev/null diff --git a/doc/user/packages/nuget_repository/index.md b/doc/user/packages/nuget_repository/index.md index 956202bb990..540db463f0a 100644 --- a/doc/user/packages/nuget_repository/index.md +++ b/doc/user/packages/nuget_repository/index.md @@ -118,27 +118,25 @@ A project-level endpoint is also required to install NuGet packages from a proje To use the [project-level](#use-the-gitlab-endpoint-for-nuget-packages) NuGet endpoint, add the Package Registry as a source with Visual Studio: 1. Open [Visual Studio](https://visualstudio.microsoft.com/vs/). -1. In Windows, select **File > Options**. On macOS, select **Visual Studio > Preferences**. +1. In Windows, select **Tools > Options**. On macOS, select **Visual Studio > Preferences**. 1. In the **NuGet** section, select **Sources** to view a list of all your NuGet sources. 1. Select **Add**. 1. Complete the following fields: - **Name**: Name for the source. - - **Location**: `https://gitlab.example.com/api/v4/projects/<your_project_id>/packages/nuget/index.json`, + - **Source**: `https://gitlab.example.com/api/v4/projects/<your_project_id>/packages/nuget/index.json`, where `<your_project_id>` is your project ID, and `gitlab.example.com` is your domain name. - - **Username**: Your GitLab username or deploy token username. - - **Password**: Your personal access token or deploy token. - -  1. Select **Save**. +1. When you access the package, you must enter your **Username** and **Password**: -The source is displayed in your list. + - **Username**: Your GitLab username or deploy token username. + - **Password**: Your personal access token or deploy token. - +The source is displayed in your list. -If you get a warning, ensure that the **Location**, **Username**, and +If you get a warning, ensure that the **Source**, **Username**, and **Password** are correct. #### Group-level endpoint @@ -148,27 +146,25 @@ To install a package from a group, use a group-level endpoint. To use the [group-level](#use-the-gitlab-endpoint-for-nuget-packages) NuGet endpoint, add the Package Registry as a source with Visual Studio: 1. Open [Visual Studio](https://visualstudio.microsoft.com/vs/). -1. In Windows, select **File > Options**. On macOS, select **Visual Studio > Preferences**. +1. In Windows, select **Tools > Options**. On macOS, select **Visual Studio > Preferences**. 1. In the **NuGet** section, select **Sources** to view a list of all your NuGet sources. 1. Select **Add**. 1. Complete the following fields: - **Name**: Name for the source. - - **Location**: `https://gitlab.example.com/api/v4/groups/<your_group_id>/-/packages/nuget/index.json`, + - **Source**: `https://gitlab.example.com/api/v4/groups/<your_group_id>/-/packages/nuget/index.json`, where `<your_group_id>` is your group ID, and `gitlab.example.com` is your domain name. - - **Username**: Your GitLab username or deploy token username. - - **Password**: Your personal access token or deploy token. - -  1. Select **Save**. +1. When you access the package, you must enter your **Username** and **Password**. -The source is displayed in your list. + - **Username**: Your GitLab username or deploy token username. + - **Password**: Your personal access token or deploy token. - +The source is displayed in your list. -If you get a warning, ensure that the **Location**, **Username**, and +If you get a warning, ensure that the **Source**, **Username**, and **Password** are correct. ### Add a source with the .NET CLI diff --git a/doc/user/packages/package_registry/index.md b/doc/user/packages/package_registry/index.md index 1aeb98fd48a..caa305999c5 100644 --- a/doc/user/packages/package_registry/index.md +++ b/doc/user/packages/package_registry/index.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/221259) from GitLab Premium to GitLab Free in 13.3. With the GitLab Package Registry, you can use GitLab as a private or public registry for a variety -of [supported package managers](#supported-package-managers). +of [supported package managers](supported_package_managers.md). You can publish and share packages, which can be consumed as a dependency in downstream projects. ## Package workflows @@ -60,6 +60,9 @@ For most package types, the following credential types are valid: allows access to packages in the project running the job for the users running the pipeline. Access to other external projects can be configured. +- If your organization uses two factor authentication (2FA), you must use a personal access token with the scope set to `api`. +- If you are publishing a package via CI/CD pipelines, you must use a CI job token. + NOTE: If you have not activated the "Packages" feature for your project at **Settings > General > Project features**, you will receive a 403 Forbidden response. Accessing package registry via deploy token is not available when external authorization is enabled. @@ -78,7 +81,7 @@ Learn more about using the GitLab Package Registry with CI/CD: - [Conan](../conan_repository/index.md#publish-a-conan-package-by-using-cicd) - [Generic](../generic_packages/index.md#publish-a-generic-package-by-using-cicd) - [Maven](../maven_repository/index.md#create-maven-packages-with-gitlab-cicd) -- [npm](../npm_registry/index.md#publish-an-npm-package-by-using-cicd) +- [npm](../npm_registry/index.md#publishing-a-package-via-a-cicd-pipeline) - [NuGet](../nuget_repository/index.md#publish-a-nuget-package-by-using-cicd) - [PyPI](../pypi_repository/index.md#authenticate-with-a-ci-job-token) - [RubyGems](../rubygems_registry/index.md#authenticate-with-a-ci-job-token) @@ -117,50 +120,40 @@ The **Packages and registries > Package Registry** entry is removed from the sid [Project-level permissions](../../permissions.md) determine actions such as downloading, pushing, or deleting packages. -The visibility of the Package Registry is independent of the repository and can't be controlled from +The visibility of the Package Registry is independent of the repository and can be controlled from your project's settings. For example, if you have a public project and set the repository visibility -to **Only Project Members**, the Package Registry is then public. However, disabling the Package +to **Only Project Members**, the Package Registry is then public. Disabling the Package Registry disables all Package Registry operations. -[GitLab-#329253](https://gitlab.com/gitlab-org/gitlab/-/issues/329253) -proposes adding the ability to control Package Registry visibility from the UI. - -| | | Anonymous<br/>(everyone on internet) | Guest | Reporter, Developer, Maintainer, Owner | -| -------------------- | --------------------- | --------- | ----- | ------------------------------------------ | -| Public project with Package Registry enabled | View Package Registry <br/> and pull packages | Yes | Yes | Yes | -| Internal project with Package Registry enabled | View Package Registry <br/> and pull packages | No | Yes | Yes | -| Private project with Package Registry enabled | View Package Registry <br/> and pull packages | No | No | Yes | -| Any project with Package Registry disabled | All operations on Package Registry | No | No | No | - -## Supported package managers +| Project visibility | Action | [Role](../../permissions.md#roles) required | +|--------------------|-----------------------|---------------------------------------------------------| +| Public | View Package Registry | `n/a`, everyone on the internet can perform this action | +| Public | Publish a package | Developer or higher | +| Public | Pull a package | `n/a`, everyone on the internet can perform this action | +| Internal | View Package Registry | Guest or higher | +| Internal | Publish a package | Developer or higher | +| Internal | Pull a package | Guest or higher(1) | +| Private | View Package Registry | Reporter or higher | +| Private | Publish a package | Developer or higher | +| Private | Pull a package | Reporter or higher(1) | -WARNING: -Not all package manager formats are ready for production use. To view each format's status, see the -table's **Status** column. +### Allow anyone to pull from Package Registry -The Package Registry supports the following formats: +> Introduced in GitLab 15.7 [with a flag](../../../administration/feature_flags.md) named `package_registry_access_level`. Enabled by default. -| Package type | GitLab version | Status | -| ------------ | -------------- |------- | -| [Maven](../maven_repository/index.md) | 11.3+ | GA | -| [npm](../npm_registry/index.md) | 11.7+ | GA | -| [NuGet](../nuget_repository/index.md) | 12.8+ | GA | -| [PyPI](../pypi_repository/index.md) | 12.10+ | GA | -| [Generic packages](../generic_packages/index.md) | 13.5+ | GA | -| [Composer](../composer_repository/index.md) | 13.2+ | [Beta](https://gitlab.com/groups/gitlab-org/-/epics/6817) | -| [Conan](../conan_repository/index.md) | 12.6+ | [Beta](https://gitlab.com/groups/gitlab-org/-/epics/6816) | -| [Helm](../helm_repository/index.md) | 14.1+ | [Beta](https://gitlab.com/groups/gitlab-org/-/epics/6366) | -| [Debian](../debian_repository/index.md) | 14.2+ | [Alpha](https://gitlab.com/groups/gitlab-org/-/epics/6057) | -| [Go](../go_proxy/index.md) | 13.1+ | [Alpha](https://gitlab.com/groups/gitlab-org/-/epics/3043) | -| [Ruby gems](../rubygems_registry/index.md) | 13.10+ | [Alpha](https://gitlab.com/groups/gitlab-org/-/epics/3200) | +FLAG: +On self-managed GitLab, by default this feature is available. To disable it, +ask an administrator to [disable the feature flag](../../../administration/feature_flags.md) named `package_registry_access_level`. -[Status](../../../policy/alpha-beta-support.md): +If you want to allow anyone (everyone on the internet) to pull from the Package Registry, no matter what the project visibility is, you can use the additional toggle `Allow anyone to pull from Package Registry` that appears when the project visibility is Private or Internal. -- Alpha: behind a feature flag and not officially supported. -- Beta: several known issues that may prevent expected use. -- GA (Generally Available): ready for production use at any scale. +Several known issues exist when you allow anyone to pull from the Package Registry: -You can also use the [API](../../../api/packages.md) to administer the Package Registry. +- Project-level endpoints are supported. Group-level and instance-level endpoints are not supported. Support for group-level endpoints is proposed in [issue 383537](https://gitlab.com/gitlab-org/gitlab/-/issues/383537). +- It does not work with the [Composer](../composer_repository/index.md#install-a-composer-package), because Composer only has a group endpoint. +- It does not work with the [Debian](../debian_repository/index.md#install-a-package) repository. Support for the Debian repository is proposed in [issue 385258](https://gitlab.com/gitlab-org/gitlab/-/issues/385258). +- It does not work with the [Ruby gems](../rubygems_registry/index.md#install-a-ruby-gem) repository. Support for the Ruby gems repository is proposed in [issue 385259](https://gitlab.com/gitlab-org/gitlab/-/issues/385259). +- It will work with Conan, but using [`conan search`](../conan_repository/index.md#search-for-conan-packages-in-the-package-registry) does not work. ## Accepting contributions @@ -170,8 +163,8 @@ guides you through the process. <!-- vale gitlab.Spelling = NO --> -| Format | Status | -| ------ | ------ | +| Format | Status | +| --------- | ------------------------------------------------------------- | | Chef | [#36889](https://gitlab.com/gitlab-org/gitlab/-/issues/36889) | | CocoaPods | [#36890](https://gitlab.com/gitlab-org/gitlab/-/issues/36890) | | Conda | [#36891](https://gitlab.com/gitlab-org/gitlab/-/issues/36891) | diff --git a/doc/user/packages/package_registry/supported_hash_types.md b/doc/user/packages/package_registry/supported_hash_types.md new file mode 100644 index 00000000000..6d7dbf87468 --- /dev/null +++ b/doc/user/packages/package_registry/supported_hash_types.md @@ -0,0 +1,25 @@ +--- +stage: Package +group: Package Registry +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Supported hash types **(FREE)** + +Hash values are used to ensure you are using the correct package. You can view these values in the user interface or with the [API](../../../api/packages.md). + +The Package Registry supports the following hash types: + +| Package type | Supported hashes | +|--------------------------------------------------|----------------------------------| +| [Maven](../maven_repository/index.md) | MD5, SHA1 | +| [npm](../npm_registry/index.md) | SHA1 | +| [NuGet](../nuget_repository/index.md) | not applicable | +| [PyPI](../pypi_repository/index.md) | MD5, SHA256 | +| [Generic packages](../generic_packages/index.md) | SHA256 | +| [Composer](../composer_repository/index.md) | not applicable | +| [Conan](../conan_repository/index.md) | MD5, SHA1 | +| [Helm](../helm_repository/index.md) | not applicable | +| [Debian](../debian_repository/index.md) | MD5, SHA1, SHA256 | +| [Go](../go_proxy/index.md) | MD5, SHA1, SHA256 | +| [Ruby gems](../rubygems_registry/index.md) | MD5, SHA1, SHA256 (gemspec only) | diff --git a/doc/user/packages/package_registry/supported_package_managers.md b/doc/user/packages/package_registry/supported_package_managers.md new file mode 100644 index 00000000000..75b8c95a0fa --- /dev/null +++ b/doc/user/packages/package_registry/supported_package_managers.md @@ -0,0 +1,34 @@ +--- +stage: Package +group: Package Registry +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Supported package managers **(FREE)** + +WARNING: +Not all package manager formats are ready for production use. + +The Package Registry supports the following package manager types: + +| Package type | GitLab version | Status | +| ------------------------------------------------ | -------------- | ---------------------------------------------------------- | +| [Maven](../maven_repository/index.md) | 11.3+ | GA | +| [npm](../npm_registry/index.md) | 11.7+ | GA | +| [NuGet](../nuget_repository/index.md) | 12.8+ | GA | +| [PyPI](../pypi_repository/index.md) | 12.10+ | GA | +| [Generic packages](../generic_packages/index.md) | 13.5+ | GA | +| [Composer](../composer_repository/index.md) | 13.2+ | [Beta](https://gitlab.com/groups/gitlab-org/-/epics/6817) | +| [Conan](../conan_repository/index.md) | 12.6+ | [Beta](https://gitlab.com/groups/gitlab-org/-/epics/6816) | +| [Helm](../helm_repository/index.md) | 14.1+ | [Beta](https://gitlab.com/groups/gitlab-org/-/epics/6366) | +| [Debian](../debian_repository/index.md) | 14.2+ | [Alpha](https://gitlab.com/groups/gitlab-org/-/epics/6057) | +| [Go](../go_proxy/index.md) | 13.1+ | [Alpha](https://gitlab.com/groups/gitlab-org/-/epics/3043) | +| [Ruby gems](../rubygems_registry/index.md) | 13.10+ | [Alpha](https://gitlab.com/groups/gitlab-org/-/epics/3200) | + +[Status](../../../policy/alpha-beta-support.md): + +- Alpha: behind a feature flag and not officially supported. +- Beta: several known issues that may prevent expected use. +- GA (Generally Available): ready for production use at any scale. + +You can also use the [API](../../../api/packages.md) to administer the Package Registry. diff --git a/doc/user/packages/pypi_repository/index.md b/doc/user/packages/pypi_repository/index.md index f6ed9654882..0e2fc7ca7da 100644 --- a/doc/user/packages/pypi_repository/index.md +++ b/doc/user/packages/pypi_repository/index.md @@ -283,6 +283,32 @@ characters are removed. A `pip install` request for `my.package` looks for packages that match any of the three characters, such as `my-package`, `my_package`, and `my....package`. +## Using `requirements.txt` + +If you want pip to access your private registry, add the `--extra-index-url` parameter along with the URL for your registry to your `requirements.txt` file. + +```plaintext +--extra-index-url https://gitlab.example.com/api/v4/projects/<project_id>/packages/pypi/simple +package-name==1.0.0 +``` + +If this is a private registry, you can authenticate in a couple of ways. For example: + +- Using your `requirements.txt` file: + +```plaintext +--extra-index-url https://__token__:<your_personal_token>@gitlab.example.com/api/v4/projects/<project_id>/packages/pypi/simple +package-name==1.0.0 +``` + +- Using a `~/.netrc` file: + +```plaintext +machine gitlab.example.com +login __token__ +password <your_personal_token> +``` + ## Troubleshooting To improve performance, the pip command caches files related to a package. Note that pip doesn't remove data by @@ -293,6 +319,18 @@ this command: pip cache purge ``` +### Multiple `index-url` or `extra-index-url` parameters + +You can define multiple `index-url` and `extra-index-url` parameters. + +If you use the same domain name (such as `gitlab.example.com`) multiple times with different authentication +tokens, `pip` may not be able to find your packages. This problem is due to how `pip` +[registers and stores your tokens](https://github.com/pypa/pip/pull/10904#issuecomment-1126690115) during commands executions. + +To workaround this issue, you can use a [group deploy token](../../project/deploy_tokens/index.md) with the +scope `read_package_registry` from a common parent group for all projects or groups targeted by the +`index-url` and `extra-index-url` values. + ## Supported CLI commands The GitLab PyPI repository supports the following CLI commands: diff --git a/doc/user/packages/terraform_module_registry/index.md b/doc/user/packages/terraform_module_registry/index.md index 2b99ff807ec..9b09d846034 100644 --- a/doc/user/packages/terraform_module_registry/index.md +++ b/doc/user/packages/terraform_module_registry/index.md @@ -50,7 +50,7 @@ error `{"error":"404 Not Found"}`. Example request using a personal access token: ```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" \ +curl --fail-with-body --header "PRIVATE-TOKEN: <your_access_token>" \ --upload-file path/to/file.tgz \ "https://gitlab.example.com/api/v4/projects/<your_project_id>/packages/terraform/modules/my-module/my-system/0.0.1/file" ``` @@ -66,7 +66,7 @@ Example response: Example request using a deploy token: ```shell -curl --header "DEPLOY-TOKEN: <deploy_token>" \ +curl --fail-with-body --header "DEPLOY-TOKEN: <deploy_token>" \ --upload-file path/to/file.tgz \ "https://gitlab.example.com/api/v4/projects/<your_project_id>/packages/terraform/modules/my-module/my-system/0.0.1/file" ``` @@ -127,14 +127,14 @@ upload: script: - TERRAFORM_MODULE_NAME=$(echo "${TERRAFORM_MODULE_NAME}" | tr " _" -) # module-name must not have spaces or underscores, so translate them to hyphens - tar -vczf ${TERRAFORM_MODULE_NAME}-${TERRAFORM_MODULE_SYSTEM}-${TERRAFORM_MODULE_VERSION}.tgz -C ${TERRAFORM_MODULE_DIR} --exclude=./.git . - - 'curl --location --header "JOB-TOKEN: ${CI_JOB_TOKEN}" + - 'curl --fail-with-body --location --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file ${TERRAFORM_MODULE_NAME}-${TERRAFORM_MODULE_SYSTEM}-${TERRAFORM_MODULE_VERSION}.tgz ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/terraform/modules/${TERRAFORM_MODULE_NAME}/${TERRAFORM_MODULE_SYSTEM}/${TERRAFORM_MODULE_VERSION}/file' rules: - if: $CI_COMMIT_TAG ``` -To trigger this upload job, add a Git tag to your commit. Ensure the tag follows the [Semantic Versioning Specification](https://semver.org/) that Terraform requires. The `rules:if: $CI_COMMIT_TAG` ensures that only tagged commits to your repo trigger the module upload job. +To trigger this upload job, add a Git tag to your commit. Ensure the tag follows the [Semantic Versioning Specification](https://semver.org/) that Terraform requires. The `rules:if: $CI_COMMIT_TAG` ensures that only tagged commits to your repository trigger the module upload job. For other ways to control jobs in your CI/CD pipeline, refer to the [`.gitlab-ci.yml`](../../../ci/yaml/index.md) keyword reference. ## Example projects diff --git a/doc/user/packages/workflows/build_packages.md b/doc/user/packages/workflows/build_packages.md index ec971195ea9..eab1e4392e3 100644 --- a/doc/user/packages/workflows/build_packages.md +++ b/doc/user/packages/workflows/build_packages.md @@ -302,7 +302,7 @@ The npm version is shown in the output: ``` 1. Enter responses to the questions. Ensure the **package name** follows - the [naming convention](../npm_registry/index.md#package-naming-convention) and is scoped to the project or group where the registry exists. + the [naming convention](../npm_registry/index.md#naming-convention) and is scoped to the project or group where the registry exists. ## Yarn @@ -334,7 +334,7 @@ The Yarn version is shown in the output: ``` 1. Enter responses to the questions. Ensure the **package name** follows - the [naming convention](../npm_registry/index.md#package-naming-convention) and is scoped to the + the [naming convention](../npm_registry/index.md#naming-convention) and is scoped to the project or group where the registry exists. A `package.json` file is created. diff --git a/doc/user/packages/workflows/project_registry.md b/doc/user/packages/workflows/project_registry.md index df4b087f6d5..371ab83a4fb 100644 --- a/doc/user/packages/workflows/project_registry.md +++ b/doc/user/packages/workflows/project_registry.md @@ -50,15 +50,15 @@ If you're using npm, create an `.npmrc` file. Add the appropriate URL for publis packages to your project. Finally, add a section to your `package.json` file. Follow the instructions in the -[GitLab Package Registry npm documentation](../npm_registry/index.md#authenticate-to-the-package-registry). After +[GitLab Package Registry npm documentation](../npm_registry/index.md#authentication-to-the-package-registry). After you do this, you can publish your npm package to your project using `npm publish`, as described in the -[publishing packages](../npm_registry/index.md#publish-an-npm-package) section. +[publishing packages](../npm_registry/index.md#publish-to-gitlab-package-registry) section. ### Maven If you are using Maven, you update your `pom.xml` file with distribution sections. These updates include the -appropriate URL for your project, as described in the [GitLab Maven Repository documentation](../maven_repository/index.md#project-level-maven-endpoint). -Then, you need to add a `settings.xml` file and [include your access token](../maven_repository/index.md#authenticate-with-a-personal-access-token-in-maven). +appropriate URL for your project, as described in the [GitLab Maven Repository documentation](../maven_repository/index.md#naming-convention). +Then, you need to add a `settings.xml` file and [include your access token](../maven_repository/index.md#authenticate-to-the-package-registry). Now you can [publish Maven packages](../maven_repository/index.md#publish-a-package) to your project. ### Conan diff --git a/doc/user/packages/yarn_repository/index.md b/doc/user/packages/yarn_repository/index.md new file mode 100644 index 00000000000..7e2f45019cd --- /dev/null +++ b/doc/user/packages/yarn_repository/index.md @@ -0,0 +1,248 @@ +--- +stage: Package +group: Package Registry +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Publish packages with Yarn + +Publish npm packages in your project's Package Registry using Yarn. Then install the +packages whenever you need to use them as a dependency. + +Learn how to build a [yarn](../workflows/build_packages.md#yarn) package. + +You can get started with Yarn 2 by following the [Yarn documentation](https://yarnpkg.com/getting-started/install/). + +## Publish to GitLab Package Registry + +### Authentication to the Package Registry + +You need a token to publish a package. Different tokens are available depending on what you're trying to +achieve. For more information, review the [guidance on tokens](../../../user/packages/package_registry/index.md#authenticate-with-the-registry). + +- If your organization uses two-factor authentication (2FA), you must use a personal access token with the scope set to `api`. +- If you publish a package via CI/CD pipelines, you must use a CI job token. + +Create a token and save it to use later in the process. + +### Naming convention + +Depending on how you install the package, you may need to adhere to the naming convention. + +You can use one of two API endpoints to install packages: + +- **Instance-level**: Use when you have many npm packages in different GitLab groups or in their own namespace. +- **Project-level**: Use when you have a few npm packages, and they are not in the same GitLab group. + +If you plan to install a package through the [project level](#install-from-the-project-level), you do not have to +adhere to the naming convention. + +If you plan to install a package through the [instance level](#install-from-the-instance-level), then you must name +your package with a [scope](https://docs.npmjs.com/misc/scope/). Scoped packages begin with a `@` and have the +`@owner/package-name` format. You can set up the scope for your package in the `.yarnrc.yml` file and by using the +`publishConfig` option in the `package.json`. + +- The value used for the `@scope` is the root of the project that hosts the packages and not the root + of the project with the package's source code. The scope should be lowercase. +- The package name can be anything you want + +| Project URL | Package Registry in | Scope | Full package name | +| ------------------------------------------------------- | ------------------- | --------- | ---------------------- | +| `https://gitlab.com/my-org/engineering-group/analytics` | Analytics | `@my-org` | `@my-org/package-name` | + +### Configuring `.yarnrc.yml` to publish from the project level + +To publish with the project-level npm endpoint, set the following configuration in +`.yarnrc.yml`: + +```yaml +npmScopes: + foo: + npmRegistryServer: 'https://<your_domain>/api/v4/projects/<your_project_id>/packages/npm/' + npmPublishRegistry: 'https://<your_domain>/api/v4/projects/<your_project_id>/packages/npm/' + +npmRegistries: + //gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/: + npmAlwaysAuth: true + npmAuthToken: '<your_token>' +``` + +In this configuration: + +- Replace `<your_domain>` with your domain name. +- Replace `<your_project_id>` with your project's ID, which you can find on the project's home page. +- Replace `<your_token>` with a deploy token, group access token, project access token, or personal access token. + +### Configuring `.yarnrc.yml` to publish from the instance level + +For the instance-level npm endpoint, use this Yarn 2 configuration in `.yarnrc.yml`: + +```yaml +npmScopes: + <scope>: + npmRegistryServer: 'https://<your_domain>/api/v4/packages/npm/' + +npmRegistries: + //gitlab.example.com/api/v4/packages/npm/: + npmAlwaysAuth: true + npmAuthToken: '<your_token>' +``` + +In this configuration: + +- Replace `<your_domain>` with your domain name. +- Your scope is `<scope>`, without `@`. +- Replace `<your_token>` with a deploy token, group access token, project access token, or personal access token. + +### Publishing a package via the command line + +Publish a package: + +```shell +npm publish +``` + +Your package should now publish to the Package Registry. + +### Publishing via a CI/CD pipeline + +In the GitLab project that houses your `yarnrc.yml`, edit or create a `.gitlab-ci.yml` file. For example: + +```yaml +image: node:latest + +stages: + - deploy + +deploy: + stage: deploy + script: + - npm publish +``` + +Your package should now publish to the Package Registry when the pipeline runs. + +## Install a package + +If multiple packages have the same name and version, the most recently-published package is retrieved when you install a package. + +You can install a package from a GitLab project or instance: + +- **Instance-level**: Use when you have many npm packages in different GitLab groups or in their own namespace. +- **Project-level**: Use when you have a few npm packages, and they are not in the same GitLab group. + +### Install from the instance level + +WARNING: +You must use packages published with the scoped [naming convention](#naming-convention) when you install a package from the instance level. + +1. Authenticate to the Package Registry + + If you install a package from a private project, you must authenticate to the Package Registry. Skip this step if the project is not private. + + ```shell + npm config set -- //your_domain_name/api/v4/packages/npm/:_authToken=your_token + ``` + + - Replace `your_domain_name` with your domain name, for example, `gitlab.com`. + - Replace `your_token` with a deploy token, group access token, project access token, or personal access token. + +1. Set the registry + + ```shell + npm config set @scope:registry https://your_domain_name.com/api/v4/packages/npm/ + ``` + + - Replace `@scope` with the [root level group](#naming-convention) of the project you're installing to the package from. + - Replace `your_domain_name` with your domain name, for example, `gitlab.com`. + - Replace `your_token` with a deploy token, group access token, project access token, or personal access token. + +1. Install the package + + ```shell + yarn add @scope/my-package + ``` + +### Install from the project level + +1. Authenticate to the Package Registry + + If you install a package from a private project, you must authenticate to the Package Registry. Skip this step if the project is not private. + + ```shell + npm config set -- //your_domain_name/api/v4/projects/your_project_id/packages/npm/:_authToken=your_token + ``` + + - Replace `your_domain_name` with your domain name, for example, `gitlab.com`. + - Replace `your_project_id` is your project ID, found on the project's home page. + - Replace `your_token` with a deploy token, group access token, project access token, or personal access token. + +1. Set the registry + + ```shell + npm config set @scope:registry=https://your_domain_name/api/v4/projects/your_project_id/packages/npm/ + ``` + + - Replace `@scope` with the [root level group](#naming-convention) of the project you're installing to the package from. + - Replace `your_domain_name` with your domain name, for example, `gitlab.com`. + - Replace `your_project_id` is your project ID, found on the project's home page. + +1. Install the package + + ```shell + yarn add @scope/my-package + ``` + +## Helpful hints + +For full helpful hints information, refer to the [npm documentation](../npm_registry/index.md#helpful-hints). + +### Supported CLI commands + +The GitLab npm repository supports the following commands for the npm CLI (`npm`) and yarn CLI +(`yarn`): + +- `npm install`: Install npm packages. +- `npm publish`: Publish an npm package to the registry. +- `npm dist-tag add`: Add a dist-tag to an npm package. +- `npm dist-tag ls`: List dist-tags for a package. +- `npm dist-tag rm`: Delete a dist-tag. +- `npm ci`: Install npm packages directly from your `package-lock.json` file. +- `npm view`: Show package metadata. +- `yarn add`: Install an npm package. +- `yarn update`: Update your dependencies. + +## Troubleshooting + +For full troubleshooting information, refer to the [npm documentation](../npm_registry/index.md#troubleshooting). + +### Error running Yarn with the Package Registry for the npm registry + +If you are using [Yarn](https://classic.yarnpkg.com/en/) with the npm registry, you may get +an error message like: + +```shell +yarn install v1.15.2 +warning package.json: No license field +info No lockfile found. +warning XXX: No license field +[1/4] 🔍 Resolving packages... +[2/4] 🚚 Fetching packages... +error An unexpected error occurred: "https://gitlab.example.com/api/v4/projects/XXX/packages/npm/XXX/XXX/-/XXX/XXX-X.X.X.tgz: Request failed \"404 Not Found\"". +info If you think this is a bug, please open a bug report with the information provided in "/Users/XXX/gitlab-migration/module-util/yarn-error.log". +info Visit https://classic.yarnpkg.com/en/docs/cli/install for documentation about this command +``` + +In this case, try adding this to your `.npmrc` file (and replace `<your_token>` +with your personal access token or deploy token): + +```plaintext +//gitlab.example.com/api/v4/projects/:_authToken=<your_token> +``` + +You can also use `yarn config` instead of `npm config` when setting your auth-token dynamically: + +```shell +yarn config set '//gitlab.example.com/api/v4/projects/<your_project_id>/packages/npm/:_authToken' "<your_token>" +yarn config set '//gitlab.example.com/api/v4/packages/npm/:_authToken' "<your_token>" +``` diff --git a/doc/user/permissions.md b/doc/user/permissions.md index 8e152a8c190..f3702b848fa 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -6,35 +6,35 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Permissions and roles **(FREE)** -Users have different abilities depending on the role they have in a -particular group or project. If a user is both in a project's group and the -project itself, the highest role is used. +When you add a user to a project or group, you assign them a role. +The role determines which actions they can take in GitLab. -On [public and internal projects](../api/projects.md#project-visibility-level), the Guest role -(not to be confused with [Guest user](#free-guest-users)) is not enforced. +If you add a user to both a project's group and the +project itself, the higher role is used. -When a member leaves a team's project, all the assigned [issues](project/issues/index.md) and -[merge requests](project/merge_requests/index.md) are automatically unassigned. +GitLab [administrators](../administration/index.md) have all permissions. -GitLab [administrators](../administration/index.md) receive all permissions. +## Roles -To add or import a user, you can follow the -[project members documentation](project/members/index.md). +The available roles are: -## Principles behind permissions +- Guest (This role applies to [private and internal projects](../user/public_access.md) only.) +- Reporter +- Developer +- Maintainer +- Owner -See our [product handbook on permissions](https://about.gitlab.com/handbook/product/gitlab-the-product/#permissions-in-gitlab). +A user assigned the Guest role has the least permissions, +and the Owner has the most. -## Instance-wide user permissions - -By default, users can create top-level groups and change their -usernames. A GitLab administrator can configure the GitLab instance to -[modify this behavior](../administration/user_settings.md). +By default, all users can create top-level groups and change their +usernames. A GitLab administrator can [change this behavior](../administration/user_settings.md) +for the GitLab instance. ## Project members permissions -- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/219299) in GitLab 14.8, personal namespace owners appear with Owner role in new projects in their namespace. Introduced [with a flag](../administration/feature_flags.md) named `personal_project_owner_with_owner_access`. Disabled by default. -- [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/351919) in GitLab 14.9. Feature flag `personal_project_owner_with_owner_access` [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/219299). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/219299) in GitLab 14.8, personal namespace owners appear with Owner role in new projects in their namespace. Introduced [with a flag](../administration/feature_flags.md) named `personal_project_owner_with_owner_access`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/351919) in GitLab 14.9. Feature flag `personal_project_owner_with_owner_access` [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/219299). A user's role determines what permissions they have on a project. The Owner role provides all permissions but is available only: @@ -79,14 +79,17 @@ The following table lists project permissions available for each role: | [GitLab Pages](project/pages/index.md):<br>Manage | | | | ✓ | ✓ | | [GitLab Pages](project/pages/index.md):<br>Manage GitLab Pages domains and certificates | | | | ✓ | ✓ | | [GitLab Pages](project/pages/index.md):<br>Remove GitLab Pages | | | | ✓ | ✓ | -| [Incident Management](../operations/incident_management/index.md):<br>View [alerts](../operations/incident_management/alerts.md) | | ✓ | ✓ | ✓ | ✓ | | [Incident Management](../operations/incident_management/index.md):<br>Assign an alert | ✓ | ✓ | ✓ | ✓ | ✓ | -| [Incident Management](../operations/incident_management/index.md):<br>[Change an alert status](../operations/incident_management/alerts.md#change-an-alerts-status) | | ✓ | ✓ | ✓ | ✓ | -| [Incident Management](../operations/incident_management/index.md):<br>View [incident](../operations/incident_management/incidents.md) | ✓ | ✓ | ✓ | ✓ | ✓ | -| [Incident Management](../operations/incident_management/index.md):<br>Create [incident](../operations/incident_management/incidents.md) | | ✓ | ✓ | ✓ | ✓ | -| [Incident Management](../operations/incident_management/index.md):<br>View [on-call schedules](../operations/incident_management/oncall_schedules.md) | | ✓ | ✓ | ✓ | ✓ | | [Incident Management](../operations/incident_management/index.md):<br>Participate in on-call rotation | ✓ | ✓ | ✓ | ✓ | ✓ | +| [Incident Management](../operations/incident_management/index.md):<br>View [incident](../operations/incident_management/incidents.md) | ✓ | ✓ | ✓ | ✓ | ✓ | +| [Incident Management](../operations/incident_management/index.md):<br>Change [alert status](../operations/incident_management/alerts.md#change-an-alerts-status) | | ✓ | ✓ | ✓ | ✓ | +| [Incident Management](../operations/incident_management/index.md):<br>Change [incident severity](../operations/incident_management/manage_incidents.md#change-severity) | | ✓ | ✓ | ✓ | ✓ | +| [Incident Management](../operations/incident_management/index.md):<br>Create [incident](../operations/incident_management/incidents.md) | | ✓ | ✓ | ✓ | ✓ | +| [Incident Management](../operations/incident_management/index.md):<br>View [alerts](../operations/incident_management/alerts.md) | | ✓ | ✓ | ✓ | ✓ | | [Incident Management](../operations/incident_management/index.md):<br>View [escalation policies](../operations/incident_management/escalation_policies.md) | | ✓ | ✓ | ✓ | ✓ | +| [Incident Management](../operations/incident_management/index.md):<br>View [on-call schedules](../operations/incident_management/oncall_schedules.md) | | ✓ | ✓ | ✓ | ✓ | +| [Incident Management](../operations/incident_management/index.md):<br>Change [incident escalation status](../operations/incident_management/manage_incidents.md#change-status) | | | ✓ | ✓ | ✓ | +| [Incident Management](../operations/incident_management/index.md):<br>Change [incident escalation policy](../operations/incident_management/manage_incidents.md#change-escalation-policy) | | | ✓ | ✓ | ✓ | | [Incident Management](../operations/incident_management/index.md):<br>Manage [on-call schedules](../operations/incident_management/oncall_schedules.md) | | | | ✓ | ✓ | | [Incident Management](../operations/incident_management/index.md):<br>Manage [escalation policies](../operations/incident_management/escalation_policies.md) | | | | ✓ | ✓ | | [Issue boards](project/issue_board.md):<br>Create or delete lists | | ✓ | ✓ | ✓ | ✓ | @@ -161,7 +164,7 @@ The following table lists project permissions available for each role: | [Projects](project/index.md):<br>Edit comments (posted by any user) | | | | ✓ | ✓ | | [Projects](project/index.md):<br>Edit project badges | | | | ✓ | ✓ | | [Projects](project/index.md):<br>Edit project settings | | | | ✓ | ✓ | -| [Projects](project/index.md):<br>Export project | | | | ✓ | ✓ | +| [Projects](project/index.md):<br>[Export project](project/settings/import_export.md) | | | | ✓ | ✓ | | [Projects](project/index.md):<br>Manage [project access tokens](project/settings/project_access_tokens.md) (*11*) | | | | ✓ (*20*) | ✓ | | [Projects](project/index.md):<br>Manage [Project Operations](../operations/index.md) | | | | ✓ | ✓ | | [Projects](project/index.md):<br>Rename project | | | | ✓ | ✓ | @@ -204,7 +207,7 @@ The following table lists project permissions available for each role: | [Security dashboard](application_security/security_dashboard/index.md):<br>Use security dashboard | | | ✓ | ✓ | ✓ | | [Security dashboard](application_security/security_dashboard/index.md):<br>View vulnerability | | | ✓ | ✓ | ✓ | | [Security dashboard](application_security/security_dashboard/index.md):<br>View vulnerability findings in [dependency list](application_security/dependency_list/index.md) | | | ✓ | ✓ | ✓ | -| [Tasks](tasks.md):<br>Create (*17*) | ✓ | ✓ | ✓ | ✓ | ✓ | +| [Tasks](tasks.md):<br>Create (*17*) | | ✓ | ✓ | ✓ | ✓ | | [Tasks](tasks.md):<br>Edit | | ✓ | ✓ | ✓ | ✓ | | [Tasks](tasks.md):<br>Remove from issue | | ✓ | ✓ | ✓ | ✓ | | [Tasks](tasks.md):<br>Delete (*21*) | | | | | ✓ | @@ -218,7 +221,7 @@ The following table lists project permissions available for each role: <!-- markdownlint-disable MD029 --> 1. On self-managed GitLab instances, guest users are able to perform this action only on - public and internal projects (not on private projects). [External users](#external-users) + public and internal projects (not on private projects). [External users](admin_area/external_users.md) must be given explicit access even if the project is internal. For GitLab.com, see the [GitLab.com visibility settings](gitlab_com/index.md#visibility-settings). 2. Guest users can only view the [confidential issues](project/issues/confidential_issues.md) they created themselves or are assigned to. @@ -327,29 +330,7 @@ This table shows granted privileges for jobs triggered by specific types of user | Push source and LFS | | | | | 1. Only if the triggering user is not an external one. -1. Only if the triggering user is a member of the project. - -### Wiki and issues - -Project features like [wikis](project/wiki/index.md) and issues can be hidden from users depending on -which visibility level you select on project settings. - -- Disabled: disabled for everyone -- Only team members: only team members can see even if your project is public or internal -- Everyone with access: everyone can see depending on your project's visibility level -- Everyone: enabled for everyone (only available for GitLab Pages) - -### Protected branches - -Additional restrictions can be applied on a per-branch basis with [protected branches](project/protected_branches.md). -Additionally, you can customize permissions to allow or prevent project Developers or Maintainers -from pushing to a protected branch. Read through the documentation on -[protected branches](project/protected_branches.md) to learn more. - -### Value stream analytics permissions - -Find the current permissions on the value stream analytics dashboard, as described in -[related documentation](analytics/value_stream_analytics.md#access-permissions-for-value-stream-analytics). +1. Only if the triggering user is a member of the project. See also [Usage of private Docker images with `if-not-present` pull policy](http://docs.gitlabl.com/runner/security/index.html#usage-of-private-docker-images-with-if-not-present-pull-policy). ### File Locking permissions **(PREMIUM)** @@ -380,6 +361,7 @@ The following table lists group permissions available for each role: | Action | Guest | Reporter | Developer | Maintainer | Owner | |-----------------------------------------------------------------------------------------|-------|----------|-----------|------------|-------| +| Add/remove [child epics](group/epics/manage_epics.md#multi-level-child-epics) | ✓ (8) | ✓ | ✓ | ✓ | ✓ | | Add an issue to an [epic](group/epics/index.md) | ✓ (7) | ✓ (7) | ✓ (7) | ✓ (7) | ✓ (7) | | Browse group | ✓ | ✓ | ✓ | ✓ | ✓ | | Pull a container image using the dependency proxy | ✓ | ✓ | ✓ | ✓ | ✓ | @@ -397,6 +379,7 @@ The following table lists group permissions available for each role: | Pull [packages](packages/index.md) | | ✓ | ✓ | ✓ | ✓ | | Delete [packages](packages/index.md) | | | | ✓ | ✓ | | Create/edit/delete [Maven and generic package duplicate settings](packages/generic_packages/index.md#do-not-allow-duplicate-generic-packages) | | | | ✓ | ✓ | +| Enable/disable package request forwarding | | | | ✓ | ✓ | | Pull a Container Registry image | ✓ (6) | ✓ | ✓ | ✓ | ✓ | | Remove a Container Registry image | | | ✓ | ✓ | ✓ | | View [Group DevOps Adoption](group/devops_adoption/index.md) | | ✓ | ✓ | ✓ | ✓ | @@ -448,6 +431,7 @@ The following table lists group permissions available for each role: 5. In addition, if your group is public or internal, all users who can see the group can also see group wiki pages. 6. Users can only view events based on their individual actions. 7. You must have permission to [view the epic](group/epics/manage_epics.md#who-can-view-an-epic) and edit the issue. +8. You must have permission to [view](group/epics/manage_epics.md#who-can-view-an-epic) the parent and child epics. <!-- markdownlint-enable MD029 --> @@ -460,109 +444,6 @@ nested groups if you have membership in one of its parents. To learn more, read through the documentation on [subgroups memberships](group/subgroups/index.md#subgroup-membership). -## External users **(FREE SELF)** - -In cases where it is desired that a user has access only to some internal or -private projects, there is the option of creating **External Users**. This -feature may be useful when for example a contractor is working on a given -project and should only have access to that project. - -External users: - -- Cannot create project, groups, and snippets in their personal namespaces. -- Can only create projects (including forks), subgroups, and snippets within top-level groups to which they are explicitly granted access. -- Can only access public projects and projects to which they are explicitly granted access, - thus hiding all other internal or private ones from them (like being - logged out). -- Can only access public groups and groups to which they are explicitly granted access, - thus hiding all other internal or private ones from them (like being - logged out). -- Can only access public snippets. - -Access can be granted by adding the user as member to the project or group. -Like usual users, they receive a role in the project or group with all -the abilities that are mentioned in the [permissions table above](#project-members-permissions). -For example, if an external user is added as Guest, and your project is internal or -private, they do not have access to the code; you need to grant the external -user access at the Reporter level or above if you want them to have access to the code. You should -always take into account the -[project's visibility and permissions settings](project/settings/index.md#configure-project-visibility-features-and-permissions) -as well as the permission level of the user. - -NOTE: -External users still count towards a license seat. - -An administrator can flag a user as external by either of the following methods: - -- [Through the API](../api/users.md#user-modification). -- Using the GitLab UI: - 1. On the top bar, select **Main menu > Admin**. - 1. On the left sidebar, select **Overview > Users** to create a new user or edit an existing one. - There, you can find the option to flag the user as external. - -Additionally, users can be set as external users using: - -- [SAML groups](../integration/saml.md#external-groups). -- [LDAP groups](../administration/auth/ldap/ldap_synchronization.md#external-groups). - -### Setting new users to external - -By default, new users are not set as external users. This behavior can be changed -by an administrator: - -1. On the top bar, select **Main menu > Admin**. -1. On the left sidebar, select **Settings > General**. -1. Expand the **Account and limit** section. - -If you change the default behavior of creating new users as external, you -have the option to narrow it down by defining a set of internal users. -The **Internal users** field allows specifying an email address regex pattern to -identify default internal users. New users whose email address matches the regex -pattern are set to internal by default rather than an external collaborator. - -The regex pattern format is in Ruby, but it needs to be convertible to JavaScript, -and the ignore case flag is set (`/regex pattern/i`). Here are some examples: - -- Use `\.internal@domain\.com$` to mark email addresses ending with - `.internal@domain.com` as internal. -- Use `^(?:(?!\.ext@domain\.com).)*$\r?` to mark users with email addresses - not including `.ext@domain.com` as internal. - -WARNING: -Be aware that this regex could lead to a -[regular expression denial of service (ReDoS) attack](https://en.wikipedia.org/wiki/ReDoS). - -## Free Guest users **(ULTIMATE)** - -When a user is given the Guest role on a project, group, or both, and holds no -higher permission level on any other project or group on the GitLab instance, -the user is considered a guest user by GitLab and does not consume a license seat. -There is no other specific "guest" designation for newly created users. - -If the user is assigned a higher role on any projects or groups, the user -takes a license seat. If a user creates a project, the user becomes a Maintainer -on the project, resulting in the use of a license seat. Also, note that if your -project is internal or private, Guest users have all the abilities that are -mentioned in the [permissions table above](#project-members-permissions) (they -are unable to browse the project's repository, for example). - -NOTE: -To prevent a guest user from creating projects, as an administrator, you can edit the -user's profile to mark the user as [external](#external-users). -Beware though that even if a user is external, if they already have Reporter or -higher permissions in any project or group, they are **not** counted as a -free guest user. - -## Auditor users **(PREMIUM SELF)** - -Auditor users are given read-only access to all projects, groups, and other -resources on the GitLab instance. - -An Auditor user should be able to access all projects and groups of a GitLab instance -with the permissions described on the documentation on [auditor users permissions](../administration/auditor_users.md#auditor-user-permissions-and-restrictions). - -[Read more about Auditor users.](../administration/auditor_users.md) - ## Users with minimal access **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40942) in GitLab 13.4. @@ -571,6 +452,11 @@ Owners can add members with a "minimal access" role to a parent group. Such user projects and subgroups underneath. Owners must explicitly add these "minimal access" users to the specific subgroups and projects. +You can use minimal access to give the same member more than one role in a group: + +1. Add the member to the parent group with a minimal access role. +1. Invite the member as a direct member with a specific role in any subgroup or project in that group. + Because of an [outstanding issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996), when minimal access users: - Sign in with standard web authentication, they receive a `404` error when accessing the parent group. @@ -584,16 +470,6 @@ Users with even a "minimal access" role are counted against your number of licen requirement does not apply for [GitLab Ultimate](https://about.gitlab.com/pricing/) subscriptions. -## Project features - -Project features like wiki and issues can be hidden from users depending on -which visibility level you select on project settings. - -- Disabled: disabled for everyone. -- Only team members: only team members can see, even if your project is public or internal. -- Everyone with access: everyone can see depending on your project visibility level. -- Everyone: enabled for everyone (only available for GitLab Pages). - ## Release permissions with protected tags [The permission to create tags](project/protected_tags.md) is used to define if a user can @@ -602,12 +478,12 @@ create, edit, and delete [Releases](project/releases/index.md). See [Release permissions](project/releases/index.md#release-permissions) for more information. -## LDAP users permissions - -LDAP user permissions can be manually overridden by an administrator. -Read through the documentation on [LDAP users permissions](group/access_and_permissions.md#manage-group-memberships-via-ldap) to learn more. - -## Project aliases +## Related topics -Project aliases can only be read, created and deleted by a GitLab administrator. -Read through the documentation on [Project aliases](../user/project/import/index.md#project-aliases) to learn more. +- [The GitLab principles behind permissions](https://about.gitlab.com/handbook/product/gitlab-the-product/#permissions-in-gitlab) +- [Members](project/members/index.md) +- Customize permissions on [protected branches](project/protected_branches.md) +- [LDAP users permissions](group/access_and_permissions.md#manage-group-memberships-via-ldap) +- [Value stream analytics permissions](analytics/value_stream_analytics.md#access-permissions-for-value-stream-analytics) +- [Project aliases](../user/project/import/index.md#project-aliases) +- [Auditor users](../administration/auditor_users.md) diff --git a/doc/user/product_analytics/index.md b/doc/user/product_analytics/index.md index 8e340fff32a..46f8b57a64c 100644 --- a/doc/user/product_analytics/index.md +++ b/doc/user/product_analytics/index.md @@ -4,25 +4,51 @@ group: Product Analytics info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Product analytics **(ULTIMATE)** **Alpha** +# Product analytics **(ULTIMATE)** -> Introduced in GitLab 15.4 [with a flag](../../administration/feature_flags.md) named `cube_api_proxy`. Disabled by default. +> Introduced in GitLab 15.4 as an [Alpha](../../policy/alpha-beta-support.md#alpha-features) feature [with a flag](../../administration/feature_flags.md) named `cube_api_proxy`. Disabled by default. FLAG: On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `cube_api_proxy`. On GitLab.com, this feature is not available. This feature is not ready for production use. -## Overview +This page is a work in progress, and we're updating the information as we add more features. +For more information, visit the [Product Analytics group direction page](https://about.gitlab.com/direction/analytics/product-analytics/). -You can view the [product category](https://about.gitlab.com/direction/analytics/product-analytics/) page for more information about our direction. This page is a work in progress and will be updated as we add more features. +## Enable product analytics + +You can enable and configure product analytics to track events +within your project applications on a self-managed instance. + +Prerequisite: + +- You must be an administrator of a self-managed GitLab instance. + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Product analytics** section. +1. Select **Enable product analytics** and enter the configuration values. + The following table shows the required configuration parameters and example values: + + | Name | Value | + |------------------------------|------------------------------------------------------------| + | Jitsu host | `https://jitsu.gitlab.com` | + | Jitsu project ID | `g0maofw84gx5sjxgse2k` | + | Jitsu administrator email | `jitsu.admin@gitlab.com` | + | Jitsu administrator password | `<your_password>` | + | Clickhouse URL | `https://<username>:<password>@clickhouse.gitlab.com:8123` | + | Cube API URL | `https://cube.gitlab.com` | + | Cube API key | `25718201b3e9...ae6bbdc62dbb` | + +1. Select **Save changes**. ## Product analytics dashboards Each project can define an unlimited number of dashboards. These dashboards are defined using our YAML schema and stored -in the `.gitlab/product_analytics/dashboards/` directory. The name of the file is the name of the dashboard, and visualizations are shared across dashboards.. +in the `.gitlab/product_analytics/dashboards/` directory of a project repository. The name of the file is the name of the dashboard, and visualizations are shared across dashboards. -Project maintainers can enforce approval rules on dashboard changes, and dashboards can be versioned in source control. +Project maintainers can enforce approval rules on dashboard changes using features such as code owners and approval rules. Dashboards are versioned in source control with the rest of a project's code. ### Define a dashboard @@ -30,8 +56,8 @@ To define a dashboard: 1. In `.gitlab/product_analytics/dashboards/`, create a directory named like the dashboard. Each dashboard should have its own directory. 1. In the new directory, create a `.yaml` file with the same name as the directory. This file contains the dashboard definition, and must conform to the JSON schema defined in `ee/app/validators/json_schemas/product_analytics_dashboard.json`. -1. In the `.gitlab/product_analytics/dashboards/visualizations/` directory, create a `yaml` file. This file defines the visualization type for the dashboard, and must conform to the schema in -`ee/app/validators/json_schemas/product_analytics_visualization.json`. +1. In the `.gitlab/product_analytics/dashboards/visualizations/` directory, create a `yaml` file. This file defines the visualization type for the dashboard, and must conform to the schema in + `ee/app/validators/json_schemas/product_analytics_visualization.json`. The example below includes three dashboards and one visualization that applies to all dashboards. diff --git a/doc/user/profile/account/delete_account.md b/doc/user/profile/account/delete_account.md index 5e2908a26e1..18b4e53fb31 100644 --- a/doc/user/profile/account/delete_account.md +++ b/doc/user/profile/account/delete_account.md @@ -56,6 +56,7 @@ When deleting users, you can either: - Issues. - Merge requests. - Notes and comments. + - Releases. - Personal access tokens. - Snippets. diff --git a/doc/user/profile/active_sessions.md b/doc/user/profile/active_sessions.md index 430d1c3dc9f..7a837258cb2 100644 --- a/doc/user/profile/active_sessions.md +++ b/doc/user/profile/active_sessions.md @@ -40,7 +40,7 @@ To revoke an active session: NOTE: When any session is revoked all **Remember me** tokens for all -devices are revoked. See [Why do I keep getting signed out?](index.md#why-do-i-keep-getting-signed-out) +devices are revoked. See [Why do you keep getting signed out?](index.md#why-do-you-keep-getting-signed-out) for more information about the **Remember me** feature. <!-- ## Troubleshooting diff --git a/doc/user/profile/contributions_calendar.md b/doc/user/profile/contributions_calendar.md index 6df7ad56c5e..d66e555970a 100644 --- a/doc/user/profile/contributions_calendar.md +++ b/doc/user/profile/contributions_calendar.md @@ -21,50 +21,20 @@ The contribution calendar only displays contributions from the last 12 months, b GitLab tracks the following contribution events: -- `approved` - - Merge request -- `closed` - - [Epic](../group/epics/index.md) - - Issue - - Merge request - - Milestone -- `commented` on any `Noteable` record. - - Alert - - Commit - - Design - - Issue - - Merge request - - Snippet -- `created` - - Design - - [Epic](../group/epics/index.md) - - Issue - - Merge request - - Milestone - - Project - - Wiki page -- `destroyed` - - Design - - Milestone - - Wiki page -- `expired` - - Project membership -- `joined` - - Project membership -- `left` - - Project membership -- `merged` - - Merge request -- `pushed` commits to (or deleted commits from) a repository, individually or in bulk. - - Project -- `reopened` - - [Epic](../group/epics/index.md) - - Issue - - Merge request - - Milestone -- `updated` - - Design - - Wiki page +| Event | Contribution | +| ----- | ------------ | +| `approved` | Merge request | +| `closed` | [Epic](../group/epics/index.md), Issue, Merge request, Milestone, Work item | +| `commented` on any `Noteable` record. | Alert, Commit, Design, Issue, Merge request, Snippet | +| `created` | Design, Epic, Issue, Merge request, Milestone, Project, Wiki page, Work item | +| `destroyed` | Design, Milestone, Wiki page | +| `expired` | Project membership | +| `joined` | Project membership | +| `left` | Project membership | +| `merged` | Merge request | +| `pushed` commits to (or deleted commits from) a repository, individually or in bulk. | Project | +| `reopened` | Epic, Issue, Merge request, Milestone | +| `updated` | Design, Wiki page | ### View daily contributions diff --git a/doc/user/profile/index.md b/doc/user/profile/index.md index 4adf6c351df..d6c5bd6c108 100644 --- a/doc/user/profile/index.md +++ b/doc/user/profile/index.md @@ -328,7 +328,7 @@ To view a user's activity in a top-level Activity view: ## Troubleshooting -### Why do I keep getting signed out? +### Why do you keep getting signed out? When you sign in to the main GitLab application, a `_gitlab_session` cookie is set. When you close your browser, the cookie is cleared client-side @@ -368,7 +368,7 @@ GitLab uses both session and persistent cookies: - Session cookie: Session cookies are normally removed at the end of the browser session when the browser is closed. The `_gitlab_session` cookie has no fixed expiration date. However, - it expires based on its [`session_expire_delay`](#why-do-i-keep-getting-signed-out). + it expires based on its [`session_expire_delay`](#why-do-you-keep-getting-signed-out). - Persistent cookie: The `remember_user_token` is a cookie with an expiration date of two weeks. GitLab activates this cookie if you select **Remember Me** when you sign in. diff --git a/doc/user/profile/notifications.md b/doc/user/profile/notifications.md index 1deb4842107..d0a420a4bbd 100644 --- a/doc/user/profile/notifications.md +++ b/doc/user/profile/notifications.md @@ -76,7 +76,7 @@ For each project and group you can select one of the following levels: | Participate | Receive notifications for threads you have participated in. | | On mention | Receive notifications when you are [mentioned](../discussions/index.md#mentions) in a comment. | | Disabled | Receive no notifications. | -| Custom | Receive notifications for selected events. | +| Custom | Receive notifications for selected events and threads you have participated in. | ### Global notification settings diff --git a/doc/user/profile/preferences.md b/doc/user/profile/preferences.md index dce8684d993..664d22959a2 100644 --- a/doc/user/profile/preferences.md +++ b/doc/user/profile/preferences.md @@ -96,7 +96,7 @@ A diff compares the old/removed content with the new/added content (for example, [Markdown inline diff](../markdown.md#inline-diff)). Typically, the colors red and green are used for removed and added lines in diffs. The exact colors depend on the selected [syntax highlighting theme](#syntax-highlighting-theme). -The colors may lead to difficulties in case of red–green color blindness. +The colors may lead to difficulties in case of red-green color blindness. For this reason, you can customize the following colors: @@ -203,6 +203,22 @@ NOTE: This feature is experimental, and choosing absolute times might break certain layouts. Open an issue if you notice that using absolute times breaks a layout. +## Web IDE + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/370139) in GitLab 15.7 [with a flag](../../administration/feature_flags.md) named `vscode_web_ide`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `vscode_web_ide`. On GitLab.com, this feature is available. + +The [Web IDE Beta](../project/web_ide_beta/index.md) is +the default editing environment when the `vscode_web_ide` feature +flag is enabled. + +To stop using the Web IDE Beta: + +1. In the **Web IDE** section, select the **Opt out of the Web IDE Beta** checkbox. +1. Select **Save changes**. + ## Integrations Configure your preferences with third-party services which provide enhancements to your GitLab experience. diff --git a/doc/user/profile/user_passwords.md b/doc/user/profile/user_passwords.md index b8dbdcdd956..9c1ba8852d2 100644 --- a/doc/user/profile/user_passwords.md +++ b/doc/user/profile/user_passwords.md @@ -61,12 +61,8 @@ Self-managed installations can configure the following additional password requi ## Block weak passwords > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23610) in GitLab 15.4 [with a flag](../../administration/feature_flags.md) named `block_weak_passwords`, weak passwords aren't accepted. Disabled by default on self-managed. -> - [Enabled](https://gitlab.com/gitlab-org/gitlab/-/issues/363445) on GitLab.com. - -FLAG: -On self-managed GitLab, by default blocking weak passwords is not available. To make it available, ask an administrator -to [enable the feature flag](../../administration/feature_flags.md) named `block_weak_passwords`. On GitLab.com, this -feature is available but can be configured by GitLab.com administrators only. +> - [Enabled](https://gitlab.com/gitlab-org/gitlab/-/issues/363445) on GitLab.com in GitLab 15.6. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/363445) and enabled on self-managed in GitLab 15.7. Feature flag `block_weak_passwords` removed. GitLab disallows weak passwords. Your password is considered weak when it: diff --git a/doc/user/project/badges.md b/doc/user/project/badges.md index 5d1d10fc37d..dc650bd9482 100644 --- a/doc/user/project/badges.md +++ b/doc/user/project/badges.md @@ -89,6 +89,8 @@ which are evaluated when displaying the badge. The following placeholders are available: - `%{project_path}`: Path of a project including the parent groups +- `%{project_title}`: Title of a project +- `%{project_name}`: Name of a project - `%{project_id}`: Database ID associated with a project - `%{default_branch}`: Default branch name configured for a project's repository - `%{commit_sha}`: ID of the most recent commit to the default branch of a diff --git a/doc/user/project/canary_deployments.md b/doc/user/project/canary_deployments.md index 3e6a9acc304..95f38c7e354 100644 --- a/doc/user/project/canary_deployments.md +++ b/doc/user/project/canary_deployments.md @@ -68,7 +68,7 @@ Here's an example setup flow from scratch: If it isn't, follow the documentation to specify the image version. 1. [Run a new Auto DevOps pipeline](../../ci/pipelines/index.md#run-a-pipeline-manually) and make sure that the `production` job succeeds and creates a production environment. -1. Configure a [`canary` deployment job for Auto DevOps pipelines](../../topics/autodevops/customize.md#deploy-policy-for-canary-environments). +1. Configure a [`canary` deployment job for Auto DevOps pipelines](../../topics/autodevops/cicd_variables.md#deploy-policy-for-canary-environments). 1. [Run a new Auto DevOps pipeline](../../ci/pipelines/index.md#run-a-pipeline-manually) and make sure that the `canary` job succeeds and creates a canary deployment with Canary Ingress. diff --git a/doc/user/project/clusters/add_eks_clusters.md b/doc/user/project/clusters/add_eks_clusters.md index b3d381c3148..52288af101a 100644 --- a/doc/user/project/clusters/add_eks_clusters.md +++ b/doc/user/project/clusters/add_eks_clusters.md @@ -195,7 +195,7 @@ If a default Storage Class doesn't already exist and is desired, follow Amazon's to create one. Alternatively, disable PostgreSQL by setting the project variable -[`POSTGRES_ENABLED`](../../../topics/autodevops/customize.md#cicd-variables) to `false`. +[`POSTGRES_ENABLED`](../../../topics/autodevops/cicd_variables.md#cicd-variables) to `false`. ## Deploy the app to EKS diff --git a/doc/user/project/clusters/cluster_access.md b/doc/user/project/clusters/cluster_access.md index c9b3596d92f..529e7a6da12 100644 --- a/doc/user/project/clusters/cluster_access.md +++ b/doc/user/project/clusters/cluster_access.md @@ -49,16 +49,16 @@ GitLab creates the following resources for RBAC clusters. | Name | Type | Details | Created when | |:----------------------|:---------------------|:-----------------------------------------------------------------------------------------------------------|:-----------------------| | `gitlab` | `ServiceAccount` | `default` namespace | Creating a new cluster | -| `gitlab-admin` | `ClusterRoleBinding` | [`cluster-admin`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) roleRef | Creating a new cluster | +| `gitlab-admin` | `ClusterRoleBinding` | [`cluster-admin`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) role | Creating a new cluster | | `gitlab-token` | `Secret` | Token for `gitlab` ServiceAccount | Creating a new cluster | | Environment namespace | `Namespace` | Contains all environment-specific resources | Deploying to a cluster | | Environment namespace | `ServiceAccount` | Uses namespace of environment | Deploying to a cluster | | Environment namespace | `Secret` | Token for environment ServiceAccount | Deploying to a cluster | -| Environment namespace | `RoleBinding` | [`admin`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) roleRef | Deploying to a cluster | +| Environment namespace | `RoleBinding` | [`admin`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) role | Deploying to a cluster | The environment namespace `RoleBinding` was [updated](https://gitlab.com/gitlab-org/gitlab/-/issues/31113) in GitLab 13.6 -to `admin` roleRef. Previously, the `edit` roleRef was used. +to `admin` role. Previously, the `edit` role was used. ## ABAC cluster resources diff --git a/doc/user/project/deploy_tokens/index.md b/doc/user/project/deploy_tokens/index.md index 3dd6f14ea70..5f279ddda5b 100644 --- a/doc/user/project/deploy_tokens/index.md +++ b/doc/user/project/deploy_tokens/index.md @@ -216,19 +216,3 @@ Prerequisites: - A deploy token with `read_registry` and `write_registry` scopes. Follow the dependency proxy [authentication instructions](../../packages/dependency_proxy/index.md). - -## Troubleshooting - -### Error: `api error: Repository or object not found:` - -When using a group deploy token to clone from LFS objects, you might get `404 Not Found` responses -and this error message. This occurs because of a bug, documented in -[issue 235398](https://gitlab.com/gitlab-org/gitlab/-/issues/235398). - -```plaintext -api error: Repository or object not found: -https://<URL-with-token>.git/info/lfs/objects/batch -Check that it exists and that you have proper access to it -``` - -The workaround is to use a project deploy token. diff --git a/doc/user/project/description_templates.md b/doc/user/project/description_templates.md index 40c36236932..ffbe7447aa8 100644 --- a/doc/user/project/description_templates.md +++ b/doc/user/project/description_templates.md @@ -40,7 +40,7 @@ To create an issue description template: where `mytemplate` is the name of your issue template. 1. Commit to your default branch. -To check if this has worked correctly, [create a new issue](issues/managing_issues.md#create-an-issue) +To check if this has worked correctly, [create a new issue](issues/create_issues.md) and see if you can find your description template in the **Choose a template** dropdown list. ## Create a merge request template @@ -81,7 +81,23 @@ To discard any changes to the description you've made after selecting the templa NOTE: You can create shortcut links to create an issue using a designated template. -For example: `https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Feature%20proposal`. Read more about [creating issues using a URL with prefilled values](issues/managing_issues.md#using-a-url-with-prefilled-values). +For example: `https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Feature%20proposal`. Read more about [creating issues using a URL with prefilled values](issues/create_issues.md#using-a-url-with-prefilled-values). + +### Supported variables in merge request templates + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89810) in GitLab 15.7. + +When you save a merge request for the first time, GitLab replaces these variables in +your merge request template with their values: + +| Variable | Description | Output example | +|----------|-------------|----------------| +| `%{all_commits}` | Messages from all commits in the merge request. Limited to 100 most recent commits. Skips commit bodies exceeding 100 KiB and merge commit messages. | `* Feature introduced` <br><br> `This commit implements feature` <br> `Changelog:added` <br><br> `* Bug fixed` <br><br> `* Documentation improved` <br><br>`This commit introduced better docs.` | +| `%{co_authored_by}` | Names and emails of commit authors in a `Co-authored-by` Git commit trailer format. Limited to authors of 100 most recent commits in merge request. | `Co-authored-by: Zane Doe <zdoe@example.com>` <br> `Co-authored-by: Blake Smith <bsmith@example.com>` | +| `%{first_commit}` | Full message of the first commit in merge request diff. | `Update README.md` | +| `%{first_multiline_commit}` | Full message of the first commit that's not a merge commit and has more than one line in message body. Merge request title if all commits aren't multiline. | `Update README.md`<br><br>`Improved project description in readme file.` | +| `%{source_branch}` | The name of the branch being merged. | `my-feature-branch` | +| `%{target_branch}` | The name of the branch that the changes are applied to. | `main` | ### Set instance-level description templates **(PREMIUM SELF)** diff --git a/doc/user/project/import/bitbucket.md b/doc/user/project/import/bitbucket.md index 3f1a2dcfe2b..98b46650b42 100644 --- a/doc/user/project/import/bitbucket.md +++ b/doc/user/project/import/bitbucket.md @@ -104,7 +104,7 @@ current Bitbucket public name, and reconnect if there's a mismatch: 1. [Use the API to get the currently authenticated user](../../../api/users.md#for-normal-users-1). -1. In the API's response, the `identities` attribute contains the Bitbucket account that exists in +1. In the API response, the `identities` attribute contains the Bitbucket account that exists in the GitLab database. If the `extern_uid` doesn't match the current Bitbucket public name, the user should reconnect their Bitbucket account in the [GitLab profile service sign-in](https://gitlab.com/-/profile/account). diff --git a/doc/user/project/import/bitbucket_server.md b/doc/user/project/import/bitbucket_server.md index 1f34c6d4ad9..d7fa1338c55 100644 --- a/doc/user/project/import/bitbucket_server.md +++ b/doc/user/project/import/bitbucket_server.md @@ -63,6 +63,10 @@ The following items are changed when they are imported: ## User assignment +Prerequisite: + +- Authentication token with administrator access. + When issues and pull requests are importing, the importer tries to find the author's email address with a confirmed email address in the GitLab user database. If no such user is available, the project creator is set as the author. The importer appends a note in the comment to mark the diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index c0b13c76322..07a21d8b941 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -59,11 +59,10 @@ their GitHub authors and assignees in the database of the GitLab instance. Pull GitLab. For this association to succeed, each GitHub author and assignee in the repository -must meet one of the following conditions prior to the import: - -- Have previously logged in to a GitLab account using the GitHub icon. -- Have a GitHub account with a [public-facing email address](https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-personal-account-on-github/managing-email-preferences/setting-your-commit-email-address) - that matches their GitLab account's email address. +must have a [public-facing email address](https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-personal-account-on-github/managing-email-preferences/setting-your-commit-email-address) +on GitHub that matches their GitLab email address (regardless of how the account was created). +If their email address from GitHub is set as their secondary email address in GitLab, it must be +confirmed. GitLab content imports that use GitHub accounts require that the GitHub public-facing email address is populated. This means all comments and contributions are properly mapped to the same user in GitLab. GitHub Enterprise does not require this @@ -73,10 +72,8 @@ field to be populated so you may have to add it on existing accounts. ### Use the GitHub integration -Before you begin, ensure that any GitHub users who you want to map to GitLab users have either: - -- A GitLab account that has logged in using the GitHub icon. -- A GitLab account with an email address that matches the [publicly visible email address](https://docs.github.com/en/rest/users#get-a-user) in the profile of the GitHub user +Before you begin, ensure that any GitHub user you want to map to a GitLab user has a GitLab email address that matches their +[publicly visible email address](https://docs.github.com/en/rest/users#get-a-user) on GitHub. If you are importing to GitLab.com, you can alternatively import GitHub repositories using a [personal access token](#use-a-github-token). We do not recommend this method, as it does not associate all user activity (such as issues and pull requests) with matching GitLab users. @@ -96,7 +93,10 @@ If you are using a self-managed GitLab instance or if you are importing from Git ### Use a GitHub token -NOTE: +Prerequisite: + +- Authentication token with administrator access. + Using a personal access token to import projects is not recommended. If you are a GitLab.com user, you can use a personal access token to import your project from GitHub, but this method cannot associate all user activity (such as issues and pull requests) with matching GitLab users. @@ -222,21 +222,26 @@ References to pull requests and issues are preserved. Each imported repository m [visibility level is restricted](../../public_access.md#restrict-use-of-public-or-internal-projects), in which case it defaults to the default project visibility. -### Branch protection rules +### Branch protection rules and project settings + +When they are imported, supported GitHub branch protection rules are mapped to either: + +- GitLab branch protection rules. +- Project-wide GitLab settings. -Supported GitHub branch protection rules are mapped to GitLab branch protection rules or project-wide GitLab settings when they are imported: +| GitHub rule | GitLab rule | Introduced in | +| :---------------------------------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------ | +| **Require conversation resolution before merging** for the project's default branch | **All threads must be resolved** [project setting](../../discussions/index.md#prevent-merge-unless-all-threads-are-resolved) | [GitLab 15.5](https://gitlab.com/gitlab-org/gitlab/-/issues/371110) | +| **Require a pull request before merging** | **No one** option in the **Allowed to push** list of [branch protection settings](../protected_branches.md#configure-a-protected-branch) | [GitLab 15.5](https://gitlab.com/gitlab-org/gitlab/-/issues/370951) | +| **Require signed commits** for the project's default branch | **Reject unsigned commits** GitLab [push rule](../repository/push_rules.md#prevent-unintended-consequences) **(PREMIUM)** | [GitLab 15.5](https://gitlab.com/gitlab-org/gitlab/-/issues/370949) | +| **Allow force pushes - Everyone** | **Allowed to force push** [branch protection setting](../protected_branches.md#allow-force-push-on-a-protected-branch) | [GitLab 15.6](https://gitlab.com/gitlab-org/gitlab/-/issues/370943) | +| **Require a pull request before merging - Require review from Code Owners** | **Require approval from code owners** [branch protection setting](../protected_branches.md#require-code-owner-approval-on-a-protected-branch) **(PREMIUM)** | [GitLab 15.6](https://gitlab.com/gitlab-org/gitlab/-/issues/376683) | -- GitHub rule **Require conversation resolution before merging** for the project's default branch is mapped to the [**All threads must be resolved** GitLab setting](../../discussions/index.md#prevent-merge-unless-all-threads-are-resolved). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/371110) in GitLab 15.5. -- GitHub rule **Require a pull request before merging** is mapped to the **No one** option in the **Allowed to push** list of the branch protection rule. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/370951) in GitLab 15.5. -- GitHub rule **Require a pull request before merging - Require review from Code Owners** is mapped to the - [**Code owner approval** branch protection rule](../protected_branches.md#require-code-owner-approval-on-a-protected-branch). Requires GitLab Premium or higher. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/376683) in GitLab 15.6. -- GitHub rule **Require signed commits** for the project's default branch is mapped to the **Reject unsigned commits** GitLab push rule. Requires GitLab Premium or higher. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/370949) in GitLab 15.5. -- GitHub rule **Allow force pushes - Everyone** is mapped to the [**Allowed to force push** branch protection rule](../protected_branches.md#allow-force-push-on-a-protected-branch). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/370943) in GitLab 15.6. -- GitHub rule **Allow force pushes - Specify who can force push** is proposed in issue [370945](https://gitlab.com/gitlab-org/gitlab/-/issues/370945). -- Support for GitHub rule **Require status checks to pass before merging** was proposed in issue [370948](https://gitlab.com/gitlab-org/gitlab/-/issues/370948). However, this rule cannot be translated during project import into GitLab due to technical difficulties. -You can still create [status checks](../merge_requests/status_checks.md) in GitLab yourself. +Mapping GitHub rule **Require status checks to pass before merging** to +[external status checks](../merge_requests/status_checks.md) was considered in issue +[370948](https://gitlab.com/gitlab-org/gitlab/-/issues/370948). However, this rule is not imported during project import +into GitLab due to technical difficulties. You can still create [external status checks](../merge_requests/status_checks.md) +manually. ## Alternative way to import notes and diff notes @@ -274,7 +279,7 @@ Feature.disable(:github_importer_lower_per_page_limit, group) ## Import from GitHub Enterprise on an internal network -If your GitHub Enterprise instance is on a internal network that is unaccessible to the internet, you can use a reverse proxy +If your GitHub Enterprise instance is on a internal network that is inaccessible to the internet, you can use a reverse proxy to allow GitLab.com to access the instance. The proxy needs to: diff --git a/doc/user/project/import/img/gitlab_import_history_page_v14_10.png b/doc/user/project/import/img/gitlab_import_history_page_v14_10.png Binary files differdeleted file mode 100644 index 812696a8faa..00000000000 --- a/doc/user/project/import/img/gitlab_import_history_page_v14_10.png +++ /dev/null diff --git a/doc/user/project/import/index.md b/doc/user/project/import/index.md index 1b5a658d209..208bce90453 100644 --- a/doc/user/project/import/index.md +++ b/doc/user/project/import/index.md @@ -5,76 +5,59 @@ group: Import info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Migrate projects to a GitLab instance **(FREE)** - -See these documents to migrate to GitLab: - -- [From Bitbucket Cloud](bitbucket.md) -- [From Bitbucket Server (also known as Stash)](bitbucket_server.md) -- [From ClearCase](clearcase.md) -- [From CVS](cvs.md) -- [From FogBugz](fogbugz.md) -- [From GitHub.com or GitHub Enterprise](github.md) -- [From GitLab.com](gitlab_com.md) -- [From Gitea](gitea.md) -- [From Perforce](perforce.md) -- [From SVN](svn.md) -- [From TFVC](tfvc.md) -- [From repository by URL](repo_by_url.md) -- [By uploading a manifest file (AOSP)](manifest.md) -- [From Phabricator](phabricator.md) -- [From Jira (issues only)](jira.md) +# Import and migrate projects **(FREE)** -You can also import any Git repository through HTTP from the **New Project** page. Note that if the -repository is too large, the import can timeout. - -You can also [connect your external repository to get CI/CD benefits](../../../ci/ci_cd_for_external_repos/index.md). +If you want to bring existing projects to GitLab or copy GitLab projects to a different location, you can: -## Project import history - -You can view all project imports created by you. This list includes the following: +- Import projects from external systems using one of the [available importers](#available-project-importers). +- Migrate GitLab projects: + - Between two GitLab self-managed instances. + - Between a self-managed instance and GitLab.com in both directions. + - In the same GitLab instance. -- Source (without credentials for security reasons) -- Destination -- Status -- Error details if the import failed +For any type of source and target, you can migrate projects: -To view project import history: +- As part of a [GitLab group migration](../../group/import/index.md). You can't migrate only chosen projects, + but you can migrate many projects at once within a group. +- Using [file exports](../settings/import_export.md). With this method you can migrate projects one by one. No network + connection between instances is required. -1. Sign in to GitLab. -1. On the top bar, select **New** (**{plus}**). -1. Select **New project/repository**. -1. Select **Import project**. -1. Select **History**. +If you only need to migrate Git repositories, you can [import each project by URL](repo_by_url.md). However, you can't +import issues and merge requests this way. To retain metadata like issues and merge requests, either: - +- [Migrate projects with groups](../../group/import/index.md). +- Use [file exports](../settings/import_export.md) to import projects. -The history also includes projects created from [built-in](../working_with_projects.md#create-a-project-from-a-built-in-template) -or [custom](../working_with_projects.md#create-a-project-from-a-built-in-template) -templates. GitLab uses [import repository by URL](repo_by_url.md) -to create a new project from a template. +Keep in mind the limitations of [migrating using file exports](../settings/import_export.md#items-that-are-exported). +When migrating from self-managed to GitLab.com, user associations (such as comment author) +are changed to the user who is importing the projects. -## LFS authentication +## Available project importers -When importing a project that contains LFS objects, if the project has an [`.lfsconfig`](https://github.com/git-lfs/git-lfs/blob/master/docs/man/git-lfs-config.5.ronn) -file with a URL host (`lfs.url`) different from the repository URL host, LFS files are not downloaded. +You can import projects from: -## Migrate from self-managed GitLab to GitLab.com +- [Bitbucket Cloud](bitbucket.md) +- [Bitbucket Server (also known as Stash)](bitbucket_server.md) +- [ClearCase](clearcase.md) +- [CVS](cvs.md) +- [FogBugz](fogbugz.md) +- [GitHub.com or GitHub Enterprise](github.md) +- [GitLab.com](gitlab_com.md) +- [Gitea](gitea.md) +- [Perforce](perforce.md) +- [From SVN](https://git-scm.com/book/en/v2/Git-and-Other-Systems-Git-as-a-Client) +- [TFVC](tfvc.md) +- [Repository by URL](repo_by_url.md) +- [Uloading a manifest file (AOSP)](manifest.md) +- [Phabricator](phabricator.md) +- [Jira (issues only)](jira.md) -If you only need to migrate Git repositories, you can [import each project by URL](repo_by_url.md). -However, you can't import issues and merge requests this way. To retain all metadata like issues and -merge requests, use the [import/export feature](../settings/import_export.md) -to export projects from self-managed GitLab and import those projects into GitLab.com. All GitLab -user associations (such as comment author) are changed to the user importing the project. For more -information, see the prerequisites and important notes in these sections: +You can also import any Git repository through HTTP from the **New Project** page. Note that if the +repository is too large, the import can timeout. -- [Export a project and its data](../settings/import_export.md#export-a-project-and-its-data). -- [Import the project](../settings/import_export.md#import-a-project-and-its-data). +You can then [connect your external repository to get CI/CD benefits](../../../ci/ci_cd_for_external_repos/index.md). -NOTE: -When migrating to GitLab.com, you must create users manually unless [SCIM](../../../user/group/saml_sso/scim_setup.md) -will be used. Creating users with the API is limited to self-managed instances as it requires -administrator access. +## Migrate using the API To migrate all data from self-managed to GitLab.com, you can leverage the [API](../../../api/index.md). Migrate the assets in this order: @@ -83,17 +66,9 @@ Migrate the assets in this order: 1. [Projects](../../../api/projects.md) 1. [Project variables](../../../api/project_level_variables.md) -Keep in mind the limitations of the [import/export feature](../settings/import_export.md#items-that-are-exported). - You must still migrate your [Container Registry](../../packages/container_registry/index.md) over a series of Docker pulls and pushes. Re-run any CI pipelines to retrieve any build artifacts. -## Migrate from GitLab.com to self-managed GitLab - -The process is essentially the same as [migrating from self-managed GitLab to GitLab.com](#migrate-from-self-managed-gitlab-to-gitlabcom). -The main difference is that an administrator can create users on the self-managed GitLab instance -through the UI or the [users API](../../../api/users.md#user-creation). - ## Migrate between two self-managed GitLab instances To migrate from an existing self-managed GitLab instance to a new self-managed GitLab instance, it's @@ -105,12 +80,36 @@ The backups produced don't depend on the operating system running GitLab. You ca the restore method to switch between different operating system distributions or versions, as long as the same GitLab version [is available for installation](../../../administration/package_information/supported_os.md). -To instead merge two self-managed GitLab instances together, use the instructions in -[Migrate from self-managed GitLab to GitLab.com](#migrate-from-self-managed-gitlab-to-gitlabcom). -This method is useful when both self-managed instances have existing data that must be preserved. +Also note that administrators can use the [Users API](../../../api/users.md) to migrate users. + +## View project import history + +You can view all project imports created by you. This list includes the following: + +- Paths of source projects if projects were imported from external systems, or import method if GitLab projects were migrated. +- Paths of destination projects. +- Start date of each import. +- Status of each import. +- Error details if any errors occurred. + +To view project import history: + +1. Sign in to GitLab. +1. On the top bar, select **Create new...** (**{plus-square}**). +1. Select **New project/repository**. +1. Select **Import project**. +1. Select **History** in the upper right corner. +1. If there are any errors for a particular import, you can see them by selecting **Details**. + +The history also includes projects created from [built-in](../working_with_projects.md#create-a-project-from-a-built-in-template) +or [custom](../working_with_projects.md#create-a-project-from-a-built-in-template) +templates. GitLab uses [import repository by URL](repo_by_url.md) +to create a new project from a template. + +## LFS authentication -Also note that administrators can use the [Users API](../../../api/users.md) -to migrate users. +When importing a project that contains LFS objects, if the project has an [`.lfsconfig`](https://github.com/git-lfs/git-lfs/blob/master/docs/man/git-lfs-config.5.ronn) +file with a URL host (`lfs.url`) different from the repository URL host, LFS files are not downloaded. ## Project aliases **(PREMIUM SELF)** diff --git a/doc/user/project/import/svn.md b/doc/user/project/import/svn.md index 1687d621e2e..6730ef862e6 100644 --- a/doc/user/project/import/svn.md +++ b/doc/user/project/import/svn.md @@ -1,91 +1,11 @@ --- -type: howto -stage: Manage -group: Import -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: 'index.md' +remove_date: '2023-03-15' --- -# Migrate from Subversion to GitLab **(FREE)** +This document was moved to [another location](index.md). -GitLab uses Git as its version control system. If you're using Subversion (SVN) as your version control system, -you can migrate to using a Git repository in GitLab using `svn2git`. - -You can follow the steps on this page to migrate to Git if your SVN repository: - -- Has a standard format (trunk, branches, and tags). -- Is not nested. - -For a non-standard repository see the [`svn2git` documentation](https://github.com/nirvdrum/svn2git). - -We recommend a hard cut over from SVN to Git and GitLab. Run the migration command once and then have all users use the -new GitLab repository immediately. - -## Install `svn2git` - -Install `svn2git` on a local workstation rather than the GitLab server: - -- On all systems you can install as a Ruby gem if you already have Ruby and Git installed: - - ```shell - sudo gem install svn2git - ``` - -- On Debian-based Linux distributions you can install the native packages: - - ```shell - sudo apt-get install git-core git-svn ruby - ``` - -## Prepare an authors file (recommended) - -Prepare an authors file so `svn2git` can map SVN authors to Git authors. If you choose not to create the authors file, -commits are not attributed to the correct GitLab user. - -To map authors, you must map every author present on changes in the SVN repository. If you don't, the -migration fails and you have to update the author file accordingly. - -1. Search through the SVN repository and output a list of authors: - - ```shell - svn log --quiet | grep -E "r[0-9]+ \| .+ \|" | cut -d'|' -f2 | sed 's/ //g' | sort | uniq - ``` - -1. Use the output from the last command to construct the authors file. Create a file called `authors.txt` and add one - mapping per line. For example: - - ```plaintext - sidneyjones = Sidney Jones <sidneyjones@example.com> - ``` - -## Migrate SVN repository to Git repository - -`svn2git` supports excluding certain file paths, branches, tags, and more. See -the [`svn2git` documentation](https://github.com/nirvdrum/svn2git) or run `svn2git --help` for full documentation on all of -the available options. - -For each repository to migrate: - -1. Create a new directory and change into it. -1. For repositories that: - - - Don't require a username and password, run: - - ```shell - svn2git https://svn.example.com/path/to/repo --authors /path/to/authors.txt - ``` - - - Do require a username and password, run: - - ```shell - svn2git https://svn.example.com/path/to/repo --authors /path/to/authors.txt --username <username> --password <password> - ``` - -1. Create a new GitLab project for your migrated code. -1. Copy the SSH or HTTP(S) repository URL from the GitLab project page. -1. Add the GitLab repository as a Git remote and push all the changes. This pushes all commits, branches, and tags. - - ```shell - git remote add origin git@gitlab.example.com:<group>/<project>.git - git push --all origin - git push --tags origin - ``` +<!-- This redirect file can be deleted after <2023-03-15>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/project/integrations/bamboo.md b/doc/user/project/integrations/bamboo.md index fceec006a1a..db90bafaaa5 100644 --- a/doc/user/project/integrations/bamboo.md +++ b/doc/user/project/integrations/bamboo.md @@ -63,7 +63,7 @@ Bamboo. For example, `https://bamboo.example.com/browse/PROJ-PLAN`. ## Update Bamboo build status in GitLab -You can use a script that uses the [commit status API](../../../api/commits.md#post-the-build-status-to-a-commit) +You can use a script that uses the [commit status API](../../../api/commits.md#set-the-pipeline-status-of-a-commit) and Bamboo build variables to: - Update the commit with the build status. diff --git a/doc/user/project/integrations/gitlab_slack_application.md b/doc/user/project/integrations/gitlab_slack_application.md index 8d6fdf882b7..50b52421a5a 100644 --- a/doc/user/project/integrations/gitlab_slack_application.md +++ b/doc/user/project/integrations/gitlab_slack_application.md @@ -10,7 +10,7 @@ NOTE: The GitLab for Slack app is only configurable for GitLab.com. It does **not** work for on-premises installations where you can configure [Slack slash commands](slack_slash_commands.md) instead. See -[Slack application integration for self-managed instances](https://gitlab.com/gitlab-org/gitlab/-/issues/28164) +[Slack application integration for self-managed instances](https://gitlab.com/groups/gitlab-org/-/epics/1211) for our plans to make the app configurable for all GitLab installations. Slack provides a native application that you can enable with your project's @@ -36,12 +36,12 @@ workspace to be able to install a new application. See To enable the GitLab integration for your Slack workspace: -1. Go to your project's **Settings > Integration > Slack application** (only +1. Go to your project's **Settings > Integration > GitLab for Slack app** (only visible on GitLab.com). -1. Select **Install Slack app**. +1. Select **Install GitLab for Slack app**. 1. Select **Allow** on Slack's confirmation screen. -You can also select **Reinstall Slack app** to update the app in your Slack workspace +You can also select **Reinstall GitLab for Slack app** to update the app in your Slack workspace to the latest version. See [Version history](#version-history) for details. ## Create a project alias for Slack @@ -50,7 +50,7 @@ To create a project alias on GitLab.com for Slack integration: 1. Go to your project's home page. 1. Go to **Settings > Integrations** (only visible on GitLab.com). -1. On the **Integrations** page, select **Slack application**. +1. On the **Integrations** page, select **GitLab for Slack app**. 1. The current **Project Alias**, if any, is displayed. To edit this value, select **Edit**. 1. Enter your desired alias, and select **Save changes**. @@ -91,7 +91,7 @@ As a workaround, ensure your app is up to date. To update an existing Slack integration: 1. Go to your [chat settings](https://gitlab.com/-/profile/chat_names). -1. Next to your project, select **Slack application**. -1. Select **Reinstall Slack app**. +1. Next to your project, select **GitLab for Slack app**. +1. Select **Reinstall GitLab for Slack app**. Alternatively, you can [configure a new Slack integration](https://about.gitlab.com/solutions/slack/). diff --git a/doc/user/project/integrations/index.md b/doc/user/project/integrations/index.md index 77444570499..769a45fc6ff 100644 --- a/doc/user/project/integrations/index.md +++ b/doc/user/project/integrations/index.md @@ -58,7 +58,6 @@ You can configure the following integrations. | [Emails on push](emails_on_push.md) | Send commits and diff of each push by email. | **{dotted-circle}** No | | [EWM](ewm.md) | Use IBM Engineering Workflow Management as the issue tracker. | **{dotted-circle}** No | | [External wiki](../wiki/index.md#link-an-external-wiki) | Link an external wiki. | **{dotted-circle}** No | -| [Flowdock](../../../api/integrations.md#flowdock) | Send notifications from GitLab to Flowdock flows. | **{dotted-circle}** No | | [GitHub](github.md) | Obtain statuses for commits and pull requests. | **{dotted-circle}** No | | [Google Chat](hangouts_chat.md) | Send notifications from your GitLab project to a room in Google Chat. | **{dotted-circle}** No | | [Harbor](harbor.md) | Use Harbor as the container registry. | **{dotted-circle}** No | @@ -77,7 +76,7 @@ You can configure the following integrations. | Pushover | Get real-time notifications on your device. | **{dotted-circle}** No | | [Redmine](redmine.md) | Use Redmine as the issue tracker. | **{dotted-circle}** No | | [Shimo Workspace](shimo.md) | Use Shimo instead of the GitLab Wiki. | **{dotted-circle}** No | -| [Slack application](gitlab_slack_application.md) | Use Slack's official GitLab application. | **{dotted-circle}** No | +| [GitLab for Slack app](gitlab_slack_application.md) | Use Slack's official GitLab application. | **{dotted-circle}** No | | [Slack notifications](slack.md) | Send notifications about project events to Slack. | **{dotted-circle}** No | | [Slack slash commands](slack_slash_commands.md) | Enable slash commands in a workspace. | **{dotted-circle}** No | | [Unify Circuit](unify_circuit.md) | Send notifications about project events to Unify Circuit. | **{dotted-circle}** No | diff --git a/doc/user/project/integrations/mlflow_client.md b/doc/user/project/integrations/mlflow_client.md index 82bfd08e926..bd14021ab1c 100644 --- a/doc/user/project/integrations/mlflow_client.md +++ b/doc/user/project/integrations/mlflow_client.md @@ -19,9 +19,9 @@ Setting up your integrations requires minimal changes to existing code. GitLab plays the role of proxy server, both for artifact storage and tracking data. It reflects the MLFlow [Scenario 5](https://www.mlflow.org/docs/latest/tracking.html#scenario-5-mlflow-tracking-server-enabled-with-proxied-artifact-storage-access). -## Enable MFlow Client Integration +## Enable MLFlow Client Integration -Complete this task to enable MFlow Client Integration. +Complete this task to enable MLFlow Client Integration. Prerequisites: @@ -49,17 +49,17 @@ that can be explored by selecting an experiment. - The API GitLab supports is the one defined at MLFlow version 1.28.0. - API endpoints not listed above are not supported. -- During creation of experiments and runs, tags are ExperimentTags and RunTags are ignored. -- MLFLow Model Registry is not supported. +- During creation of experiments and runs, tags are ExperimentTags and RunTags are stored, even though they are not displayed. +- MLFlow Model Registry is not supported. ## Supported methods and caveats This is a list of methods we support from the MLFlow client. Other methods might be supported but were not tested. More information can be found in the [MLFlow Documentation](https://www.mlflow.org/docs/1.28.0/python_api/mlflow.html). -### `set_experiment` +### `set_experiment()` -Accepts both experiment_name and experiment_id +Accepts both `experiment_name` and `experiment_id` ### `start_run()` diff --git a/doc/user/project/integrations/slack.md b/doc/user/project/integrations/slack.md index d34c558ebbc..2ded5799c23 100644 --- a/doc/user/project/integrations/slack.md +++ b/doc/user/project/integrations/slack.md @@ -36,10 +36,6 @@ to control GitLab from Slack. Slash commands are configured separately. - *To send messages to channels,* enter the Slack channel names, separated by commas. - *To send direct messages,* use the Member ID found in the user's Slack profile. - - NOTE: - Usernames and private channels are not supported. - 1. In **Webhook**, enter the webhook URL you copied in the [Slack configuration](#configure-slack) step. 1. Optional. In **Username**, enter the username of the Slack bot that sends diff --git a/doc/user/project/integrations/webhook_events.md b/doc/user/project/integrations/webhook_events.md index 60187b9a682..63282d6ec6e 100644 --- a/doc/user/project/integrations/webhook_events.md +++ b/doc/user/project/integrations/webhook_events.md @@ -1019,7 +1019,7 @@ Payload example: ``` NOTE: -The fields `assignee_id`, `state`, `merge_status` are deprecated. +The fields `assignee_id`, `state`, `merge_status` are [deprecated](../../../api/merge_requests.md). ## Wiki page events @@ -1171,7 +1171,7 @@ Payload example: "project":{ "id": 41, "web_url": "https://gitlab.example.com/gitlab-org/upstream-project", - "path_with_namespace": "gitlab-org/upstream-project", + "path_with_namespace": "gitlab-org/upstream-project" }, "pipeline_id": 30, "job_id": 3401 @@ -1475,6 +1475,8 @@ Payload example: "deployable_id": 796, "deployable_url": "http://10.126.0.2:3000/root/test-deployment-webhooks/-/jobs/796", "environment": "staging", + "environment_slug": "staging", + "environment_external_url": "https://staging.example.com", "project": { "id": 30, "name": "test-deployment-webhooks", diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index ef6957ac2d8..3d45e947c4c 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w [Webhooks](https://en.wikipedia.org/wiki/Webhook) are custom HTTP callbacks that you define. They are usually triggered by an -event, such as pushing code to a repository or posting a comment on a blog. +event, such as pushing code to a repository or posting a comment on an issue. When the event occurs, the source app makes an HTTP request to the URI configured for the webhook. The action to take may be anything. For example, you can use webhooks to: @@ -52,7 +52,7 @@ specific to a group, including: ## Configure a webhook in GitLab -You can configure a webhook for a group or a project. +To configure a webhook for a project or group: 1. In your project or group, on the left sidebar, select **Settings > Webhooks**. 1. In **URL**, enter the URL of the webhook endpoint. @@ -62,6 +62,40 @@ You can configure a webhook for a group or a project. 1. Optional. Clear the **Enable SSL verification** checkbox to disable [SSL verification](index.md#manage-ssl-verification). 1. Select **Add webhook**. +## Mask sensitive portions of webhook URLs + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/99995) in GitLab 15.5 [with a flag](../../../administration/feature_flags.md) named `webhook_form_mask_url`. Disabled by default. +> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/376106) in GitLab 15.6. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/376106) in GitLab 15.7. Feature flag `webhook_form_mask_url` removed. + +You can define and mask sensitive portions of webhook URLs and replace them +with configured values any number of times when webhooks are executed. +Sensitive portions do not get logged and are encrypted at rest in the database. + +To mask sensitive portions of the webhook URL: + +1. In your project or group, on the left sidebar, select **Settings > Webhooks**. +1. In **URL**, enter the full webhook URL. +1. Select **Mask portions of URL**. +1. In **Sensitive portion of URL**, enter the portion you want to mask. +1. In **How it looks in the UI**, enter the masking value. + +To interpolate sensitive portions for each webhook, use `url_variables`. +For example, if a webhook has the following URL: + +```plaintext +https://{subdomain}.example.com/{path}?key={value} +``` + +You must define the following variables: + +- `subdomain` +- `path` +- `value` + +Variable names can contain only lowercase letters (`a-z`), numbers (`0-9`), or underscores (`_`). +You can define URL variables directly using the REST API. + ## Configure your webhook receiver endpoint Webhook receiver endpoints should be fast and stable. @@ -86,27 +120,22 @@ Endpoints should follow these best practices: ### Failing webhooks -> Introduced in GitLab 13.12 [with a flag](../../../administration/feature_flags.md) named `web_hooks_disable_failed`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `web_hooks_disable_failed`. -On GitLab.com, this feature is not available. -The feature is not ready for production use. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60837) in GitLab 13.12 [with a flag](../../../administration/feature_flags.md) named `web_hooks_disable_failed`. Disabled by default. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/329849) in GitLab 15.7. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/329849) in GitLab 15.7. Feature flag `web_hooks_disable_failed` removed. If a webhook fails repeatedly, it may be disabled automatically. Webhooks that return response codes in the `5xx` range are understood to be failing -intermittently, and are temporarily disabled. This lasts initially -for 10 minutes. If the hook continues to fail, the back-off period is -extended on each retry, up to a maximum disabled period of 24 hours. +intermittently and are temporarily disabled. These webhooks are initially disabled +for 1 minute, which is extended on each retry up to a maximum of 24 hours. -Webhooks that return failure codes in the `4xx` range are understood to be -misconfigured, and these are disabled until you manually re-enable -them. These webhooks are not automatically retried. +Webhooks that return response codes in the `4xx` range are understood to be +misconfigured and are permanently disabled until you manually re-enable +them yourself. -See [troubleshooting](#troubleshoot-webhooks) for information on -how to see if a webhook is disabled, and how to re-enable it. +See [Troubleshooting](#troubleshoot-webhooks) for more information on +disabled webhooks and how to re-enable them. ## Test a webhook @@ -190,10 +219,10 @@ You can filter push events by branch. Use one of the following options to filter - **All branches**: push events from all branches. - **Wildcard pattern**: push events from a branch that matches a wildcard pattern (for example, `*-stable` or `production/*`). -- **Regular expression**: push events from a branch that matches a regular expression (for example, `(feature|hotfix)/*`). +- **Regular expression**: push events from a branch that matches a regular expression (for example, `^(feature|hotfix)/`). -You can configure branch filtering -in the [webhook settings](#configure-a-webhook-in-gitlab) in your project. +To configure branch filtering for a project or group, see +[Configure a webhook in GitLab](#configure-a-webhook-in-gitlab). ## How image URLs are displayed in the webhook body @@ -302,13 +331,7 @@ GitLab expects a response in [10 seconds](../../../user/gitlab_com/index.md#othe ### Re-enable disabled webhooks > - Introduced in GitLab 15.2 [with a flag](../../../administration/feature_flags.md) named `webhooks_failed_callout`. Disabled by default. -> - The [`web_hooks_disable_failed` flag](#failing-webhooks) must also be enabled for this feature to work. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flags](../../../administration/feature_flags.md) named `webhooks_failed_callout` and `web_hooks_disable_failed`. -On GitLab.com, this feature is not available. -The feature is not ready for production use. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/365535) in GitLab 15.7. Feature flag `webhooks_failed_callout` removed. If a webhook is failing, a banner displays at the top of the edit page explaining why it is disabled, and when it will be automatically re-enabled. For example: diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index c2952b23615..234faa893eb 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -463,6 +463,7 @@ You can edit the following issue attributes in the right sidebar: - Confidentiality - Due date - [Epic](../group/epics/index.md) +- [Health status](issues/managing_issues.md#health-status) - [Iteration](../group/iterations/index.md) - Labels - Milestone diff --git a/doc/user/project/issues/create_issues.md b/doc/user/project/issues/create_issues.md new file mode 100644 index 00000000000..3c2e20c1250 --- /dev/null +++ b/doc/user/project/issues/create_issues.md @@ -0,0 +1,221 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Create an issue **(FREE)** + +When you create an issue, you are prompted to enter the fields of the issue. +If you know the values you want to assign to an issue, you can use +[quick actions](../quick_actions.md) to enter them. + +You can create an issue in many ways in GitLab: + +- [From a project](#from-a-project) +- [From a group](#from-a-group) +- [From another issue or incident](#from-another-issue-or-incident) +- [From an issue board](#from-an-issue-board) +- [By sending an email](#by-sending-an-email) +- [Using a URL with prefilled values](#using-a-url-with-prefilled-values) +- [Using Service Desk](#using-service-desk) + +## From a project + +Prerequisites: + +- You must have at least the Guest role for the project. + +To create an issue: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. Either: + + - On the left sidebar, select **Issues**, and then, in the top right corner, select **New issue**. + - On the top bar, select the plus sign (**{plus-square}**) and then, under **This project**, + select **New issue**. + +1. Complete the [fields](#fields-in-the-new-issue-form). +1. Select **Create issue**. + +The newly created issue opens. + +## From a group + +Issues belong to projects, but when you're in a group, you can access and create issues that belong +to the projects in the group. + +Prerequisites: + +- You must have at least the Guest role for the project in the group. + +To create an issue from a group: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Issues**. +1. In the top right corner, select **Select project to create issue**. +1. Select the project you'd like to create an issue for. The button now reflects the selected + project. +1. Select **New issue in `<project name>`**. +1. Complete the [fields](#fields-in-the-new-issue-form). +1. Select **Create issue**. + +The newly created issue opens. + +The project you selected most recently becomes the default for your next visit. +This can save you a lot of time, if you mostly create issues for the same project. + +## From another issue or incident + +> - New issue becoming linked to the issue of origin [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68226) in GitLab 14.3. +> - **Relate to…** checkbox [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198494) in GitLab 14.9. + +You can create a new issue from an existing one. The two issues can then be marked as related. + +Prerequisites: + +- You must have at least the Guest role for the project. + +To create an issue from another issue: + +1. In an existing issue, select the vertical ellipsis (**{ellipsis_v}**). +1. Select **New related issue**. +1. Complete the [fields](#fields-in-the-new-issue-form). + The new issue form has a **Relate to issue #123** checkbox, where `123` is the ID of the + issue of origin. If you keep this checkbox checked, the two issues become + [linked](related_issues.md). +1. Select **Create issue**. + +The newly created issue opens. + +## From an issue board + +You can create a new issue from an [issue board](../issue_board.md). + +Prerequisites: + +- You must have at least the Guest role for the project. + +To create an issue from a project issue board: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. Select **Issues > Boards**. +1. At the top of a board list, select **New issue** (**{plus-square}**). +1. Enter the issue's title. +1. Select **Create issue**. + +To create an issue from a group issue board: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. Select **Issues > Boards**. +1. At the top of a board list, select **New issue** (**{plus-square}**). +1. Enter the issue's title. +1. Under **Projects**, select the project in the group that the issue should belong to. +1. Select **Create issue**. + +The issue is created and shows up in the board list. It shares the list's characteristic, so, for +example, if the list is scoped to a label `Frontend`, the new issue also has this label. + +## By sending an email + +> - Generated email address format changed in GitLab 11.7. +> - The older format is still supported, so existing aliases and contacts still work. + +You can send an email to create an issue in a project on the project's +**Issues List** page. + +Prerequisites: + +- Your GitLab instance must have [incoming email](../../../administration/incoming_email.md) + configured. +- There must be at least one issue in the issue list. +- You must have at least the Guest role for the project. + +To email an issue to a project: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. Select **Issues**. +1. At the bottom of the page, select **Email a new issue to this project**. +1. To copy the email address, select **Copy** (**{copy-to-clipboard}**). +1. From your email client, send an email to this address. + The subject is used as the title of the new issue, and the email body becomes the description. + You can use [Markdown](../../markdown.md) and [quick actions](../quick_actions.md). + +A new issue is created, with your user as the author. +You can save this address as a contact in your email client to use it again. + +WARNING: +The email address you see is a private email address, generated just for you. +**Keep it to yourself**, because anyone who knows it can create issues or merge requests as if they +were you. + +To regenerate the email address: + +1. On the issues list, select **Email a new issue to this project**. +1. Select **reset this token**. + +## Using a URL with prefilled values + +> - Ability to use both `issuable_template` and `issue[description]` in the same URL [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80554) in GitLab 14.9. +> - Ability to specify `add_related_issue` [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198494) in GitLab 14.9. + +To link directly to the new issue page with prefilled fields, use query +string parameters in a URL. You can embed a URL in an external +HTML page to create issues with certain fields prefilled. + +| Field | URL parameter | Notes | +| -------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| Title | `issue[title]` | Must be [URL-encoded](../../../api/index.md#namespaced-path-encoding). | +| Issue type | `issue[issue_type]` | Either `incident` or `issue`. | +| Description template | `issuable_template` | Must be [URL-encoded](../../../api/index.md#namespaced-path-encoding). | +| Description | `issue[description]` | Must be [URL-encoded](../../../api/index.md#namespaced-path-encoding). If used in combination with `issuable_template` or a [default issue template](../description_templates.md#set-a-default-template-for-merge-requests-and-issues), the `issue[description]` value is appended to the template. | +| Confidential | `issue[confidential]` | If `true`, the issue is marked as confidential. | +| Relate to… | `add_related_issue` | A numeric issue ID. If present, the issue form shows a [**Relate to…** checkbox](#from-another-issue-or-incident) to optionally link the new issue to the specified existing issue. | + +Adapt these examples to form your new issue URL with prefilled fields. +To create an issue in the GitLab project: + +- With a prefilled title and description: + + ```plaintext + https://gitlab.com/gitlab-org/gitlab/-/issues/new?issue[title]=Whoa%2C%20we%27re%20half-way%20there&issue[description]=Whoa%2C%20livin%27%20in%20a%20URL + ``` + +- With a prefilled title and description template: + + ```plaintext + https://gitlab.com/gitlab-org/gitlab/-/issues/new?issue[title]=Validate%20new%20concept&issuable_template=Feature%20Proposal%20-%20basic + ``` + +- With a prefilled title, description, and marked as confidential: + + ```plaintext + https://gitlab.com/gitlab-org/gitlab/-/issues/new?issue[title]=Validate%20new%20concept&issue[description]=Research%20idea&issue[confidential]=true + ``` + +## Using Service Desk + +To offer email support, enable [Service Desk](../service_desk.md) for your project. + +Now, when your customer sends a new email, a new issue can be created in +the appropriate project and followed up from there. + +### Fields in the new issue form + +> - Adding the new issue to an epic [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13847) in GitLab 13.1. +> - Iteration field [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/233517) in GitLab 15.6. + +When you're creating a new issue, you can complete the following fields: + +- Title +- Type: either issue (default) or incident +- [Description template](../description_templates.md): overwrites anything in the Description text box +- Description: you can use [Markdown](../../markdown.md) and [quick actions](../quick_actions.md) +- Checkbox to make the issue [confidential](confidential_issues.md) +- [Assignees](managing_issues.md#assignee) +- [Weight](issue_weight.md) +- [Epic](../../group/epics/index.md) +- [Due date](due_dates.md) +- [Milestone](../milestones/index.md) +- [Labels](../labels.md) +- [Iteration](../../group/iterations/index.md) diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index 09067b69696..6c9a645d817 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -29,7 +29,7 @@ To learn how the GitLab Strategic Marketing department uses GitLab issues with [ ## Related topics -- [Create issues](managing_issues.md#create-an-issue) +- [Create issues](create_issues.md) - [Create an issue from a template](../../project/description_templates.md#use-the-templates) - [Edit issues](managing_issues.md#edit-an-issue) - [Move issues](managing_issues.md#move-an-issue) diff --git a/doc/user/project/issues/issue_weight.md b/doc/user/project/issues/issue_weight.md index 1ba5a4415e0..d852ad3262b 100644 --- a/doc/user/project/issues/issue_weight.md +++ b/doc/user/project/issues/issue_weight.md @@ -36,7 +36,7 @@ When you change the weight of an issue, the new value overwrites the previous va ### When you create an issue -To set the issue weight when you [create an issue](managing_issues.md#create-an-issue), enter a +To set the issue weight when you [create an issue](create_issues.md), enter a number under **Weight**. ### From an existing issue diff --git a/doc/user/project/issues/managing_issues.md b/doc/user/project/issues/managing_issues.md index 8cd211a51c7..ea90dda88f6 100644 --- a/doc/user/project/issues/managing_issues.md +++ b/doc/user/project/issues/managing_issues.md @@ -6,224 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Manage issues **(FREE)** -[GitLab Issues](index.md) are the fundamental medium for collaborating on ideas and -planning work in GitLab. - -## Create an issue - -When you create an issue, you are prompted to enter the fields of the issue. -If you know the values you want to assign to an issue, you can use -[quick actions](../quick_actions.md) to enter them. - -You can create an issue in many ways in GitLab: - -- [From a project](#from-a-project) -- [From a group](#from-a-group) -- [From another issue or incident](#from-another-issue-or-incident) -- [From an issue board](#from-an-issue-board) -- [By sending an email](#by-sending-an-email) -- [Using a URL with prefilled values](#using-a-url-with-prefilled-values) -- [Using Service Desk](#using-service-desk) - -### From a project - -Prerequisites: - -- You must have at least the Guest role for the project. - -To create an issue: - -1. On the top bar, select **Main menu > Projects** and find your project. -1. Either: - - - On the left sidebar, select **Issues**, and then, in the top right corner, select **New issue**. - - On the top bar, select the plus sign (**{plus-square}**) and then, under **This project**, - select **New issue**. - -1. Complete the [fields](#fields-in-the-new-issue-form). -1. Select **Create issue**. - -The newly created issue opens. - -### From a group - -Issues belong to projects, but when you're in a group, you can access and create issues that belong -to the projects in the group. - -Prerequisites: - -- You must have at least the Guest role for the project in the group. - -To create an issue from a group: - -1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Issues**. -1. In the top right corner, select **Select project to create issue**. -1. Select the project you'd like to create an issue for. The button now reflects the selected - project. -1. Select **New issue in `<project name>`**. -1. Complete the [fields](#fields-in-the-new-issue-form). -1. Select **Create issue**. - -The newly created issue opens. - -The project you selected most recently becomes the default for your next visit. -This can save you a lot of time, if you mostly create issues for the same project. - -### From another issue or incident - -> - New issue becoming linked to the issue of origin [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68226) in GitLab 14.3. -> - **Relate to…** checkbox [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198494) in GitLab 14.9. - -You can create a new issue from an existing one. The two issues can then be marked as related. - -Prerequisites: - -- You must have at least the Guest role for the project. - -To create an issue from another issue: - -1. In an existing issue, select the vertical ellipsis (**{ellipsis_v}**). -1. Select **New related issue**. -1. Complete the [fields](#fields-in-the-new-issue-form). - The new issue form has a **Relate to issue #123** checkbox, where `123` is the ID of the - issue of origin. If you keep this checkbox checked, the two issues become - [linked](related_issues.md). -1. Select **Create issue**. - -The newly created issue opens. - -### From an issue board - -You can create a new issue from an [issue board](../issue_board.md). - -Prerequisites: - -- You must have at least the Guest role for the project. - -To create an issue from a project issue board: - -1. On the top bar, select **Main menu > Projects** and find your project. -1. Select **Issues > Boards**. -1. At the top of a board list, select **New issue** (**{plus-square}**). -1. Enter the issue's title. -1. Select **Create issue**. - -To create an issue from a group issue board: - -1. On the top bar, select **Main menu > Groups** and find your group. -1. Select **Issues > Boards**. -1. At the top of a board list, select **New issue** (**{plus-square}**). -1. Enter the issue's title. -1. Under **Projects**, select the project in the group that the issue should belong to. -1. Select **Create issue**. - -The issue is created and shows up in the board list. It shares the list's characteristic, so, for -example, if the list is scoped to a label `Frontend`, the new issue also has this label. - -### By sending an email - -> - Generated email address format changed in GitLab 11.7. -> - The older format is still supported, so existing aliases and contacts still work. - -You can send an email to create an issue in a project on the project's -**Issues List** page. - -Prerequisites: - -- Your GitLab instance must have [incoming email](../../../administration/incoming_email.md) - configured. -- There must be at least one issue in the issue list. -- You must have at least the Guest role for the project. - -To email an issue to a project: - -1. On the top bar, select **Main menu > Projects** and find your project. -1. Select **Issues**. -1. At the bottom of the page, select **Email a new issue to this project**. -1. To copy the email address, select **Copy** (**{copy-to-clipboard}**). -1. From your email client, send an email to this address. - The subject is used as the title of the new issue, and the email body becomes the description. - You can use [Markdown](../../markdown.md) and [quick actions](../quick_actions.md). - -A new issue is created, with your user as the author. -You can save this address as a contact in your email client to use it again. - -WARNING: -The email address you see is a private email address, generated just for you. -**Keep it to yourself**, because anyone who knows it can create issues or merge requests as if they -were you. - -To regenerate the email address: - -1. On the issues list, select **Email a new issue to this project**. -1. Select **reset this token**. - -### Using a URL with prefilled values - -> - Ability to use both `issuable_template` and `issue[description]` in the same URL [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80554) in GitLab 14.9. -> - Ability to specify `add_related_issue` [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198494) in GitLab 14.9. - -To link directly to the new issue page with prefilled fields, use query -string parameters in a URL. You can embed a URL in an external -HTML page to create issues with certain fields prefilled. - -| Field | URL parameter | Notes | -| -------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------- | -| Title | `issue[title]` | Must be [URL-encoded](../../../api/index.md#namespaced-path-encoding). | -| Issue type | `issue[issue_type]` | Either `incident` or `issue`. | -| Description template | `issuable_template` | Must be [URL-encoded](../../../api/index.md#namespaced-path-encoding). | -| Description | `issue[description]` | Must be [URL-encoded](../../../api/index.md#namespaced-path-encoding). If used in combination with `issuable_template` or a [default issue template](../description_templates.md#set-a-default-template-for-merge-requests-and-issues), the `issue[description]` value is appended to the template. | -| Confidential | `issue[confidential]` | If `true`, the issue is marked as confidential. | -| Relate to… | `add_related_issue` | A numeric issue ID. If present, the issue form shows a [**Relate to…** checkbox](#from-another-issue-or-incident) to optionally link the new issue to the specified existing issue. | - -Adapt these examples to form your new issue URL with prefilled fields. -To create an issue in the GitLab project: - -- With a prefilled title and description: - - ```plaintext - https://gitlab.com/gitlab-org/gitlab/-/issues/new?issue[title]=Whoa%2C%20we%27re%20half-way%20there&issue[description]=Whoa%2C%20livin%27%20in%20a%20URL - ``` - -- With a prefilled title and description template: - - ```plaintext - https://gitlab.com/gitlab-org/gitlab/-/issues/new?issue[title]=Validate%20new%20concept&issuable_template=Feature%20Proposal%20-%20basic - ``` - -- With a prefilled title, description, and marked as confidential: - - ```plaintext - https://gitlab.com/gitlab-org/gitlab/-/issues/new?issue[title]=Validate%20new%20concept&issue[description]=Research%20idea&issue[confidential]=true - ``` - -### Using Service Desk - -To offer email support, enable [Service Desk](../service_desk.md) for your project. - -Now, when your customer sends a new email, a new issue can be created in -the appropriate project and followed up from there. - -### Fields in the new issue form - -> - Adding the new issue to an epic [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13847) in GitLab 13.1. -> - Iteration field [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/233517) in GitLab 15.6. - -When you're creating a new issue, you can complete the following fields: - -- Title -- Type: either issue (default) or incident -- [Description template](../description_templates.md): overwrites anything in the Description text box -- Description: you can use [Markdown](../../markdown.md) and [quick actions](../quick_actions.md) -- Checkbox to make the issue [confidential](confidential_issues.md) -- [Assignees](#assignee) -- [Weight](issue_weight.md) -- [Epic](../../group/epics/index.md) -- [Due date](due_dates.md) -- [Milestone](../milestones/index.md) -- [Labels](../labels.md) -- [Iteration](../../group/iterations/index.md) +After you create an issue, you can start working with it. ## Edit an issue @@ -239,27 +22,7 @@ To edit an issue: 1. Edit the available fields. 1. Select **Save changes**. -### Reorder list items in the issue description - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15260) in GitLab 15.0. - -When you view an issue that has a list in the description, you can also reorder the list items. - -Prerequisites: - -- You must have at least the Reporter role for the project, be the author of the issue, or be - assigned to the issue. -- The issue's description must have an [ordered, unordered](../../markdown.md#lists), or - [task](../../markdown.md#task-lists) list. - -To reorder list items, when viewing an issue: - -1. Hover over the list item row to make the drag icon (**{drag-vertical}**) visible. -1. Select and hold the drag icon. -1. Drag the row to the new position in the list. -1. Release the drag icon. - -### Bulk edit issues from a project +## Bulk edit issues from a project > - Assigning epic [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/210470) in GitLab 13.2. > - Editing health status [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218395) in GitLab 13.2. @@ -283,7 +46,7 @@ To edit multiple issues at the same time: When bulk editing issues in a project, you can edit the following attributes: - Status (open or closed) -- [Assignees](#assignee) +- [Assignees](managing_issues.md#assignee) - [Epic](../../group/epics/index.md) - [Milestone](../milestones/index.md) - [Labels](../labels.md) @@ -396,6 +159,26 @@ To do it: 1. To exit the Rails console, enter `quit`. +## Reorder list items in the issue description + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15260) in GitLab 15.0. + +When you view an issue that has a list in the description, you can also reorder the list items. + +Prerequisites: + +- You must have at least the Reporter role for the project, be the author of the issue, or be + assigned to the issue. +- The issue's description must have an [ordered, unordered](../../markdown.md#lists), or + [task](../../markdown.md#task-lists) list. + +To reorder list items, when viewing an issue: + +1. Hover over the list item row to make the drag icon (**{drag-vertical}**) visible. +1. Select and hold the drag icon. +1. Drag the row to the new position in the list. +1. Release the drag icon. + ## Close an issue When you decide that an issue is resolved or no longer needed, you can close it. @@ -514,8 +297,7 @@ Prerequisites: - You must have [administrator access](../../../administration/index.md) to your GitLab instance. -To change the default issue closing pattern, edit the -[`gitlab.rb` or `gitlab.yml` file](../../../administration/issue_closing_pattern.md) +Learn how to change the default [issue closing pattern](../../../administration/issue_closing_pattern.md). of your installation. ## Change the issue type @@ -622,7 +404,7 @@ GitLab displays the results on-screen, but you can also ### Filter with the OR operator -> OR filtering for assignees was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23532) in GitLab 15.6 [with a flag](../../../administration/feature_flags.md) named `or_issuable_queries`. Disabled by default. +> OR filtering for author and assignee was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23532) in GitLab 15.6 [with a flag](../../../administration/feature_flags.md) named `or_issuable_queries`. Disabled by default. FLAG: On self-managed GitLab, by default this feature is not available. @@ -633,7 +415,7 @@ When this feature is enabled, you can use the OR operator (**is one of: `||`**) when you [filter the list of issues](#filter-the-list-of-issues). `is one of` represents an inclusive OR. For example, if you filter by `Assignee is one of Sidney Jones` and -`Assignee is one of Zhang Wei`, GitLab shows issues where either Sidney, Zhang, or both of them are assignees. +`Assignee is one of Zhang Wei`, GitLab shows issues where either `Sidney`, `Zhang`, or both of them are assignees. ### Filter issues by ID @@ -674,22 +456,6 @@ To copy the issue's email address: 1. Go to the issue. 1. On the right sidebar, next to **Issue email**, select **Copy Reference** (**{copy-to-clipboard}**). -## Real-time sidebar - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/17589) in GitLab 13.3. Disabled by default. -> - [Enabled on GitLab.com](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/3413) in GitLab 13.9. -> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/17589) in GitLab 14.5. -> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/17589) in GitLab 14.9. Feature flags `real_time_issue_sidebar` and `broadcast_issue_updates` removed. - -Some sections of the right sidebar are updated in real time. -When you're viewing an issue and somebody changes one of the values, -you can see the change without having to refresh the page. - -The following sections are updated in real time: - -- [Assignee](#assignee) -- [Labels](../labels.md#assign-and-unassign-labels) - ## Assignee An issue can be assigned to one or [more users](multiple_assignees_for_issues.md). @@ -708,6 +474,8 @@ To change the assignee on an issue: 1. From the dropdown list, select the user to add as an assignee. 1. Select any area outside the dropdown list. +The assignee is changed without having to refresh the page. + ## Similar issues To prevent duplication of issues on the same topic, GitLab searches for similar issues diff --git a/doc/user/project/issues/sorting_issue_lists.md b/doc/user/project/issues/sorting_issue_lists.md index 6a1a791645e..a2f90d5c444 100644 --- a/doc/user/project/issues/sorting_issue_lists.md +++ b/doc/user/project/issues/sorting_issue_lists.md @@ -101,6 +101,19 @@ title in this order: - Numbers - Letters: first Latin, then accented (for example, `ö`) +## Sorting by health status **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/377841) in GitLab 15.7. + +When you sort by **Health**, the issue list changes to sort by the +[health status](managing_issues.md#health-status) of the issues +When in descending order, the issues are shown in the following order: + +1. **At risk** issues +1. **Needs attention** issues +1. **On track** issues +1. All other issues + ## Sorting by weight When you sort by **Weight**, the issue list changes to sort ascending by the diff --git a/doc/user/project/members/index.md b/doc/user/project/members/index.md index e8ec954df8f..a7627f12657 100644 --- a/doc/user/project/members/index.md +++ b/doc/user/project/members/index.md @@ -61,7 +61,7 @@ To add a user to a project: 1. Select **Invite members**. 1. Enter an email address and select a [role](../../permissions.md). 1. Optional. Select an **Access expiration date**. - From that date onwards, the user can no longer access the project. + From that date onward, the user can no longer access the project. 1. Select **Invite**. If the user has a GitLab account, they are added to the members list. @@ -110,7 +110,7 @@ To add a group to a project: 1. Select a group. 1. Select the highest [role](../../permissions.md) for users in the group. 1. Optional. Select an **Access expiration date**. - From that date onwards, the group can no longer access the project. + From that date onward, the group can no longer access the project. 1. Select **Invite**. The members of the group are not displayed on the **Members** tab. diff --git a/doc/user/project/members/share_project_with_groups.md b/doc/user/project/members/share_project_with_groups.md index 52cc9fc4f9b..b9887212be0 100644 --- a/doc/user/project/members/share_project_with_groups.md +++ b/doc/user/project/members/share_project_with_groups.md @@ -9,83 +9,77 @@ info: To determine the technical writer assigned to the Stage/Group associated w You can share projects with other [groups](../../group/index.md). This makes it possible to add a group of users to a project with a single action. -## Groups as collections of users +For example, if `Project A` belongs to `Group 1`, the members of `Group 1` have access to the project. +If `Project A` already belongs to another `Group 2`, the owner of `Group 2` can share `Project A` +with `Group 1`, so that both members of `Group 1` and `Group 2` have access to the project. -Groups are used primarily to [create collections of projects](../../group/index.md), but you can also -take advantage of the fact that groups define collections of _users_, namely the group -members. +When a project is shared with a group: -## Share a project with a group of users - -> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/247208) in GitLab 13.11 from a form to a modal - window [with a flag](../../feature_flags.md). Disabled by default. -> - Modal window [enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/247208) - in GitLab 14.8. -> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/352526) in GitLab 14.9. - [Feature flag `invite_members_group_modal`](https://gitlab.com/gitlab-org/gitlab/-/issues/352526) removed. - -You can share a project only with: - -- Groups for which you have an explicitly defined [membership](index.md). -- Groups that contain a nested subgroup or project for which you have an explicitly defined role. +- All group members, including members of subgroups or projects that belong to the group, +are assigned the same role in the project. +Each member's role is displayed in **Project information > Members > Max role**. +- The group is listed in the **Groups** tab. +- The project is listed on the group dashboard. -Administrators can share projects with any group in the instance. +Be aware of the restrictions that apply when you share projects with: -The primary mechanism to give a group of users, say 'Engineering', access to a project, -say 'Project Acme', in GitLab is to make the 'Engineering' group the owner of 'Project -Acme'. But what if 'Project Acme' already belongs to another group, say 'Open Source'? -This is where the group sharing feature can be of use. +- [Groups with a more restrictive visibility level](#share-projects-with-groups-with-a-more-restrictive-visibility-level). +- [Restricted sharing](#prevent-project-sharing). -To share 'Project Acme' with the 'Engineering' group: +## Share projects with groups with a more restrictive visibility level -1. For 'Project Acme' use the left navigation menu to go to **Project information > Members**. -1. Select **Invite a group**. -1. Add the 'Engineering' group with the maximum access level of your choice. -1. Optional. Select an **Access expiration date**. -1. Select **Invite**. +You can share projects only down the group's organization structure. +This means you can share a project with a group that has a more restrictive +[visibility level](../../public_access.md#project-and-group-visibility) than the project, +but not with a group that has a less restrictive visibility level. -After sharing 'Project Acme' with 'Engineering': +For example, you can share: -- The group is listed in the **Groups** tab. -- The project is listed on the group dashboard. -- All members, including members from the ancestors of the 'Engineering' group, gain access to 'Project Acme' with an access level based on the outcome of [maximum access level](#maximum-access-level). +- A public project with a private group. +- A public project with an internal group. +- An internal project with a private group. -When you share a project, be aware of the following restrictions and outcomes: +This restriction applies to subgroups as well. For example, `group/subgroup01/project`: -- [Maximum access level](#maximum-access-level) -- [Sharing projects with groups of a higher restrictive visibility level](#sharing-projects-with-groups-of-a-higher-restrictive-visibility-level) -- [Sharing project with group lock](#share-project-with-group-lock) +- Can't be shared with `group`. +- Can be shared with `group/subgroup02` or `group/subgroup01/subgroup03`. -## Maximum access level +When you share a project with a group that has a more restrictive visibility level than the project: -In the example above, the maximum access level of 'Developer' for members from 'Engineering' means that users with higher access levels in 'Engineering' ('Maintainer' or 'Owner') only have 'Developer' access to 'Project Acme'. - -### Share a project with a subgroup - -You can't share a project with a group that's an ancestor of a [subgroup](../../group/subgroups/index.md) the project is -in. That means you can only share down the hierarchy. For example, `group/subgroup01/project`: +- The group name is visible to all users that can view the project members page. +- Owners of the project have access to members of the group when they mention them in issues or merge requests. +- Project members who are direct or indirect members of the group can see +group members listed in addition to members of the project. -- Can not be shared with `group`. -- Can be shared with `group/subgroup02` or `group/subgroup01/subgroup03`. +## Share a project with a group -## Sharing projects with groups of a higher restrictive visibility level +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/247208) in GitLab 13.11 from a form to a modal + window [with a flag](../../feature_flags.md). Disabled by default. +> - Modal window [enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/247208) + in GitLab 14.8. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/352526) in GitLab 14.9. + [Feature flag `invite_members_group_modal`](https://gitlab.com/gitlab-org/gitlab/-/issues/352526) removed. -There are several outcomes you must be aware of when you share a project with a group that has a more restrictive [visibility level](../../public_access.md#project-and-group-visibility) than the project. For example, when you: +You can share a project only with groups: -- Share a public project with a private group. -- Share a public project with an internal group. -- Share an internal project with a private group. +- Where you have an explicitly defined [membership](index.md). +- That contain a nested subgroup or project you have an explicitly defined role for. +- You are an administrator of. -The following outcomes occur: +To share a project with a group: -- The group name is visible to all users that can view the project members page. -- Owners of the project have access to members of the group when they mention them in issues or merge requests. -- Project members who are direct or indirect members of the group can see group members listed in addition to members of the project. +1. On the top bar, select **Main menu > Projects** and find your project. +1. In the left navigation menu, select **Project information > Members**. +1. Select **Invite a group**. +1. **Select a group** you want to add to the project. +1. **Select a role** you want to assign to the group. +1. Optional. Select an **Access expiration date**. +1. Select **Invite**. -## Share project with group lock +## Prevent project sharing -It is possible to prevent projects in a group from +You can prevent members of a group from [sharing a project with another group](../members/share_project_with_groups.md). -This allows for tighter control over project access. +This restriction allows for tighter control over project access. -Learn more about [Share with group lock](../../group/access_and_permissions.md#prevent-a-project-from-being-shared-with-groups). +For more information, see [Prevent a project from being shared with groups](../../group/access_and_permissions.md#prevent-a-project-from-being-shared-with-groups). diff --git a/doc/user/project/merge_requests/approvals/index.md b/doc/user/project/merge_requests/approvals/index.md index eb460225858..92ff78082e3 100644 --- a/doc/user/project/merge_requests/approvals/index.md +++ b/doc/user/project/merge_requests/approvals/index.md @@ -22,8 +22,10 @@ flexibility: - Specify a list of users who act as [code owners](../../code_owners.md) for specific files, and require their approval before work can merge. -You can configure merge request approvals on a per-project basis, and -[on the group level](../../../group/manage.md#group-merge-request-approval-settings). Administrators of +You can configure merge request approvals on a per-project basis, and some approvals can be configured +[on the group level](../../../group/manage.md#group-merge-request-approval-settings). Support for +group-level settings for merge request approval rules is tracked in this +[epic](https://gitlab.com/groups/gitlab-org/-/epics/4367). Administrators of [GitLab Premium](https://about.gitlab.com/pricing/) and [GitLab Ultimate](https://about.gitlab.com/pricing/) self-managed GitLab instances can also configure approvals diff --git a/doc/user/project/merge_requests/approvals/rules.md b/doc/user/project/merge_requests/approvals/rules.md index e09a1318981..5f81db10cf4 100644 --- a/doc/user/project/merge_requests/approvals/rules.md +++ b/doc/user/project/merge_requests/approvals/rules.md @@ -182,7 +182,7 @@ granting them push access: 1. [Create a new group](../../../group/manage.md#create-a-group). 1. [Add the user to the group](../../../group/manage.md#add-users-to-a-group), and select the Reporter role for the user. -1. [Share the project with your group](../../members/share_project_with_groups.md#share-a-project-with-a-group-of-users), +1. [Share the project with your group](../../members/share_project_with_groups.md#share-a-project-with-a-group), based on the Reporter role. 1. Go to your project and select **Settings > Merge requests**. 1. In the **Merge request approvals** section, scroll to **Approval rules**, and either: diff --git a/doc/user/project/merge_requests/approvals/settings.md b/doc/user/project/merge_requests/approvals/settings.md index a2a12b22c3b..a8acab3898b 100644 --- a/doc/user/project/merge_requests/approvals/settings.md +++ b/doc/user/project/merge_requests/approvals/settings.md @@ -21,22 +21,24 @@ To view or edit merge request approval settings: ### Approval settings -These settings limit who can approve merge requests. - -| Setting | Description | -| ------ | ------ | -| [Prevent approval by author](#prevent-approval-by-author) | When enabled, the author of a merge request cannot approve it. | -| [Prevent approvals by users who add commits](#prevent-approvals-by-users-who-add-commits) | When enabled, users who have committed to a merge request cannot approve it. | -| [Prevent editing approval rules in merge requests](#prevent-editing-approval-rules-in-merge-requests) | When enabled, users can't override the project's approval rules on merge requests. | -| [Require user password to approve](#require-user-password-to-approve) | Force potential approvers to first authenticate with a password. | - -You can further define what happens to existing approvals when commits are added to the merge request. - -| Setting | Description | -| ------ | ------ | -| Keep approvals | Do not remove approvals. | -| [Remove all approvals](#remove-all-approvals-when-commits-are-added-to-the-source-branch) | Remove all existing approvals. | -| [Remove approvals by Code Owners if their files changed](#remove-approvals-by-code-owners-if-their-files-changed) | If a Code Owner has approved the merge request, and the commit changes files they are the Code Owner for, their approval is removed. | +These settings limit who can approve merge requests: + +- [**Prevent approval by author**](#prevent-approval-by-author): + Prevents the author of a merge request from approving it. +- [**Prevent approvals by users who add commits**](#prevent-approvals-by-users-who-add-commits): + Prevents users who add commits to a merge request from also approving it. +- [**Prevent editing approval rules in merge requests**](#prevent-editing-approval-rules-in-merge-requests): + Prevents users from overriding project level approval rules on merge requests. +- [**Require user password to approve**](#require-user-password-to-approve): + Force potential approvers to first authenticate with a password. +- Code Owner approval removals: Define what happens to existing approvals when + commits are added to the merge request. + - **Keep approvals**: Do not remove any approvals. + - [**Remove all approvals**](#remove-all-approvals-when-commits-are-added-to-the-source-branch): + Remove all existing approvals. + - [**Remove approvals by Code Owners if their files changed**](#remove-approvals-by-code-owners-if-their-files-changed): + If a Code Owner approves a merge request, and a later commit changes files + they are a Code Owner for, their approval is removed. ## Prevent approval by author diff --git a/doc/user/project/merge_requests/changes.md b/doc/user/project/merge_requests/changes.md index 6e8b0cb1a75..f6e02dc0dfe 100644 --- a/doc/user/project/merge_requests/changes.md +++ b/doc/user/project/merge_requests/changes.md @@ -136,15 +136,10 @@ Files marked as viewed are not shown to you again unless either: - New changes are made to its content. - You clear the **Viewed** checkbox. -## Show merge request conflicts in diff **(FREE SELF)** +## Show merge request conflicts in diff -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/232484) in GitLab 13.5 [with a flag](../../../administration/feature_flags.md) named `display_merge_conflicts_in_diff`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) -named `display_merge_conflicts_in_diff`. On GitLab.com, this feature is not available. -The feature is not ready for production use. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/232484) in GitLab 13.5 [with a flag](../../../administration/feature_flags.md) named `display_merge_conflicts_in_diff`. Disabled by default. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/276918) in GitLab 15.7. To avoid displaying the changes that are already on target branch in the diff, we compare the merge request's source branch with HEAD of the target branch. diff --git a/doc/user/project/merge_requests/commit_templates.md b/doc/user/project/merge_requests/commit_templates.md index 75c2bdffae8..a14d8bddd24 100644 --- a/doc/user/project/merge_requests/commit_templates.md +++ b/doc/user/project/merge_requests/commit_templates.md @@ -70,6 +70,7 @@ GitLab creates a squash commit message with this template: > - [Added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75639) `url`, `approved_by`, and `merged_by` variables in GitLab 14.7. > - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/20421) `co_authored_by` variable in GitLab 14.7. > - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/26303) `all_commits` variable in GitLab 14.9. +> - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/378352) `reviewed_by` variable in GitLab 15.7. Commit message templates support these variables: @@ -84,7 +85,8 @@ Commit message templates support these variables: | `%{first_commit}` | Full message of the first commit in merge request diff. | `Update README.md` | | `%{first_multiline_commit}` | Full message of the first commit that's not a merge commit and has more than one line in message body. Merge request title if all commits aren't multiline. | `Update README.md`<br><br>`Improved project description in readme file.` | | `%{url}` | Full URL to the merge request. | `https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1` | -| `%{approved_by}` | Line-separated list of the merge request approvers. | `Approved-by: Sidney Jones <sjones@example.com>` <br> `Approved-by: Zhang Wei <zwei@example.com>` | +| `%{reviewed_by}` | Line-separated list of the merge request reviewers, based on users who submit a review via batch comments, in a `Reviewed-by` Git commit trailer format. | `Reviewed-by: Sidney Jones <sjones@example.com>` <br> `Reviewed-by: Zhang Wei <zwei@example.com>` | +| `%{approved_by}` | Line-separated list of the merge request approvers in a `Approved-by` Git commit trailer format. | `Approved-by: Sidney Jones <sjones@example.com>` <br> `Approved-by: Zhang Wei <zwei@example.com>` | | `%{merged_by}` | User who merged the merge request. | `Alex Garcia <agarcia@example.com>` | | `%{co_authored_by}` | Names and emails of commit authors in a `Co-authored-by` Git commit trailer format. Limited to authors of 100 most recent commits in merge request. | `Co-authored-by: Zane Doe <zdoe@example.com>` <br> `Co-authored-by: Blake Smith <bsmith@example.com>` | | `%{all_commits}` | Messages from all commits in the merge request. Limited to 100 most recent commits. Skips commit bodies exceeding 100KiB and merge commit messages. | `* Feature introduced` <br><br> `This commit implements feature` <br> `Changelog:added` <br><br> `* Bug fixed` <br><br> `* Documentation improved` <br><br>`This commit introduced better docs.`| diff --git a/doc/user/project/merge_requests/commits.md b/doc/user/project/merge_requests/commits.md index a6ae3ac80a5..a9f67c39ae8 100644 --- a/doc/user/project/merge_requests/commits.md +++ b/doc/user/project/merge_requests/commits.md @@ -1,56 +1,11 @@ --- -stage: Create -group: Code Review -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -type: index, reference +redirect_to: '../merge_requests/index.md' +remove_date: '2023-03-12' --- -# Commits tab in merge requests **(FREE)** +This document was removed. -The **Commits** tab in a merge request displays a sequential list of commits -to the Git branch your merge request is based on. From this page, you can review -full commit messages and copy a commit's SHA when you need to -[cherry-pick changes](cherry_pick_changes.md). - -## Merge requests commit navigation - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18140) in GitLab 13.0. - -To seamlessly navigate among commits in a merge request: - -1. Select the **Commits** tab. -1. Select a commit to open it in the single-commit view. -1. Navigate through the commits by either: - - - Selecting **Prev** and **Next** buttons below the tab buttons. - - Using the <kbd>X</kbd> and <kbd>C</kbd> keyboard shortcuts. - - - -## View merge request commits in context - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29274) in GitLab 13.12 [with a flag](../../../administration/feature_flags.md) named `context_commits`. Enabled by default. -> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/320757) in GitLab 14.8. -> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/320757) in GitLab 14.9. [Feature flag `context_commits`](https://gitlab.com/gitlab-org/gitlab/-/issues/320757) removed. - -When reviewing a merge request, it helps to have more context about the changes -made. That includes unchanged lines in unchanged files, and previous commits -that have already merged that the change is built on. - -To add previously merged commits to a merge request for more context: - -1. Go to your merge request. -1. Select the **Commits** tab. -1. Scroll to the end of the list of commits, and select **Add previously merged commits**: - -  - -1. Select the commits that you want to add. -1. Select **Save changes**. - -To view the changes done on those previously merged commits: - -1. On your merge request, select the **Changes** tab. -1. Scroll to **(file-tree)** **Compare** and select **previously merged commits**: - -  +<!-- This redirect file can be deleted after <2023-03-12>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/project/merge_requests/conflicts.md b/doc/user/project/merge_requests/conflicts.md index 24f22924a08..6063d64a721 100644 --- a/doc/user/project/merge_requests/conflicts.md +++ b/doc/user/project/merge_requests/conflicts.md @@ -75,7 +75,7 @@ To resolve less-complex conflicts from the GitLab user interface: Resolving conflicts merges the target branch of the merge request into the source branch, using the version of the text you chose. If the source branch is `feature` and the target branch is `main`, these actions are similar to running -`git checkout feature; git merge main` locally. +`git switch feature; git merge main` locally. ## Resolve conflicts in the inline editor @@ -101,7 +101,7 @@ most control over each change: 1. Open the terminal and check out your feature branch. For example, `my-feature-branch`: ```shell - git checkout my-feature-branch + git switch my-feature-branch ``` 1. [Rebase your branch](../../../topics/git/git_rebase.md#regular-rebase) against the diff --git a/doc/user/project/merge_requests/creating_merge_requests.md b/doc/user/project/merge_requests/creating_merge_requests.md index df11d5a1d8d..eae4db2d4f7 100644 --- a/doc/user/project/merge_requests/creating_merge_requests.md +++ b/doc/user/project/merge_requests/creating_merge_requests.md @@ -102,7 +102,7 @@ You can create a merge request from your fork to contribute back to the main pro change the default target branch (which can be useful if you are working in a forked project). 1. Select **Compare branches and continue**. -1. Select **Submit merge request**. +1. Select **Create merge request**. After your work is merged, if you don't intend to make any other contributions to the upstream project, you can unlink your diff --git a/doc/user/project/merge_requests/img/add_previously_merged_commits_button_v14_1.png b/doc/user/project/merge_requests/img/add_previously_merged_commits_button_v14_1.png Binary files differdeleted file mode 100644 index e60e869f854..00000000000 --- a/doc/user/project/merge_requests/img/add_previously_merged_commits_button_v14_1.png +++ /dev/null diff --git a/doc/user/project/merge_requests/img/commit_nav_v13_11.png b/doc/user/project/merge_requests/img/commit_nav_v13_11.png Binary files differdeleted file mode 100644 index a9bc8fa6bee..00000000000 --- a/doc/user/project/merge_requests/img/commit_nav_v13_11.png +++ /dev/null diff --git a/doc/user/project/merge_requests/img/previously_merged_commits_v14_1.png b/doc/user/project/merge_requests/img/previously_merged_commits_v14_1.png Binary files differdeleted file mode 100644 index 4f49fad10ad..00000000000 --- a/doc/user/project/merge_requests/img/previously_merged_commits_v14_1.png +++ /dev/null diff --git a/doc/user/project/merge_requests/merge_request_dependencies.md b/doc/user/project/merge_requests/merge_request_dependencies.md deleted file mode 100644 index 6242a77e931..00000000000 --- a/doc/user/project/merge_requests/merge_request_dependencies.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'dependencies.md' -remove_date: '2022-11-22' ---- - -This document was moved to [another location](dependencies.md). - -<!-- This redirect file can be deleted after <2022-11-22>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/project/merge_requests/methods/index.md b/doc/user/project/merge_requests/methods/index.md index e72c927198e..249a98f1779 100644 --- a/doc/user/project/merge_requests/methods/index.md +++ b/doc/user/project/merge_requests/methods/index.md @@ -10,47 +10,107 @@ type: reference, concepts The merge method you select for your project determines how the changes in your merge requests are merged into an existing branch. +The examples on this page assume a `main` branch with commits A, C, and E, and a +`feature` branch with commits B and D: + +```mermaid +gitGraph + commit id: "A" + branch feature + commit id: "B" + commit id: "D" + checkout main + commit id: "C" + commit id: "E" +``` + ## Configure a project's merge method 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Settings > Merge requests**. -1. In the **Merge method** section, select your desired merge method. +1. Select your desired **Merge method** from these options: + - Merge commit + - Merge commit with semi-linear history + - Fast-forward merge +1. In **Squash commits when merging**, select the default behavior for handling commits: + - **Do not allow**: Squashing is never performed, and the user cannot change the behavior. + - **Allow**: Squashing is off by default, but the user can change the behavior. + - **Encourage**: Squashing is on by default, but the user can change the behavior. + - **Require**: Squashing is always performed, and the user cannot change the behavior. 1. Select **Save changes**. ## Merge commit -This setting is the default. It always creates a separate merge commit, -even when using [squash](../squash_and_merge.md). An example commit graph generated using this merge method: +By default, GitLab creates a merge commit when a branch is merged into `main`. +A separate merge commit is always created, regardless of whether or not commits +are [squashed when merging](../squash_and_merge.md). This strategy can result +in both a squash commit and a merge commit being added to your `main` branch. + +These diagrams show how the `feature` branch merges into `main` if you use the +**Merge commit** strategy. They are equivalent to the command `git merge --no-ff <feature>`, +and selecting `Merge commit` as the **Merge method** in the GitLab UI: + +The merge strategy: ```mermaid +%%{init: { 'gitGraph': {'logLevel': 'debug', 'showBranches': true, 'showCommitLabel':true,'mainBranchName': 'main'}} }%% gitGraph - commit id: "Init" - branch mr-branch-1 - commit - checkout main - commit - branch mr-branch-2 - commit - checkout mr-branch-1 - commit - checkout main - branch squash-mr - commit id: "Squashed commits" - checkout main - merge squash-mr - merge mr-branch-1 - commit - merge mr-branch-2 + commit id: "A" + branch feature + commit id: "B" + commit id: "D" + checkout main + commit id: "C" + commit id: "E" + merge feature +``` + +After a feature branch is merged with the **Merge commit** method, your `main` branch +looks like this: + +```mermaid +%%{init: { 'gitGraph': {'logLevel': 'debug', 'showBranches': true, 'showCommitLabel':true,'mainBranchName': 'main'}} }%% +gitGraph + commit id: "A" + commit id: "C" + commit id: "E" + commit id: "squash commit" + commit id: "merge commit" ``` -- For regular merges, it is equivalent to the command `git merge --no-ff <source-branch>`. -- For squash merges, it squashes all commits in the source branch before merging it normally. It performs actions similar to: +In comparison, a **squash merge** constructs a squash commit, a virtual copy of all commits +from the `feature` branch. The original commits (B and D) remain unchanged +on the `feature` branch, and the squash commit is placed on the `main` branch: + +```mermaid +%%{init: { 'gitGraph': {'showBranches': true, 'showCommitLabel':true,'mainBranchName': 'main'}} }%% +gitGraph + commit id:"A" + branch feature + checkout main + commit id:"C" + checkout feature + commit id:"B" + commit id:"D" + checkout main + commit id:"E" + commit id:"squash commit" type: HIGHLIGHT +``` + +The squash merge graph is equivalent to these settings in the GitLab UI: + +- **Merge method**: Merge commit. +- **Squash commits when merging** should be set to either: + - Require. + - Either Allow or Encourage, and squashing must be selected on the merge request. + +The squash merge graph is also equivalent to these commands: ```shell - git checkout `git merge-base <source-branch> <target-branch>` - git merge --squash <source-branch> + git checkout `git merge-base feature main` + git merge --squash <feature> SOURCE_SHA=`git rev-parse HEAD` - git checkout <target-branch> + git checkout <main> git merge --no-ff $SOURCE_SHA ``` @@ -58,7 +118,8 @@ gitGraph A merge commit is created for every merge, but the branch is only merged if a fast-forward merge is possible. This ensures that if the merge request build -succeeded, the target branch build also succeeds after the merge. An example commit graph generated using this merge method: +succeeded, the target branch build also succeeds after the merge. An example +commit graph generated using this merge method: ```mermaid gitGraph @@ -113,8 +174,8 @@ This method is equivalent to `git merge --ff <source-branch>` for regular merges When the fast-forward merge ([`--ff-only`](https://git-scm.com/docs/git-merge#git-merge---ff-only)) setting -is enabled, no merge commits are created and all merges are fast-forwarded, -which means that merging is only allowed if the branch can be fast-forwarded. +is enabled, no merge commits are created and all merges are fast-forwarded. +Merging is only allowed if the branch can be fast-forwarded. When a fast-forward merge is not possible, the user is given the option to rebase, see [Rebasing in (semi-)linear merge methods](#rebasing-in-semi-linear-merge-methods). @@ -136,11 +197,16 @@ In these merge methods, you can merge only when your source branch is up-to-date - Fast-forward merge. If a fast-forward merge is not possible but a conflict-free rebase is possible, -GitLab offers you the [`/rebase` quick action](../../../../topics/git/git_rebase.md#rebase-from-the-gitlab-ui), -and the ability to select **Rebase** from the user interface. +GitLab provides: + +- The [`/rebase` quick action](../../../../topics/git/git_rebase.md#rebase-from-the-gitlab-ui). +- The option to select **Rebase** in the user interface. + +You must rebase the source branch locally before a fast-forward merge if both +conditions are true: -If the target branch is ahead of the source branch and a conflict-free rebase is -not possible, you must rebase the source branch locally before you can do a fast-forward merge. +- The target branch is ahead of the source branch. +- A conflict-free rebase is not possible.  diff --git a/doc/user/project/merge_requests/reviews/data_usage.md b/doc/user/project/merge_requests/reviews/data_usage.md index 3a3af7a24bc..f17015aef4e 100644 --- a/doc/user/project/merge_requests/reviews/data_usage.md +++ b/doc/user/project/merge_requests/reviews/data_usage.md @@ -27,7 +27,7 @@ This feature is designed as a progressive enhancement to the existing GitLab Rev ## Model Accuracy -Organizations use many different processes for code review. Some focus on senior engineers reviewing junior engineer's code, others have hierarchical organizational structure based reviews. Suggested Reviewers is focused on contextual reviewers based on historical merge request activity by users. While we will continue evolving the underlying ML model to better serve various code review use cases and processes Suggested Reviewers does not replace the usage of other code review features like Code Owners and [Approval Rules](../approvals/rules.md). Reviewer selection is highly subjective therefore, we do not expect Suggested Reviewers to provide perfect suggestions everytime. +Organizations use many different processes for code review. Some focus on senior engineers reviewing junior engineer's code, others have hierarchical organizational structure based reviews. Suggested Reviewers is focused on contextual reviewers based on historical merge request activity by users. While we will continue evolving the underlying ML model to better serve various code review use cases and processes Suggested Reviewers does not replace the usage of other code review features like Code Owners and [Approval Rules](../approvals/rules.md). Reviewer selection is highly subjective therefore, we do not expect Suggested Reviewers to provide perfect suggestions every time. Through analysis of beta customer usage, we find that the Suggested Reviewers ML model provides suggestions that are adopted in 60% of cases. We will be introducing a feedback mechanism into the Suggested Reviewers feature in the future to allow users to flag bad reviewer suggestions to help improve the model. Additionally we will be offering an opt-in feature in the future which will allow the model to use your project's data for training the underlying model. @@ -39,6 +39,6 @@ Suggested Reviewers is off by default and requires a Project Owner or Admin to e Suggested Reviewers operates completely within the GitLab.com infrastructure providing the same level of [privacy](https://about.gitlab.com/privacy/) and [security](https://about.gitlab.com/security/) of any other feature of GitLab.com. -No new additional data is collected to enable this feature. GitLab is inferencing your merge request against a trained machine learning model. The content of your source code is not used as training data. Your data also never leaves GitLab.com, all training and inference is done within GitLab.com infrastructure. +No new additional data is collected to enable this feature. GitLab infers your merge request against a trained machine learning model. The content of your source code is not used as training data. Your data also never leaves GitLab.com, all training and inference is done within GitLab.com infrastructure. [Read more about the security of GitLab.com](https://about.gitlab.com/security/faq/) diff --git a/doc/user/project/merge_requests/reviews/img/suggestion_code_block_editor_v12_8.png b/doc/user/project/merge_requests/reviews/img/suggestion_code_block_editor_v12_8.png Binary files differdeleted file mode 100644 index 927b4f812a5..00000000000 --- a/doc/user/project/merge_requests/reviews/img/suggestion_code_block_editor_v12_8.png +++ /dev/null diff --git a/doc/user/project/merge_requests/reviews/index.md b/doc/user/project/merge_requests/reviews/index.md index 4c503211513..9a75c038dbc 100644 --- a/doc/user/project/merge_requests/reviews/index.md +++ b/doc/user/project/merge_requests/reviews/index.md @@ -71,6 +71,52 @@ if you [approve a merge request](../approvals/index.md#approve-a-merge-request) are shown in the reviewer list, a green check mark **{check-circle-filled}** displays next to your name. +### Download merge request changes as a diff + +To download the changes included in a merge request as a diff: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Merge requests**. +1. Select your merge request. +1. On the top right, select **Code > Plain diff**. + +If you know the URL of the merge request, you can also download the diff from +the command line by appending `.diff` to the URL. This example downloads the diff +for merge request `000000`: + +```plaintext +https://gitlab.com/gitlab-org/gitlab/-/merge_requests/000000.diff +``` + +To download and apply the diff in a one-line CLI command: + +```shell +curl "https://gitlab.com/gitlab-org/gitlab/-/merge_requests/000000.diff" | git apply +``` + +### Download merge request changes as a patch file + +To download the changes included in a merge request as a patch file: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Merge requests**. +1. Select your merge request. +1. On the top right, select **Code > Email patches**. + +If you know the URL of the merge request, you can also download the patch from +the command line by appending `.patch` to the URL. This example downloads the patch +file for merge request `000000`: + +```plaintext +https://gitlab.com/gitlab-org/gitlab/-/merge_requests/000000.patch +``` + +To download and apply the patch in a one-line CLI command using [`git am`](https://git-scm.com/docs/git-am): + +```shell +curl "https://gitlab.com/gitlab-org/gitlab/-/merge_requests/000000.patch" | git am +``` + ### Submit a review You can submit your completed review in multiple ways: diff --git a/doc/user/project/merge_requests/reviews/suggestions.md b/doc/user/project/merge_requests/reviews/suggestions.md index 832f78d18a1..668dece9fda 100644 --- a/doc/user/project/merge_requests/reviews/suggestions.md +++ b/doc/user/project/merge_requests/reviews/suggestions.md @@ -74,7 +74,13 @@ To add a suggestion that includes a [fenced code block](../../../markdown.md#code-spans-and-blocks), wrap your suggestion in four backticks instead of three: - +~~~markdown +````suggestion:-0+2 +```shell +git config --global receive.advertisepushoptions true +``` +```` +~~~  diff --git a/doc/user/project/merge_requests/status_checks.md b/doc/user/project/merge_requests/status_checks.md index d330ccdefb6..74c3b3e24b6 100644 --- a/doc/user/project/merge_requests/status_checks.md +++ b/doc/user/project/merge_requests/status_checks.md @@ -6,7 +6,7 @@ type: reference, concepts disqus_identifier: 'https://docs.gitlab.com/ee/user/project/merge_requests/status_checks.html' --- -# External Status Checks **(ULTIMATE)** +# External status checks **(ULTIMATE)** > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3869) in GitLab 14.0, disabled behind the `:ff_external_status_checks` feature flag. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/320783) in GitLab 14.1. diff --git a/doc/user/project/ml/experiment_tracking/img/candidate_v15_7.png b/doc/user/project/ml/experiment_tracking/img/candidate_v15_7.png Binary files differnew file mode 100644 index 00000000000..fb2e2e706d6 --- /dev/null +++ b/doc/user/project/ml/experiment_tracking/img/candidate_v15_7.png diff --git a/doc/user/project/ml/experiment_tracking/img/candidates.png b/doc/user/project/ml/experiment_tracking/img/candidates.png Binary files differdeleted file mode 100644 index df70a01a2bd..00000000000 --- a/doc/user/project/ml/experiment_tracking/img/candidates.png +++ /dev/null diff --git a/doc/user/project/ml/experiment_tracking/img/candidates_v15_7.png b/doc/user/project/ml/experiment_tracking/img/candidates_v15_7.png Binary files differnew file mode 100644 index 00000000000..58dfe94a108 --- /dev/null +++ b/doc/user/project/ml/experiment_tracking/img/candidates_v15_7.png diff --git a/doc/user/project/ml/experiment_tracking/img/experiments.png b/doc/user/project/ml/experiment_tracking/img/experiments.png Binary files differdeleted file mode 100644 index a6472406b90..00000000000 --- a/doc/user/project/ml/experiment_tracking/img/experiments.png +++ /dev/null diff --git a/doc/user/project/ml/experiment_tracking/img/experiments_v15_7.png b/doc/user/project/ml/experiment_tracking/img/experiments_v15_7.png Binary files differnew file mode 100644 index 00000000000..a7d4a3e559f --- /dev/null +++ b/doc/user/project/ml/experiment_tracking/img/experiments_v15_7.png diff --git a/doc/user/project/ml/experiment_tracking/index.md b/doc/user/project/ml/experiment_tracking/index.md index e274bd7f38e..a7096d633a0 100644 --- a/doc/user/project/ml/experiment_tracking/index.md +++ b/doc/user/project/ml/experiment_tracking/index.md @@ -16,9 +16,11 @@ engineering, and so on, to improve the performance of the model. Keeping track o artifacts so that the data scientist can later replicate the experiment is not trivial. Machine learning experiment tracking enables them to log parameters, metrics, and artifacts directly into GitLab, giving easy access later on. - + - + + + ## What is an experiment? @@ -53,18 +55,19 @@ integration. More information on how to use GitLab as a backend for MLFlow Clien ### Exploring model candidates To list the current active experiments, navigate to `https/-/ml/experiments`. To display all trials -that have been logged, along with their metrics and parameters, selecting an experiment. +that have been logged, along with their metrics and parameters, select an experiment. To display details for a candidate, +select **Details**. ### Logging artifacts Trial artifacts are saved as [generic packages](../../../packages/generic_packages/index.md), and follow all their conventions. After an artifact is logged for a candidate, all artifacts logged for the candidate are listed in the -package registry. The package name for a candidate is `ml_candidate_<candidate_id>`, with version `-`. +package registry. The package name for a candidate is `ml_candidate_<candidate_id>`, with version `-`. The link to the +artifacts can also be accessed from the **Experiment Candidates** list or **Candidate detail**. ### Limitations and future - Searching experiments, searching trials, visual comparison of trials, and creating, deleting and updating experiments and trials through GitLab UI is under development. -- No support for experiment and trial metadata that do not classify as parameters or metrics. ## Disabling or enabling the Feature @@ -73,4 +76,6 @@ On GitLab.com, this feature is currently on private testing. ## Feedback, roadmap and reports -For updates on the development, feedback and bug reports, refer to the [development epic](https://gitlab.com/groups/gitlab-org/-/epics/8560). +For updates on the development, refer to the [development epic](https://gitlab.com/groups/gitlab-org/-/epics/8560). + +For feedback, bug reports and feature requests, refer to the [feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/381660). diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md index 1d32091b294..197524f2fc5 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md @@ -35,7 +35,7 @@ for the most popular hosting services: - [123-reg](https://www.123-reg.co.uk/support/domains/domain-name-server-dns-management-guide/) - [Amazon](https://docs.aws.amazon.com/AmazonS3/latest/dev/website-hosting-custom-domain-walkthrough.html) - [Bluehost](https://www.bluehost.com/help/article/dns-management-add-edit-or-delete-dns-entries) -- [Cloudflare](https://support.cloudflare.com/hc/en-us/articles/201720164-Creating-a-Cloudflare-account-and-adding-a-website) +- [Cloudflare](https://developers.cloudflare.com/fundamentals/get-started/setup/) - [cPanel](https://documentation.cpanel.net/display/84Docs/Edit+DNS+Zone) - [DigitalOcean](https://docs.digitalocean.com/products/networking/dns/how-to/manage-records/) - [DreamHost](https://help.dreamhost.com/hc/en-us/articles/360035516812) diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/img/add_certificate_to_pages.png b/doc/user/project/pages/custom_domains_ssl_tls_certification/img/add_certificate_to_pages.png Binary files differdeleted file mode 100644 index d92a981dc60..00000000000 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/img/add_certificate_to_pages.png +++ /dev/null diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md index 6378d962ffe..dc23540bd1b 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md @@ -52,15 +52,14 @@ this document for an [overview on DNS records](dns_concepts.md). #### 1. Add a custom domain -Navigate to your project's **Setting > Pages** and select **+ New domain** -to add your custom domain to GitLab Pages. You can choose whether to: +To add your custom domain to GitLab Pages: -- Add an [SSL/TLS certificate](#adding-an-ssltls-certificate-to-pages). -- Leave it blank (it can be added later). - -Select **Create New Domain**. - - +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Pages**. +1. In the top right, select **New Domain**. +1. In **Domain**, enter your domain. +1. Optional. In **Certificate**, turn off the **Automatic certificate management using Let's Encrypt** toggle to add an [SSL/TLS certificate](#adding-an-ssltls-certificate-to-pages). You can also add the certificate and key later. +1. Select **Create New Domain**. #### 2. Get the verification code @@ -292,8 +291,6 @@ meet these requirements. - To add the certificate to a domain previously added, go to your project's **Settings > Pages**, locate your domain name, select **Details** and **Edit** to add the certificate. - - 1. Add the PEM certificate to its corresponding field. 1. If your certificate is missing its intermediate, copy and paste the root certificate (usually available from your CA website) diff --git a/doc/user/project/pages/getting_started/pages_ci_cd_template.md b/doc/user/project/pages/getting_started/pages_ci_cd_template.md index caf98e8a8a4..339ab239588 100644 --- a/doc/user/project/pages/getting_started/pages_ci_cd_template.md +++ b/doc/user/project/pages/getting_started/pages_ci_cd_template.md @@ -26,13 +26,12 @@ these steps, you may have to do additional configuration for the Pages site to g If everything is configured correctly, the site can take approximately 30 minutes to deploy. -You can watch the pipeline run by navigating to **CI/CD > Pipelines**. +To view the pipeline, go to **CI/CD > Pipelines**. When the pipeline is finished, go to **Settings > Pages** to find the link to your Pages website. -To view the HTML and other assets that were created for the site, -go to the **Pipelines** tab, view the job, and on the right side, -select **Download artifacts**. - For every change pushed to your repository, GitLab CI/CD runs a new pipeline that immediately publishes your changes to the Pages site. + +To view the HTML and other assets that were created for the site, +[download the job artifacts](../../../../ci/pipelines/job_artifacts.md#download-job-artifacts). diff --git a/doc/user/project/pages/getting_started/pages_forked_sample_project.md b/doc/user/project/pages/getting_started/pages_forked_sample_project.md index 69c60cab4b3..9841e52a089 100644 --- a/doc/user/project/pages/getting_started/pages_forked_sample_project.md +++ b/doc/user/project/pages/getting_started/pages_forked_sample_project.md @@ -29,32 +29,37 @@ When the pipeline is finished, go to **Settings > Pages** to find the link to yo For every change pushed to your repository, GitLab CI/CD runs a new pipeline that immediately publishes your changes to the Pages site. -To view the HTMl and other assets that were created for the site, -go to the **Pipelines** tab, view the job, and on the right side, -select **Download artifacts**. +## Remove the fork relationship -You can take some **optional** further steps: +If you want to contribute to the project you forked from, +you can keep the forked relationship. Otherwise: -- Remove the fork relationship. If you want to contribute to the project you forked from, - you can keep this relationship. Otherwise, go to your project's **Settings > General**, - expand **Advanced settings**, and scroll down to **Remove fork relationship**: +1. On the left sidebar, select **Settings > General**. +1. Expand **Advanced settings**. +1. Select **Remove fork relationship**. -  +## Change the URL -- Change the URL to match your namespace. If your Pages site is hosted on GitLab.com, - you can rename it to `<namespace>.gitlab.io`, where `<namespace>` is your GitLab namespace - (the one you chose when you forked the project). +You can change the URL to match your namespace. +If your Pages site is hosted on GitLab.com, +you can rename it to `<namespace>.gitlab.io`, where `<namespace>` is your GitLab namespace +(the one you chose when you forked the project). - - Go to your project's **Settings > General** and expand **Advanced**. Scroll down to - **Change path** and change the path to `<namespace>.gitlab.io`. +1. On the left sidebar, select **Settings > General**. +1. Expand **Advanced**. +1. In **Change path**, update the path to `<namespace>.gitlab.io`. - For example, if your project's URL is `gitlab.com/gitlab-tests/jekyll`, your namespace is - `gitlab-tests`. + For example, if your project's URL is `gitlab.com/gitlab-tests/jekyll`, your namespace is + `gitlab-tests`. - If you set the repository path to `gitlab-tests.gitlab.io`, - the resulting URL for your Pages website is `https://gitlab-tests.gitlab.io`. + If you set the repository path to `gitlab-tests.gitlab.io`, + the resulting URL for your Pages website is `https://gitlab-tests.gitlab.io`. -  +  - - Now go to your SSG's configuration file and change the [base URL](../getting_started_part_one.md#urls-and-base-urls) - from `"project-name"` to `""`. The project name setting varies by SSG and may not be in the configuration file. +1. Open your SSG configuration file and change the [base URL](../getting_started_part_one.md#urls-and-base-urls) + from `"project-name"` to `""`. The project name setting varies by SSG and may not be in the configuration file. + +## Related topics + +- [Download the job artifacts](../../../../ci/pipelines/job_artifacts.md#download-job-artifacts) diff --git a/doc/user/project/pages/getting_started/pages_from_scratch.md b/doc/user/project/pages/getting_started/pages_from_scratch.md index c0a1e8f16e0..ebadf39a984 100644 --- a/doc/user/project/pages/getting_started/pages_from_scratch.md +++ b/doc/user/project/pages/getting_started/pages_from_scratch.md @@ -420,9 +420,8 @@ Now GitLab CI/CD not only builds the website, but also: - **Caches** dependencies installed with Bundler. - **Continuously deploys** every push to the `main` branch. -To view the HTMl and other assets that were created for the site, -go to the **Pipelines** tab, view the job, and on the right side, -select **Download artifacts**. +To view the HTML and other assets that were created for the site, +[download the job artifacts](../../../../ci/pipelines/job_artifacts.md#download-job-artifacts). ## Related topics diff --git a/doc/user/project/pages/getting_started/pages_new_project_template.md b/doc/user/project/pages/getting_started/pages_new_project_template.md index e4890954d13..a0f9753a40c 100644 --- a/doc/user/project/pages/getting_started/pages_new_project_template.md +++ b/doc/user/project/pages/getting_started/pages_new_project_template.md @@ -29,6 +29,5 @@ your Pages website. For every change pushed to your repository, GitLab CI/CD runs a new pipeline that immediately publishes your changes to the Pages site. -To view the HTMl and other assets that were created for the site, -go to the **Pipelines** tab, view the job, and on the right side, -select **Download artifacts**. +To view the HTML and other assets that were created for the site, +[download the job artifacts](../../../../ci/pipelines/job_artifacts.md#download-job-artifacts). diff --git a/doc/user/project/pages/getting_started/pages_ui.md b/doc/user/project/pages/getting_started/pages_ui.md index ba97fcb8749..a6069b473e6 100644 --- a/doc/user/project/pages/getting_started/pages_ui.md +++ b/doc/user/project/pages/getting_started/pages_ui.md @@ -4,59 +4,69 @@ group: Incubation info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Tutorial: Use the GitLab UI to deploy your static site **(FREE)** +# Create a Pages deployment for your static site **(FREE)** -This tutorial assumes you have a project that either: +If you already have a GitLab project that contains your static site or framework, +you can generate a GitLab Pages website from it. -- Generates static sites or a client-rendered single-page application (SPA), - such as [Eleventy](https://www.11ty.dev), [Astro](https://astro.build), or [Jekyll](https://jekyllrb.com). -- Contains a framework configured for static output, such as [Next.js](https://nextjs.org), - [Nuxt.js](https://nuxtjs.org), or [SvelteKit](https://kit.svelte.dev). +When you provide basic information in the UI, a `.gitlab-ci.yml` file is created +and a merge request opened. When you commit the merge request, +a pipeline deploys your Pages website. -## Update your app to output files to the `public` folder +## Prerequisites -GitLab Pages requires all files intended to be part of the published website to -be in a root-level folder called `public`. If you create this folder during the build -pipeline, committing it to Git is not required. +- Your app must [output files to the `public` folder](../public_folder.md). If you create + this folder during the build pipeline, you do not need to commit it to Git. -For detailed instructions, read [Configure the public files folder](../public_folder.md). + WARNING: + This step is important. Ensure your files are in a root-level `public` folder. -## Set up the `.gitlab-ci.yml` file +- You must have a project that either: + - Generates static sites or a client-rendered single-page application (SPA), + like [Eleventy](https://www.11ty.dev), [Astro](https://astro.build), or [Jekyll](https://jekyllrb.com). + - Contains a framework configured for static output, such as [Next.js](https://nextjs.org), + [Nuxt.js](https://nuxtjs.org), or [SvelteKit](https://kit.svelte.dev). +- GitLab Pages must be enabled for the project. (To enable, go to **Settings > General**, + expand **Visibility, project features, permissions**, and turn on the **Pages** toggle.) -GitLab helps you write the `.gitlab-ci.yml` needed to create your first GitLab Pages -deployment pipeline. Rather than building the file from scratch, it asks you to -provide the build commands, and creates the necessary boilerplate for you. +## Create the Pages deployment -To build your YAML file from the GitLab UI: +To complete the setup and generate a GitLab Pages deployment: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Settings > Pages** to display the friendly - interface **Get Started With Pages**. -1. If your framework's build process does not need one of the provided build - commands, you can either: +1. On the left sidebar, select **Settings > Pages**. A **Get Started with Pages** form appears. + If this form is not available, see [Troubleshooting](#if-the-get-started-with-pages-form-is-not-available). +1. For **Step 1**, enter an image name and verify that your files are in a `public` folder. +1. Select **Next**. +1. For **Step 2**, enter your installation steps. If your framework's build process does not + need one of the provided build commands, you can either: - Skip the step by selecting **Next**. - Enter `:` (the bash "do nothing" command) if you still want to incorporate that step's boilerplate into your `.gitlab-ci.yml` file. -1. Optional. Edit and adjust the generated `.gitlab-ci.yml` file as needed. -1. Commit your `.gitlab-ci.yml` to your repository. This commit triggers your first +1. Select **Next**. +1. For **Step 3**, enter scripts that indicate how to build your application. +1. Select **Next**. +1. Optional. Edit the generated `.gitlab-ci.yml` file as needed. +1. For **Step 4**, add a commit message and select **Commit**. This commit triggers your first GitLab Pages deployment. -To view the HTMl and other assets that were created for the site, -go to **CI/CD > Pipelines**, view the job, and on the right side, -select **Download artifacts**. +To view the running pipeline, go to **CI/CD > Pipelines**. + +To view the artifacts that were created during the deployment, view the job, +and on the right side, select **Download artifacts**. ## Troubleshooting -### If you can't see the "Get Started with Pages" interface +### If the `Get Started with Pages` form is not available -GitLab doesn't show this interface if you have either: +When you go to **Settings > Pages**, the form is not available if you: - Deployed a GitLab Pages site before. -- Committed a `.gitlab-ci.yml` through this interface at least once. +- Committed a `.gitlab-ci.yml` through the forms at least one time. -To fix this problem: +To fix this issue: -- If you see the message **Waiting for the Pages Pipeline to complete**, select - **Start over** to start the wizard again. +- If the message **Waiting for the Pages Pipeline to complete** appears, select + **Start over** to start the form again. - If your project has previously deployed GitLab Pages successfully, - [manually update](pages_from_scratch.md) your `.gitlab-ci.yml`. + [manually update](pages_from_scratch.md) your `.gitlab-ci.yml` file. diff --git a/doc/user/project/pages/getting_started_part_one.md b/doc/user/project/pages/getting_started_part_one.md index 588d94729e2..a0c8073d6eb 100644 --- a/doc/user/project/pages/getting_started_part_one.md +++ b/doc/user/project/pages/getting_started_part_one.md @@ -87,7 +87,7 @@ Every Static Site Generator (SSG) default configuration expects to find your website under a (sub)domain (`example.com`), not in a subdirectory of that domain (`example.com/subdir`). Therefore, whenever you publish a project website (`namespace.gitlab.io/project-name`), -you must look for this configuration (base URL) on your SSG's +you must look for this configuration (base URL) on your static site generator's documentation and set it up to reflect this pattern. For example, for a Jekyll site, the `baseurl` is defined in the Jekyll diff --git a/doc/user/project/pages/img/remove_fork_relationship_v13_1.png b/doc/user/project/pages/img/remove_fork_relationship_v13_1.png Binary files differdeleted file mode 100644 index 84aa2e571c7..00000000000 --- a/doc/user/project/pages/img/remove_fork_relationship_v13_1.png +++ /dev/null diff --git a/doc/user/project/pages/public_folder.md b/doc/user/project/pages/public_folder.md index a19e296b954..8c9f1cbec86 100644 --- a/doc/user/project/pages/public_folder.md +++ b/doc/user/project/pages/public_folder.md @@ -2,40 +2,39 @@ description: 'Learn how to configure the build output folder for the most common static site generators' stage: Create -group: Incubation +group: Editor info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Configure the public files folder **(FREE)** -GitLab Pages requires all files you intend to be available in the published website to -be in a root-level folder called `public`. This page describe how -to set this up for some common static site generators. +All the files that should be accessible by the browser must be in a root-level folder called `public`. -## Guide by framework +Follow these instructions to configure the `public` folder +for the following frameworks. -### Eleventy +## Eleventy -For Eleventy, you should either: +For Eleventy, you should do one of the following: -1. Add the `--output=public` flag in Eleventy's build commands, for example: +- Add the `--output=public` flag in Eleventy's build commands, for example: - `npx @11ty/eleventy --input=path/to/sourcefiles --output=public` + `npx @11ty/eleventy --input=path/to/sourcefiles --output=public` -1. Add the following to your `.eleventy.js` file: +- Add the following to your `.eleventy.js` file: - ```javascript - // .eleventy.js - module.exports = function(eleventyConfig) { - return { - dir: { - output: "public" - } - } - }; - ``` + ```javascript + // .eleventy.js + module.exports = function(eleventyConfig) { + return { + dir: { + output: "public" + } + } + }; + ``` -### Astro +## Astro By default, Astro uses the `public` folder to store static assets. For GitLab Pages, rename that folder to a collision-free alternative first: @@ -65,11 +64,11 @@ rename that folder to a collision-free alternative first: }); ``` -### SvelteKit +## SvelteKit NOTE: GitLab Pages supports only static sites. For SvelteKit, -we recommend using [`adapter-static`](https://kit.svelte.dev/docs/adapters#supported-environments-static-sites). +you can use [`adapter-static`](https://kit.svelte.dev/docs/adapters#supported-environments-static-sites). When using `adapter-static`, add the following to your `svelte.config.js`: @@ -86,11 +85,11 @@ export default { }; ``` -### Next.js +## Next.js NOTE: -GitLab Pages supports only static sites. For Next.js, we -recommend using Next's [Static HTML export functionality](https://nextjs.org/docs/advanced-features/static-html-export) +GitLab Pages supports only static sites. For Next.js, you can use +Next's [Static HTML export functionality](https://nextjs.org/docs/advanced-features/static-html-export). Use the `-o public` flag after `next export` as the build command, for example: @@ -118,7 +117,7 @@ GitLab Pages supports only static sites. 1. Configure your Nuxt.js application for [Static Site Generation](https://nuxtjs.org/docs/features/deployment-targets/#static-hosting). -### Vite +## Vite Update your `vite.config.js` to include the following: @@ -131,7 +130,7 @@ export default { } ``` -### Webpack +## Webpack Update your `webpack.config.js` to include the following: @@ -147,9 +146,9 @@ module.exports = { ## Should you commit the `public` folder? Not necessarily. However, when the GitLab Pages deploy pipeline runs, it looks -for an [artifact](../../../ci/pipelines/job_artifacts.md) of that name. So +for an [artifact](../../../ci/pipelines/job_artifacts.md) of that name. If you set up a job that creates the `public` folder before deploy, such as by running `npm run build`, committing the folder isn't required. If you prefer to build your site locally, you can commit the `public` folder and -omit the build step during the job, instead. +omit the build step during the job instead. diff --git a/doc/user/project/pages/redirects.md b/doc/user/project/pages/redirects.md index 96de457c7f7..cf0c0dbff82 100644 --- a/doc/user/project/pages/redirects.md +++ b/doc/user/project/pages/redirects.md @@ -108,9 +108,8 @@ and an [HTTP status code](#http-status-codes): ## Rewrites -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/458) in GitLab 14.3. -> - Enabled on GitLab.com. -> - Disabled by default in self-managed GitLab behind the [`FF_ENABLE_PLACEHOLDERS` feature flag](#feature-flag-for-rewrites). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/458) in GitLab 14.3 [with a flag](../../../administration/feature_flags.md) named `FF_ENABLE_PLACEHOLDERS`. Disabled by default. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/619) in GitLab 15.2. Provide a status code of `200` to serve the content of the `to` path when the request matches the `from`: @@ -267,28 +266,3 @@ However, there are some minor differences: - Netlify redirects to `/new/:placeholder` (with a literal `:placeholder`). - GitLab redirects to `/new/`. - -## Feature flag for rewrites - -FLAG: -Rewrites in GitLab Pages is under development, and is deployed behind a feature flag -that is **disabled by default**. - -To enable rewrites, for [Omnibus installations](../../../administration/pages/index.md), define the -`FF_ENABLE_PLACEHOLDERS` environment variable in the -[global settings](../../../administration/pages/index.md#global-settings). -Add the following line to `/etc/gitlab/gitlab.rb` and -[reconfigure the instance](../../../administration/restart_gitlab.md#omnibus-gitlab-reconfigure): - -```ruby -gitlab_pages['env']['FF_ENABLE_PLACEHOLDERS'] = 'true' -``` - -For [source installations](../../../administration/pages/source.md), define the -`FF_ENABLE_PLACEHOLDERS` environment variable, then -[restart GitLab](../../../administration/restart_gitlab.md#installations-from-source): - -```shell -export FF_ENABLE_PLACEHOLDERS="true" -/path/to/pages/bin/gitlab-pages -config gitlab-pages.conf -``` diff --git a/doc/user/project/push_options.md b/doc/user/project/push_options.md index b31ef858d59..9e5413b020e 100644 --- a/doc/user/project/push_options.md +++ b/doc/user/project/push_options.md @@ -61,10 +61,10 @@ time as pushing changes: | `merge_request.target=<branch_name>` | Set the target of the merge request to a particular branch or upstream project, such as: `git push -o merge_request.target=project_path/branch` | [11.10](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26752) | | `merge_request.merge_when_pipeline_succeeds` | Set the merge request to [merge when its pipeline succeeds](merge_requests/merge_when_pipeline_succeeds.md). | [11.10](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26752) | | `merge_request.remove_source_branch` | Set the merge request to remove the source branch when it's merged. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64320) | -| `merge_request.title="<title>"` | Set the title of the merge request. Ex: `git push -o merge_request.title="The title I want"`. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64320) | -| `merge_request.description="<description>"` | Set the description of the merge request. Ex: `git push -o merge_request.description="The description I want"`. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64320) | -| `merge_request.draft` | Mark the merge request as a draft. Ex: `git push -o merge_request.draft`. | [15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/296673) | -| `merge_request.milestone="<milestone>"` | Set the milestone of the merge request. Ex: `git push -o merge_request.milestone="3.0"`. | [14.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63960) | +| `merge_request.title="<title>"` | Set the title of the merge request. For example: `git push -o merge_request.title="The title I want"`. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64320) | +| `merge_request.description="<description>"` | Set the description of the merge request. For example: `git push -o merge_request.description="The description I want"`. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64320) | +| `merge_request.draft` | Mark the merge request as a draft. For example: `git push -o merge_request.draft`. | [15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/296673) | +| `merge_request.milestone="<milestone>"` | Set the milestone of the merge request. For example: `git push -o merge_request.milestone="3.0"`. | [14.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63960) | | `merge_request.label="<label>"` | Add labels to the merge request. If the label does not exist, it is created. For example, for two labels: `git push -o merge_request.label="label1" -o merge_request.label="label2"`. | [12.3](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/31831) | | `merge_request.unlabel="<label>"` | Remove labels from the merge request. For example, for two labels: `git push -o merge_request.unlabel="label1" -o merge_request.unlabel="label2"`. | [12.3](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/31831) | | `merge_request.assign="<user>"` | Assign users to the merge request. Accepts username or user ID. For example, for two users: `git push -o merge_request.assign="user1" -o merge_request.assign="user2"`. | [13.10](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25904), support for usernames added in [15.5](https://gitlab.com/gitlab-org/gitlab/-/issues/344276) | diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index 7b7619cfeb5..a73a5329688 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -72,13 +72,13 @@ threads. Some quick actions might not be available to all subscription tiers. | `/done` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Mark to do as done. | | `/draft` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Set the [draft status](merge_requests/drafts.md). Use for toggling the draft status ([deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/92654) in GitLab 15.4.) | | `/due <date>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Set due date. Examples of valid `<date>` include `in 2 days`, `this Friday` and `December 31st`. | -| `/duplicate <#issue>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Close this issue and mark as a duplicate of another issue. Also, mark both as related. | +| `/duplicate <#issue>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Close this issue. Mark as a duplicate of, and related to, issue `<#issue>`. | | `/epic <epic>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Add to epic `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. | | `/estimate <time>` or `/estimate_time <time>` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Set time estimate. For example, `/estimate 1mo 2w 3d 4h 5m`. Learn more about [time tracking](time_tracking.md). Alias `/estimate_time` [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16501) in GitLab 15.6. | | `/health_status <value>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Set [health status](issues/managing_issues.md#health-status). Valid options for `<value>` are `on_track`, `needs_attention`, and `at_risk` ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213814) in GitLab 14.7). | | `/invite_email email1 email2` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Add up to six email participants. This action is behind feature flag `issue_email_participants` and is not yet supported in issue templates. | | `/iteration *iteration:"iteration name"` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Set iteration. For example, to set the `Late in July` iteration: `/iteration *iteration:"Late in July"` ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196795) in GitLab 13.1). | -| `/label ~label1 ~label2` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Add one or more labels. Label names can also start without a tilde (`~`), but mixed syntax is not supported. | +| `/label ~label1 ~label2` or `/labels ~label1 ~label2` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Add one or more labels. Label names can also start without a tilde (`~`), but mixed syntax is not supported. | | `/lock` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Lock the discussions. | | `/link` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Add a link and description to [linked resources](../../operations/incident_management/linked_resources.md) in an incident ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/374964) in GitLab 15.5). | | `/merge` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Merge changes. Depending on the project setting, this may be [when the pipeline succeeds](merge_requests/merge_when_pipeline_succeeds.md), or adding to a [Merge Train](../../ci/pipelines/merge_trains.md). | diff --git a/doc/user/project/releases/index.md b/doc/user/project/releases/index.md index 75a25678125..6d5378bddb7 100644 --- a/doc/user/project/releases/index.md +++ b/doc/user/project/releases/index.md @@ -184,20 +184,20 @@ is not available. ## Edit a release -Only users with at least the Developer role can edit releases. -Read more about [Release permissions](#release-permissions). +To edit the details of a release after it's created, you can use the +[Update a release API](../../../api/releases/index.md#update-a-release) or the UI. -To edit the details of a release: +Prerequisites: + +- You must have at least the Developer role. + +In the UI: 1. On the left sidebar, select **Deployments > Releases**. 1. In the top-right corner of the release you want to modify, select **Edit this release** (the pencil icon). 1. On the **Edit Release** page, change the release's details. 1. Select **Save changes**. -You can edit the release title, notes, associated milestones, and asset links. -To change the release date use the -[Releases API](../../../api/releases/index.md#update-a-release). - ## Delete a release > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213862) in GitLab 15.2 @@ -209,11 +209,15 @@ Prerequisites: - You must have at least the Developer role. Read more about [Release permissions](#release-permissions). -To delete a release in the UI: +To delete a release, use either the +[Delete a release API](../../../api/releases/index.md#delete-a-release) or the UI. + +In the UI: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Deployments > Releases**. -1. In the top-right corner of the release you want to delete, select **Edit this release** (**{pencil}**). +1. In the top-right corner of the release you want to delete, select **Edit this release** + (**{pencil}**). 1. On the **Edit Release** page, select **Delete**. 1. Select **Delete release**. @@ -449,11 +453,11 @@ In the API: ## Release permissions -> [The permission model for create, update and delete actions was fixed](https://gitlab.com/gitlab-org/gitlab/-/issues/327505) in GitLab 14.1. +> Fixes to the permission model for create, update and delete actions [were introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327505) in GitLab 14.1. ### View a release and download assets -> [Changes were made to the Guest role access](https://gitlab.com/gitlab-org/gitlab/-/issues/335209) in GitLab 14.5. +> Changes to the Guest role [were introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335209) in GitLab 14.5. - Users with at least the Reporter role have read and download access to the project releases. diff --git a/doc/user/project/repository/branches/default.md b/doc/user/project/repository/branches/default.md index 6801899160d..87caeee73e3 100644 --- a/doc/user/project/repository/branches/default.md +++ b/doc/user/project/repository/branches/default.md @@ -36,11 +36,15 @@ the [Git commands you need](#update-the-default-branch-name-in-your-repository) ## Change the default branch name for a project -To update the default branch name for an individual [project](../../index.md): +Prerequisites: -1. Sign in to GitLab with at least the Maintainer role. +- You have the Owner or Maintainer role in the project. + +To update the default branch for an individual [project](../../index.md): + +1. On the top bar, select **Main menu > Projects** and find your project. 1. In the left navigation menu, go to **Settings > Repository**. -1. Expand **Default branch**, and select a new default branch. +1. Expand **Default branch**. For **Initial default branch name**, select a new default branch. 1. Optional. Select the **Auto-close referenced issues on default branch** checkbox to close issues when a merge request [uses a closing pattern](../../issues/managing_issues.md#closing-issues-automatically). @@ -66,8 +70,8 @@ groups and subgroups can override this instance-wide setting for their projects. 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > Repository**. -1. Expand **Default initial branch name**. -1. Change the default initial branch to a custom name of your choice. +1. Expand **Default branch**. +1. For **Initial default branch name**, select a new default branch. 1. Select **Save changes**. Projects created on this instance after you change the setting use the @@ -78,11 +82,12 @@ overrides it. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221014) in GitLab 13.6. -Users with at least the Owner role of groups and subgroups can configure the default branch name for a group: +Users with the Owner role of groups and subgroups can configure the default branch name for a group: -1. Go to the group **Settings > Repository**. +1. On the top bar, select **Main menu > Group** and find your group. +1. On the left sidebar, select **Settings > Repository**. 1. Expand **Default branch**. -1. Change the default initial branch to a custom name of your choice. +1. For **Initial default branch name**, select a new default branch. 1. Select **Save changes**. Projects created in this group after you change the setting use the custom branch name, diff --git a/doc/user/project/repository/branches/index.md b/doc/user/project/repository/branches/index.md index 6cc7394e7b3..a86e32b4721 100644 --- a/doc/user/project/repository/branches/index.md +++ b/doc/user/project/repository/branches/index.md @@ -101,10 +101,13 @@ This feature allows you to search and select branches quickly. Search results ap - Branches with names that matched search terms exactly. - Other branches with names that include search terms, sorted alphabetically. -Sometimes when you have hundreds of branches you may want a more flexible matching pattern. In such cases you can use the following: +Sometimes when you have hundreds of branches you may want a more flexible matching pattern. In such cases you can use the following operators: -- `^feature` matches only branch names that begin with 'feature'. -- `feature$` matches only branch names that end with 'feature'. +- `^` matches beginning of branch name, for example `^feat` would match `feat/user-authentication` +- `$` matches end of branch name, for example `widget$` would match `feat/search-box-widget` +- `*` wildcard matcher, for example `branch*cache*` would match `fix/branch-search-cache-expiration` + +These operators can be mixed, for example `^chore/*migration$` would match `chore/user-data-migration` ## Swap revisions @@ -116,14 +119,26 @@ The Swap revisions feature allows you to swap the Source and Target revisions. W  -<!-- ## Troubleshooting +## Troubleshooting + +### Error: ambiguous `HEAD` branch exists + +In versions of Git earlier than 2.16.0, you could create a branch named `HEAD`. +This branch named `HEAD` collides with the internal reference (also named `HEAD`) +Git uses to describe the active (checked out) branch. This naming collision can +prevent you from updating the default branch of your repository: + +```plaintext +Error: Could not set the default branch. Do you have a branch named 'HEAD' in your repository? +``` + +To fix this problem: -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Repository > Branches**. +1. Search for a branch named `HEAD`. +1. Make sure the branch has no uncommitted changes. +1. Select **Delete branch**, then **Yes, delete branch**. -Each scenario can be a third-level heading, for example `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +Git versions [2.16.0 and later](https://github.com/git/git/commit/a625b092cc59940521789fe8a3ff69c8d6b14eb2), +prevent you from creating a branch with this name. diff --git a/doc/user/project/repository/gpg_signed_commits/index.md b/doc/user/project/repository/gpg_signed_commits/index.md index a1f57f51f26..6b67ffd0e59 100644 --- a/doc/user/project/repository/gpg_signed_commits/index.md +++ b/doc/user/project/repository/gpg_signed_commits/index.md @@ -4,7 +4,7 @@ group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Signing commits with GPG **(FREE)** +# Sign commits with GPG **(FREE)** You can sign the commits you make in a GitLab repository with a GPG ([GNU Privacy Guard](https://gnupg.org/)) key. When you add a cryptographic @@ -160,14 +160,6 @@ to use this key: git config --global user.signingkey <KEY ID> ``` -1. Optional. If Git uses `gpg` and you get errors like `secret key not available` - or `gpg: signing failed: secret key not available`, run this command to - use `gpg2` instead: - - ```shell - git config --global gpg.program gpg2 - ``` - ### Sign your Git commits After you [add your public key to your account](#add-a-gpg-key-to-your-account), @@ -246,6 +238,7 @@ If you must unverify both future and past commits, ## Related topics - [Sign commits and tags with X.509 certificates](../x509_signed_commits/index.md) +- [Sign commits with SSH keys](../ssh_signed_commits/index.md) - [Commits API](../../../../api/commits.md) - GPG resources: - [Git Tools - Signing Your Work](https://git-scm.com/book/en/v2/Git-Tools-Signing-Your-Work) @@ -269,3 +262,27 @@ or a GPG key. The verification process for both methods can fail for multiple re | `UNVERIFIED_KEY` | The key associated with the GPG signature has no verified email address associated with the committer. | Add and verify the email to your GitLab profile, [update the GPG key to include the email address](https://security.stackexchange.com/a/261468), or amend the commit to use a different committer email address. | | `UNKNOWN_KEY` | The GPG key associated with the GPG signature for this commit is unknown to GitLab. | [Add the GPG key](#add-a-gpg-key-to-your-account) to your GitLab profile. | | `MULTIPLE_SIGNATURES` | Multiple GPG or X.509 signatures have been found for the commit. | Amend the commit to use only one GPG or X.509 signature. | + +### Secret key not available + +If you receive the errors `secret key not available` +or `gpg: signing failed: secret key not available`, try using `gpg2` instead of `gpg`: + +```shell +git config --global gpg.program gpg2 +``` + +If your GPG key is password protected and the password entry prompt does not appear, +add `export GPG_TTY=$(tty)` to your shell's `rc` file (commonly `~/.bashrc` or `~/.zshrc`) + +### GPG failed to sign the data + +If your GPG key is password protected and you receive the error: + +```shell +error: gpg failed to sign the data +fatal: failed to write commit object +``` + +If the password entry prompt does not appear, add `export GPG_TTY=$(tty)` to your shell's `rc` file +(commonly `~/.bashrc` or `~/.zshrc`) and restart your terminal. diff --git a/doc/user/project/repository/ssh_signed_commits/index.md b/doc/user/project/repository/ssh_signed_commits/index.md new file mode 100644 index 00000000000..06affa54a51 --- /dev/null +++ b/doc/user/project/repository/ssh_signed_commits/index.md @@ -0,0 +1,174 @@ +--- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Sign commits with SSH keys **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343879) in GitLab 15.7 [with a flag](../../../../administration/feature_flags.md) named `ssh_commit_signatures`. Enabled by default. + +FLAG: +On self-managed GitLab, by default this feature is available. To hide the feature, +ask an administrator to [disable the feature flag](../../../../administration/feature_flags.md) named `ssh_commit_signatures`. +On GitLab.com, this feature is available. + +Use SSH keys to sign Git commits in the same manner as +[GPG signed commits](../gpg_signed_commits/index.md). When you sign commits +with SSH keys, GitLab uses the SSH public keys associated with your +GitLab account to cryptographically verify the commit signature. +If successful, GitLab displays a **Verified** label on the commit. + +You may use the same SSH keys for `git+ssh` authentication to GitLab +and signing commit signatures as long as their usage type is **Authentication & Signing**. +It can be verified on the page for [adding an SSH key to your GitLab account](../../../ssh.md#add-an-ssh-key-to-your-gitlab-account). + +To learn more about managing the SSH keys associated with your GitLab account, read +[use SSH keys to communicate with GitLab](../../../ssh.md). + +## Configure Git to sign commits with your SSH key + +After you [create an SSH key](../../../ssh.md#generate-an-ssh-key-pair) and +[add it to your GitLab account](../../../ssh.md#add-an-ssh-key-to-your-gitlab-account) +or [generate it using a password manager](../../../ssh.md#generate-an-ssh-key-pair-with-a-password-manager), +configure Git to begin using the key. + +Prerequisites: + +- Git 2.34.0 or newer. +- OpenSSH 8.0 or newer. + + NOTE: + OpenSSH 8.7 has broken signing functionality. If you are on OpenSSH 8.7, upgrade to OpenSSH 8.8. + +- A SSH key with the usage type of either **Authentication & Signing** or **Signing**. + The SSH key must be one of these types: + - [ED25519](../../../ssh.md#ed25519-ssh-keys) (recommended) + - [RSA](../../../ssh.md#rsa-ssh-keys) + +To configure Git to use your key: + +1. Configure Git to use SSH for commit signing: + + ```shell + git config --global gpg.format ssh + ``` + +1. Specify which SSH key should be used as the signing key, changing the filename + (here, `~/.ssh/examplekey`) to the location of your key. The filename may + differ, depending on how you generated your key: + + ```shell + git config --global user.signingkey ~/.ssh/examplekey + ``` + +## Sign commits with your SSH key + +Prerequisites: + +- You've [created an SSH key](../../../ssh.md#generate-an-ssh-key-pair). +- You've [added the key](../../../ssh.md#add-an-ssh-key-to-your-gitlab-account) to your GitLab account. +- You've [configured Git to sign commits](#configure-git-to-sign-commits-with-your-ssh-key) with your SSH key. + +To sign a commit: + +1. Use the `-S` flag when signing your commits: + + ```shell + git commit -S -m "My commit msg" + ``` + +1. Optional. If you don't want to type the `-S` flag every time you commit, tell + Git to sign your commits automatically: + + ```shell + git config --global commit.gpgsign true + ``` + +1. If your SSH key is protected, Git prompts you to enter your passphrase. +1. Push to GitLab. +1. Check that your commits [are verified](#verify-commits). + Signature verification uses the `allowed_signers` file to associate emails and SSH keys. + For help configuring this file, read [Verify commits locally](#verify-commits-locally). + +## Verify commits + +You can review commits for a merge request, or for an entire project, to confirm +they are signed: + +1. To review commits for a project: + 1. On the top bar, select **Main menu > Projects** and find your project. + 1. On the left sidebar, select **Repository > Commits**. +1. To review commits for a merge request: + 1. On the top bar, select **Main menu > Projects** and find your project. + 1. On the left sidebar, select **Merge requests**, then select your merge request. + 1. Select **Commits**. +1. Identify the commit you want to review. Signed commits show either a **Verified** + or **Unverified** badge, depending on the verification status of the signature. + Unsigned commits do not display a badge. +1. To display the signature details for a commit, select **Verified**. GitLab shows + the SSH key's fingerprint. + +## Verify commits locally + +To verify commits locally, create an +[allowed signers file](https://man7.org/linux/man-pages/man1/ssh-keygen.1.html#ALLOWED_SIGNERS) +for Git to associate SSH public keys with users: + +1. Create an allowed signers file: + + ```shell + touch allowed_signers + ``` + +1. Configure the `allowed_signers` file in Git: + + ```shell + git config gpg.ssh.allowedSignersFile "$(pwd)/allowed_signers" + ``` + +1. Add your entry to the allowed signers file. Use this command to add your + email address and public SSH key to the `allowed_signers` file. Replace `<MY_KEY>` + with the name of your key, and `~/.ssh/allowed_signers` + with the location of your project's `allowed_signers` file: + + ```shell + # Modify this line to meet your needs. + # Declaring the `git` namespace helps prevent cross-protocol attacks. + echo "$(git config --get user.email) namespaces=\"git\" $(cat ~/.ssh/<MY_KEY>.pub)" >> ~/.ssh/allowed_signers + ``` + + The resulting entry in the `allowed_signers` file contains your email address, key type, + and key contents, like this: + + ```plaintext + example@gitlab.com namespaces="git" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAmaTS47vRmsKyLyK1jlIFJn/i8wdGQ3J49LYyIYJ2hv + ``` + +1. Repeat the previous step for each user who you want to verify signatures for. + Consider checking this file in to your Git repository if you want to locally + verify signatures for many different contributors. + +1. Use `git log --show-signature` to view the signature status for the commits: + + ```shell + $ git log --show-signature + + commit e2406b6cd8ebe146835ceab67ff4a5a116e09154 (HEAD -> main, origin/main, origin/HEAD) + Good "git" signature for johndoe@example.com with ED25519 key SHA256:Ar44iySGgxic+U6Dph4Z9Rp+KDaix5SFGFawovZLAcc + Author: John Doe <johndoe@example.com> + Date: Tue Nov 29 06:54:15 2022 -0600 + + SSH signed commit + ``` + +## Revoke an SSH key for signing commits + +You can't revoke an SSH key used for signing commits. To learn more, read +[Add revocation for SSH keys](https://gitlab.com/gitlab-org/gitlab/-/issues/382984). + +## Related topics + +- [Sign commits and tags with X.509 certificates](../x509_signed_commits/index.md) +- [Sign commits with GPG](../gpg_signed_commits/index.md) +- [Commits API](../../../../api/commits.md) diff --git a/doc/user/project/repository/web_editor.md b/doc/user/project/repository/web_editor.md index 773662edb17..cc89ca0fb1a 100644 --- a/doc/user/project/repository/web_editor.md +++ b/doc/user/project/repository/web_editor.md @@ -73,7 +73,7 @@ To close the preview panel, do one of the following: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/56159) in GitLab 13.11 for self-managed instances. Web Editor enables you to highlight a single line by adding specially formatted -hash information to the URL's file path segment. For example, the file path segment +hash information to the file path segment of the URL. For example, the file path segment `MY_FILE.js#L3` instructs the Web Editor to highlight line 3. The Web Editor also enables you to highlight multiple lines using a similar pattern. In diff --git a/doc/user/project/repository/x509_signed_commits/index.md b/doc/user/project/repository/x509_signed_commits/index.md index e16f5e4defe..42f7be30822 100644 --- a/doc/user/project/repository/x509_signed_commits/index.md +++ b/doc/user/project/repository/x509_signed_commits/index.md @@ -160,6 +160,8 @@ can start signing your tags: ## Related topics - [Rake task for X.509 signatures](../../../../raketasks/x509_signatures.md) +- [Sign commits with GPG](../gpg_signed_commits/index.md) +- [Sign commits with SSH keys](../ssh_signed_commits/index.md) ## Troubleshooting diff --git a/doc/user/project/service_desk.md b/doc/user/project/service_desk.md index 199f25f1122..0b52440d1e6 100644 --- a/doc/user/project/service_desk.md +++ b/doc/user/project/service_desk.md @@ -202,39 +202,52 @@ worker and it would not recognize `incoming_email` emails. To configure a custom mailbox for Service Desk with IMAP, add the following snippets to your configuration file in full: -- Example for installations from source: - - ```yaml - service_desk_email: - enabled: true - address: "project_contact+%{key}@example.com" - user: "project_contact@example.com" - password: "[REDACTED]" - host: "imap.gmail.com" - port: 993 - ssl: true - start_tls: false - log_path: "log/mailroom.log" - mailbox: "inbox" - idle_timeout: 60 - expunge_deleted: true - ``` +::Tabs -- Example for Omnibus GitLab installations: +:::TabTitle Linux package (Omnibus) - ```ruby - gitlab_rails['service_desk_email_enabled'] = true - gitlab_rails['service_desk_email_address'] = "project_contact+%{key}@gmail.com" - gitlab_rails['service_desk_email_email'] = "project_contact@gmail.com" - gitlab_rails['service_desk_email_password'] = "[REDACTED]" - gitlab_rails['service_desk_email_mailbox_name'] = "inbox" - gitlab_rails['service_desk_email_idle_timeout'] = 60 - gitlab_rails['service_desk_email_log_file'] = "/var/log/gitlab/mailroom/mail_room_json.log" - gitlab_rails['service_desk_email_host'] = "imap.gmail.com" - gitlab_rails['service_desk_email_port'] = 993 - gitlab_rails['service_desk_email_ssl'] = true - gitlab_rails['service_desk_email_start_tls'] = false - ``` +NOTE: +In GitLab 15.3 and later, Service Desk uses `webhook` (internal API call) by default instead of enqueuing a Sidekiq job. +To use `webhook` on an Omnibus installation running GitLab 15.3, you must generate a secret file. +For more context, visit [Omnibus GitLab MR 5927](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5927). +In GitLab 15.4, reconfiguring an Omnibus installation generates this secret file automatically, so no secret file configuration setting is needed. +For details, visit [issue 1462](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1462). + +```ruby +gitlab_rails['service_desk_email_enabled'] = true +gitlab_rails['service_desk_email_address'] = "project_contact+%{key}@gmail.com" +gitlab_rails['service_desk_email_email'] = "project_contact@gmail.com" +gitlab_rails['service_desk_email_password'] = "[REDACTED]" +gitlab_rails['service_desk_email_mailbox_name'] = "inbox" +gitlab_rails['service_desk_email_idle_timeout'] = 60 +gitlab_rails['service_desk_email_log_file'] = "/var/log/gitlab/mailroom/mail_room_json.log" +gitlab_rails['service_desk_email_host'] = "imap.gmail.com" +gitlab_rails['service_desk_email_port'] = 993 +gitlab_rails['service_desk_email_ssl'] = true +gitlab_rails['service_desk_email_start_tls'] = false +``` + +:::TabTitle Self-compiled (source) + +```yaml +service_desk_email: + enabled: true + address: "project_contact+%{key}@example.com" + user: "project_contact@example.com" + password: "[REDACTED]" + host: "imap.gmail.com" + delivery_method: webhook + secret_file: .gitlab-mailroom-secret + port: 993 + ssl: true + start_tls: false + log_path: "log/mailroom.log" + mailbox: "inbox" + idle_timeout: 60 + expunge_deleted: true +``` + +::EndTabs The configuration options are the same as for configuring [incoming email](../../administration/incoming_email.md#set-it-up). @@ -360,12 +373,17 @@ does not count toward the license limit count. ### Moving a Service Desk issue +> [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/372246) in GitLab 15.7: customers continue receiving notifications when a Service Desk issue is moved. + Service Desk issues can be moved like any other issue in GitLab. You can move a Service Desk issue the same way you [move a regular issue](issues/managing_issues.md#move-an-issue) in GitLab. -If a Service Desk issue is moved to a different project the customer who created the issue stops receiving emails. +If a Service Desk issue is moved to a different project with Service Desk enabled, +the customer who created the issue continues to receive email notifications. +Because a moved issue is first closed, then copied, the customer is considered to be a participant +in both issues. They continue to receive any notifications in the old issue and the new one. ## Troubleshooting Service Desk diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md index 3c12bb9b80f..d83a80ddb13 100644 --- a/doc/user/project/settings/import_export.md +++ b/doc/user/project/settings/import_export.md @@ -12,7 +12,22 @@ then imported into a new GitLab instance. You can also: - [Migrate groups](../../group/import/index.md) using the preferred method. - [Migrate groups using file exports](../../group/settings/import_export.md). -## Set up project import/export +GitLab maps user contributions correctly when an admin access token is used to perform the import. + +As a result, migrating projects using file exports does not map user contributions correctly when you are importing +projects from a self-managed instance to GitLab.com. + +Instead, all GitLab user associations (such as comment author) are changed to the user importing the project. For more +information, see the prerequisites and important notes in these sections: + +- [Export a project and its data](../settings/import_export.md#export-a-project-and-its-data). +- [Import the project](../settings/import_export.md#import-a-project-and-its-data). + +To preserve contribution history, [migrate using direct transfer](../../group/import/index.md#migrate-groups-by-direct-transfer-recommended). + +If you migrate from GitLab.com to self-managed GitLab, an administrator can create users on the self-managed GitLab instance. + +## Set up project to migrate using file exports Before you can import or export a project and its data, you must set it up. @@ -24,8 +39,7 @@ Before you can import or export a project and its data, you must set it up. ## Between CE and EE You can export projects from the [Community Edition to the Enterprise Edition](https://about.gitlab.com/install/ce-or-ee/) -and vice versa. This assumes [version history](#version-history) -requirements are met. +and vice versa. This assumes [version history](#version-history) requirements are met. If you're exporting a project from the Enterprise Edition to the Community Edition, you may lose data that is retained only in the Enterprise Edition. For more information, see @@ -37,8 +51,7 @@ Before you can import a project, you must export it. Prerequisites: -- Review the list of [items that are exported](#items-that-are-exported). - Not all items are exported. +- Review the list of [items that are exported](#items-that-are-exported). Not all items are exported. - You must have at least the Maintainer role for the project. To export a project and its data, follow these steps: @@ -134,7 +147,7 @@ To import a project: 1. Enter your project name and URL. Then select the file you exported previously. 1. Select **Import project** to begin importing. Your newly imported project page appears shortly. -To get the status of an import, you can query it through the [Project import/export API](../../../api/project_import_export.md#import-status). +To get the status of an import, you can query it through the [API](../../../api/project_import_export.md#import-status). As described in the API documentation, the query may return an import error or exceptions. ### Changes to imported items @@ -187,7 +200,7 @@ Imported users can be mapped by their public email addresses on self-managed ins For project migration imports performed over GitLab.com groups, preserving author information is possible through a [professional services engagement](https://about.gitlab.com/services/migration/). -## Rate Limits +## Rate limits To help avoid abuse, by default, users are rate limited to: @@ -197,9 +210,6 @@ To help avoid abuse, by default, users are rate limited to: | Download export | 1 download per group per minute | | Import | 6 projects per minute | -GitLab.com may have [different settings](../../gitlab_com/index.md#importexport) -from the defaults. - ## Version history ### 14.0+ @@ -211,8 +221,8 @@ is NDJSON. ### 13.0+ Starting with GitLab 13.0, GitLab can import bundles that were exported from a different GitLab deployment. -This ability is limited to two previous GitLab [minor](../../../policy/maintenance.md#versioning) -releases, which is similar to our process for [Security Releases](../../../policy/maintenance.md#security-releases). +**This ability is limited to two previous GitLab [minor](../../../policy/maintenance.md#versioning) +releases**, which is similar to our process for [Security Releases](../../../policy/maintenance.md#security-releases). For example: @@ -221,36 +231,9 @@ For example: | 13.0 | 13.0, 12.10, 12.9 | | 13.1 | 13.1, 13.0, 12.10 | -### 12.x - -Prior to 13.0 this was a defined compatibility table: - -| Exporting GitLab version | Importing GitLab version | -| -------------------------- | -------------------------- | -| 11.7 to 12.10 | 11.7 to 12.10 | -| 11.1 to 11.6 | 11.1 to 11.6 | -| 10.8 to 11.0 | 10.8 to 11.0 | -| 10.4 to 10.7 | 10.4 to 10.7 | -| 10.3 | 10.3 | -| 10.0 to 10.2 | 10.0 to 10.2 | -| 9.4 to 9.6 | 9.4 to 9.6 | -| 9.2 to 9.3 | 9.2 to 9.3 | -| 8.17 to 9.1 | 8.17 to 9.1 | -| 8.13 to 8.16 | 8.13 to 8.16 | -| 8.12 | 8.12 | -| 8.10.3 to 8.11 | 8.10.3 to 8.11 | -| 8.10.0 to 8.10.2 | 8.10.0 to 8.10.2 | -| 8.9.5 to 8.9.11 | 8.9.5 to 8.9.11 | -| 8.9.0 to 8.9.4 | 8.9.0 to 8.9.4 | - -Projects can be exported and imported only between versions of GitLab with matching Import/Export versions. - -For example, 8.10.3 and 8.11 have the same Import/Export version (0.1.3) -and the exports between them are compatible. - ## Related topics -- [Project import/export API](../../../api/project_import_export.md) -- [Project import/export administration Rake tasks](../../../administration/raketasks/project_import_export.md) -- [Group import/export](../../group/settings/import_export.md) -- [Group import/export API](../../../api/group_import_export.md) +- [Project import and export API](../../../api/project_import_export.md) +- [Project import and export administration Rake tasks](../../../administration/raketasks/project_import_export.md) +- [Migrating GitLab groups](../../group/import/index.md) +- [Group import and export API](../../../api/group_import_export.md) diff --git a/doc/user/project/settings/index.md b/doc/user/project/settings/index.md index a872a339433..a88427ab20b 100644 --- a/doc/user/project/settings/index.md +++ b/doc/user/project/settings/index.md @@ -1,7 +1,7 @@ --- stage: Manage group: Workspace -info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" +info: 'To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments' type: reference, index, howto --- @@ -75,41 +75,44 @@ To configure visibility, features, and permissions for a project: Use the toggles to enable or disable features in the project. -| Option | More access limit options | Description | -|:---------------------------------|:--------------------------|:--------------| -| **Issues** | ✓ | Activates the GitLab issues tracker. | -| **Repository** | ✓ | Enables [repository](../repository/index.md) functionality | +| Option | More access limit options | Description | +| :------------------------------- | :------------------------ | :----------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Issues** | ✓ | Activates the GitLab issues tracker. | +| **Repository** | ✓ | Enables [repository](../repository/index.md) functionality | | **Merge requests** | ✓ | Enables [merge request](../merge_requests/index.md) functionality; also see [Merge request settings](#configure-merge-request-settings-for-a-project). | -| **Forks** | ✓ | Enables [forking](../repository/forking_workflow.md) functionality. | -| **Git Large File Storage (LFS)** | | Enables the use of [large files](../../../topics/git/lfs/index.md#git-large-file-storage-lfs). | -| **Packages** | | Supports configuration of a [package registry](../../../administration/packages/index.md#gitlab-package-registry-administration) functionality. | -| **CI/CD** | ✓ | Enables [CI/CD](../../../ci/index.md) functionality. | -| **Container Registry** | | Activates a [registry](../../packages/container_registry/index.md) for your Docker images. | -| **Analytics** | ✓ | Enables [analytics](../../analytics/index.md). | -| **Requirements** | ✓ | Control access to [Requirements Management](../requirements/index.md). | -| **Security & Compliance** | ✓ | Control access to [security features](../../application_security/index.md). | -| **Wiki** | ✓ | Enables a separate system for [documentation](../wiki/index.md). | -| **Snippets** | ✓ | Enables [sharing of code and text](../../snippets.md). | -| **Pages** | ✓ | Allows you to [publish static websites](../pages/index.md). | -| **Metrics Dashboard** | ✓ | Control access to [metrics dashboard](../integrations/prometheus.md). | -| **Releases** | ✓ | Control access to [Releases](../releases/index.md). | +| **Forks** | ✓ | Enables [forking](../repository/forking_workflow.md) functionality. | +| **Git Large File Storage (LFS)** | | Enables the use of [large files](../../../topics/git/lfs/index.md#git-large-file-storage-lfs). | +| **Packages** | | Supports configuration of a [package registry](../../../administration/packages/index.md#gitlab-package-registry-administration) functionality. | +| **CI/CD** | ✓ | Enables [CI/CD](../../../ci/index.md) functionality. | +| **Container Registry** | | Activates a [registry](../../packages/container_registry/index.md) for your Docker images. | +| **Analytics** | ✓ | Enables [analytics](../../analytics/index.md). | +| **Requirements** | ✓ | Control access to [Requirements Management](../requirements/index.md). | +| **Security & Compliance** | ✓ | Control access to [security features](../../application_security/index.md). | +| **Wiki** | ✓ | Enables a separate system for [documentation](../wiki/index.md). | +| **Snippets** | ✓ | Enables [sharing of code and text](../../snippets.md). | +| **Pages** | ✓ | Allows you to [publish static websites](../pages/index.md). | +| **Metrics Dashboard** | ✓ | Control access to [metrics dashboard](../integrations/prometheus.md). | +| **Releases** | ✓ | Control access to [Releases](../releases/index.md). | | **Environments** | ✓ | Control access to [Environments and Deployments](../../../ci/environments/index.md). | | **Feature flags** | ✓ | Control access to [Feature Flags](../../../operations/feature_flags.md). | -| **Monitor** | ✓ | Control access to [Monitor](../../../operations/index.md) features. | -| **Infrastructure** | ✓ | Control access to [Infrastructure](../../infrastructure/index.md) features. | +| **Monitor** | ✓ | Control access to [Monitor](../../../operations/index.md) features. | +| **Infrastructure** | ✓ | Control access to [Infrastructure](../../infrastructure/index.md) features. | When you disable a feature, the following additional features are also disabled: - If you disable the **Issues** feature, project users cannot use: + - **Issue Boards** - **Service Desk** - Project users can still access **Milestones** from merge requests. - If you disable **Issues** and **Merge Requests**, project users cannot use: + - **Labels** - **Milestones** - If you disable **Repository**, project users cannot access: + - **Merge requests** - **CI/CD** - **Container Registry** @@ -236,10 +239,8 @@ Prerequisites: - You must have at least the Maintainer role for the [group](../../group/manage.md#create-a-group) to which you are transferring. - You must be the Owner of the project you transfer. - The group must allow creation of new projects. -- The project must not contain any [container images](../../packages/container_registry/index.md#limitations). - - If you transfer a project to a different root namespace, - the project must not contain any - [NPM packages](../../packages/npm_registry/index.md#limitations). +- The project must not contain any [container images](../../packages/container_registry/index.md#known-issues). +- Remove any npm packages. If you transfer a project to a different root namespace, the project must not contain any npm packages. When you update the path of a user or group, or transfer a subgroup or project, you must remove any npm packages first. You cannot update the root namespace of a project with npm packages. Make sure you update your .npmrc files to follow the naming convention and run npm publish if necessary. To transfer a project: @@ -262,7 +263,7 @@ When you transfer a project from a namespace licensed for GitLab SaaS Premium or - [Project access tokens](../../../user/project/settings/project_access_tokens.md) are revoked - [Pipeline subscriptions](../../../ci/pipelines/index.md#trigger-a-pipeline-when-an-upstream-project-is-rebuilt) -and [test cases](../../../ci/test_cases/index.md) are deleted. + and [test cases](../../../ci/test_cases/index.md) are deleted. ## Delete a project @@ -270,7 +271,7 @@ You can mark a project to be deleted. Prerequisite: -- You must have at least the Owner role for a project. +- You must have the Owner role for a project. To delete a project: @@ -308,7 +309,7 @@ If you don't want to wait, you can delete a project immediately. Prerequisites: -- You must have at least the Owner role for a project. +- You must have the Owner role for a project. - You have [marked the project for deletion](#delete-a-project). To immediately delete a project marked for deletion: @@ -361,11 +362,11 @@ Configure [alert integrations](../../../operations/incident_management/integrati #### Alert integration -Automatically [create](../../../operations/incident_management/incidents.md#create-incidents-automatically), [notify on](../../../operations/incident_management/paging.md#email-notifications-for-alerts), and [resolve](../../../operations/incident_management/incidents.md#automatically-close-incidents-via-recovery-alerts) incidents based on GitLab alerts. +Automatically [create](../../../operations/metrics/alerts.md#trigger-actions-from-alerts), [notify on](../../../operations/incident_management/paging.md#email-notifications-for-alerts), and [resolve](../../../operations/incident_management/manage_incidents.md#automatically-close-incidents-via-recovery-alerts) incidents based on GitLab alerts. #### PagerDuty integration -[Create incidents in GitLab for each PagerDuty incident](../../../operations/incident_management/incidents.md#create-incidents-via-the-pagerduty-webhook). +[Create incidents in GitLab for each PagerDuty incident](../../../operations/incident_management/manage_incidents.md#using-the-pagerduty-webhook). #### Incident settings diff --git a/doc/user/project/settings/project_access_tokens.md b/doc/user/project/settings/project_access_tokens.md index f27672a1b07..cb7ba2a7df2 100644 --- a/doc/user/project/settings/project_access_tokens.md +++ b/doc/user/project/settings/project_access_tokens.md @@ -69,6 +69,11 @@ To create a project access token: A project access token is displayed. Save the project access token somewhere safe. After you leave or refresh the page, you can't view it again. +WARNING: +Project access tokens are treated as [internal users](../../../development/internal_users.md). +If an internal user creates a project access token, that token is able to access +all projects that have visibility level set to [Internal](../../public_access.md). + ## Revoke a project access token To revoke a project access token: diff --git a/doc/user/project/time_tracking.md b/doc/user/project/time_tracking.md index 71b8c911839..186ca0ba9f0 100644 --- a/doc/user/project/time_tracking.md +++ b/doc/user/project/time_tracking.md @@ -74,6 +74,25 @@ Prerequisites: - You must have at least the Reporter role for the project. +#### Using the user interface + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101563) in GitLab 15.7. + +To add a time entry using the user interface: + +1. In the **Time tracking** section of the sidebar, select **Add time entry** (**{plus}**). A modal opens. +1. Enter: + + - The amount of time spent. + - Optional. When it was spent. + - Optional. A summary. + +1. Select **Save**. + +The **Spent** total in the sidebar is updated and you can view all entries in a [time tracking report](#view-a-time-tracking-report). + +#### Using a quick action + To enter time spent, use the `/spend` [quick action](quick_actions.md), followed by the time. For example, if you need @@ -90,15 +109,10 @@ Draft MR and respond to initial comments /spend 30m ``` -### Add time spent on a specific date - -Prerequisites: - -- You must have at least the Reporter role for the project. +To log when time was spent, enter a date after the time, using the `YYYY-MM-DD` format. -You can log time in the past by providing a date after the time. For example, to log 1 hour of time spent on 31 January 2021, -type `/spend 1h 2021-01-31`. +enter `/spend 1h 2021-01-31`. If you type a future date, no time is logged. diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index 0200e9c4e7d..fb100986df9 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -10,6 +10,10 @@ The Web Integrated Development Environment (IDE) editor streamlines the process to contribute changes to your projects, by providing an advanced editor with commit staging. +NOTE: +The Web IDE is being updated to use VS Code. For details, +see [Web IDE Beta](../web_ide_beta/index.md). + ## Open the Web IDE Use the <kbd>.</kbd> [keyboard shortcut](../../shortcuts.md) to open the Web IDE. @@ -80,7 +84,7 @@ If you are missing Syntax Highlighting support for any language, we prepared a s > - Full Solarized Dark Theme [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/219228) in GitLab 13.1. > - Full [Solarized Light](https://gitlab.com/gitlab-org/gitlab/-/issues/221035) and [Monokai](https://gitlab.com/gitlab-org/gitlab/-/issues/221034) Themes introduced in GitLab 13.6. -All the themes GitLab supports for syntax highlighting are applied to the Web IDE's entire screen. +All the themes GitLab supports for syntax highlighting are applied to the entire Web IDE screen. You can pick a theme from your [profile preferences](../../profile/preferences.md). | Solarized Dark Theme | Dark Theme | @@ -459,12 +463,3 @@ The Web IDE has a few limitations: - If the terminal displays **Connection Failure**, then the terminal could not connect to the runner. Try to stop and restart the terminal. If the problem persists, double check your runner configuration. - -## VSCode Reimplementation - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95169) in GitLab 15.4 [with a flag](../../../administration/feature_flags.md) named `vscode_web_ide`. Disabled by default. - -As announced in [this blog post](https://about.gitlab.com/blog/2022/05/23/the-future-of-the-gitlab-web-ide/), -the current implementation of the Web IDE will be replaced with a [VSCode inspired implementation](https://gitlab.com/groups/gitlab-org/-/epics/7683). - -This effort is currently under development. Follow [this epic](https://gitlab.com/groups/gitlab-org/-/epics/7683) for updates and more information. diff --git a/doc/user/project/web_ide_beta/img/fuzzy_finder_v15_7.png b/doc/user/project/web_ide_beta/img/fuzzy_finder_v15_7.png Binary files differnew file mode 100644 index 00000000000..66ebae15e98 --- /dev/null +++ b/doc/user/project/web_ide_beta/img/fuzzy_finder_v15_7.png diff --git a/doc/user/project/web_ide_beta/index.md b/doc/user/project/web_ide_beta/index.md new file mode 100644 index 00000000000..ef07cca465d --- /dev/null +++ b/doc/user/project/web_ide_beta/index.md @@ -0,0 +1,103 @@ +--- +stage: Create +group: Editor +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Web IDE Beta **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95169) in GitLab 15.4 [with a flag](../../../administration/feature_flags.md) named `vscode_web_ide`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `vscode_web_ide`. On GitLab.com, this feature is available. + +As announced in [this blog post](https://about.gitlab.com/blog/2022/05/23/the-future-of-the-gitlab-web-ide/), +the current implementation of the Web IDE is being replaced with an +implementation inspired by Visual Studio Code. + +This effort is currently under development. For updates, +see [this epic](https://gitlab.com/groups/gitlab-org/-/epics/7683). + +## Enable the Web IDE Beta + +To use the Web IDE Beta on a self-managed GitLab instance, +ensure that the `vscode_web_ide` feature flag +[is enabled](../../../administration/feature_flags.md). + +On GitLab.com, this feature is available by default. However, you can +[stop using it if you choose](#stop-using-the-web-ide-beta). + +## Use the Web IDE Beta + +To open the Web IDE Beta from anywhere in the UI: + +- Use the <kbd>.</kbd> [keyboard shortcut](../../shortcuts.md). + +You can also open the Web IDE Beta when viewing a file, the repository file list, +and from merge requests. + +### Use when viewing a file or the repository file list + +To open the Web IDE Beta from a file or the repository file list: + +- On the top right of the page, select **Open in Web IDE**. + +If **Open in Web IDE** is not visible: + +1. Next to **Edit** or **Gitpod**, select the down arrow (**{chevron-lg-down}**). +1. From the list, select **Open in Web IDE**. +1. Select **Open in Web IDE**. + +### Use when viewing a merge request + +To open the Web IDE Beta from a merge request: + +1. Go to your merge request. +1. In the upper right corner, select **Code > Open in Web IDE**. + +## Open a file in the Web IDE Beta + +To open any file by its name: + +1. Type **Command** + **`P`** (<kbd>⌘</kbd> + <kbd>P</kbd>). +1. Type the name of your file. + + + +## Search across files + +You can use VS Code to quickly search all files in the currently opened folder. + +To enter your search term: + +1. Type **Shift** + **Command** + **`F`** (<kbd>⇧</kbd> + <kbd>⌘</kbd> + <kbd>F</kbd>). +1. Enter your search term. + +In the Web IDE Beta, only partial results from opened files are displayed. +Full file search is planned for a later date. + +## View list of changed files + +To view the list of files you changed in the Web IDE Beta: + +- On the VS Code Activity Bar, on the left, select the Source Control icon: + +Your `CHANGES`, `STAGED CHANGES` and `MERGE CHANGES` are displayed. + +For details, see [the VS Code documentation](https://code.visualstudio.com/docs/sourcecontrol/overview#_commit). + +## Known issues + +The [Web Terminal](../web_ide/index.md#interactive-web-terminals-for-the-web-ide) +and [Live Preview](../web_ide/index.md#live-preview) are not available in the Web IDE Beta. + +These features may become available at a later date. + +### Stop using the Web IDE Beta + +If you do not want to use the Web IDE Beta, you can change your personal preferences. + +1. On the top bar, in the top right corner, select your avatar. +1. Select **Preferences**. +1. In the **Web IDE** section, select the **Opt out of the Web IDE Beta** checkbox. +1. Select **Save changes**. diff --git a/doc/user/project/wiki/group.md b/doc/user/project/wiki/group.md index 0e9b76e943d..53aaa41319b 100644 --- a/doc/user/project/wiki/group.md +++ b/doc/user/project/wiki/group.md @@ -16,8 +16,6 @@ Group wikis are similar to [project wikis](index.md), with a few limitations: - [Git LFS](../../../topics/git/lfs/index.md) is not supported. - Group wikis are not included in [global search](../../search/advanced_search.md). - Changes to group wikis don't show up in the [group's activity feed](../../group/manage.md#group-activity-analytics). -- Group wikis are enabled by default for GitLab Premium and higher tiers. - You [can't turn them off from the GitLab user interface](https://gitlab.com/gitlab-org/gitlab/-/issues/208413). For updates, follow [the epic that tracks feature parity with project wikis](https://gitlab.com/groups/gitlab-org/-/epics/2782). diff --git a/doc/user/project/wiki/index.md b/doc/user/project/wiki/index.md index eedc44be3f9..04a6f9bee80 100644 --- a/doc/user/project/wiki/index.md +++ b/doc/user/project/wiki/index.md @@ -14,7 +14,7 @@ to keep it in the same project as your code, you can use the wiki GitLab provide in each GitLab project. Every wiki is a separate Git repository, so you can create wiki pages in the web interface, or [locally using Git](#create-or-edit-wiki-pages-locally). -GitLab wikis support Markdown, RDoc, AsciiDoc, and Org for content. +GitLab wikis support Markdown, Rdoc, AsciiDoc, and Org for content. Wiki pages written in Markdown support all [Markdown features](../../markdown.md), and also provide some [wiki-specific behavior](../../markdown.md#wiki-specific-markdown) for links. @@ -338,11 +338,12 @@ GitLab provides a WYSIWYG editing experience for GitLab Flavored Markdown in wik Support includes: -- Text formatting options, including bold, italics, block quotes, headings, and inline code. -- List formatting for unordered, numbered, and checklists. -- Creating and editing the structure of tables. +- Formatting text, including using bold, italics, block quotes, headings, and inline code. +- Formatting ordered lists, unordered lists, and checklists. +- Creating and editing table structure. - Inserting and formatting code blocks with syntax highlighting. -- Live preview of Mermaid, PlantUML, and Kroki diagrams ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86701) in GitLab 15.2). +- Previewing Mermaid, PlantUML, and Kroki diagrams ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86701) in GitLab 15.2). +- Creating and editing HTML comments ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104084) in GitLab 15.7). ### Use the Content Editor diff --git a/doc/user/project/working_with_projects.md b/doc/user/project/working_with_projects.md index 067a0303277..77a7c48658e 100644 --- a/doc/user/project/working_with_projects.md +++ b/doc/user/project/working_with_projects.md @@ -347,10 +347,27 @@ You can sort projects by: You can also choose to hide or show archived projects. +## Change the visibility of individual features in a project + +You can change the visibility of individual features in a project. + +Prerequisite: + +- You must have the Owner role for the project. + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand **Visibility, project features, permissions**. +1. Use the toggle by each feature you want to turn on or off, or change access for. +1. Select **Save changes**. + ## Leave a project -If you leave a project, you are no longer a project -member and cannot contribute. +When you leave a project: + +- You are no longer a project member and cannot contribute. +- All the issues and merge requests that were assigned + to you are unassigned. To leave a project: diff --git a/doc/user/public_access.md b/doc/user/public_access.md index bdc711f2098..acfc28a6f65 100644 --- a/doc/user/public_access.md +++ b/doc/user/public_access.md @@ -7,16 +7,17 @@ type: reference # Project and group visibility **(FREE)** -GitLab allows users with the Owner role to set a project's or group's visibility as: +If you have the Owner role, you can set a project's or group's visibility as: - **Public** -- **Internal** +- **[Internal](#internal-projects-and-groups)** - **Private** -These visibility levels affect who can see the project in the public access directory (`/public` -for your GitLab instance). For example, <https://gitlab.com/public>. -You can control the visibility of individual features with -[project feature settings](permissions.md#project-features). +These visibility levels affect who can see the project in the public access directory +(for example, <https://gitlab.com/public>). + +For more granular control, you can determine +[which features in a project are visible](project/working_with_projects.md#change-the-visibility-of-individual-features-in-a-project). The visibility setting of a project must be at least as restrictive as the visibility of its parent group. @@ -38,16 +39,16 @@ By default, `/public` is visible to unauthenticated users. However, if the [**Public** visibility level](admin_area/settings/visibility_and_access_controls.md#restrict-visibility-levels) is restricted, `/public` is visible only to signed-in users. -## Internal projects and groups +## Internal projects and groups **(FREE SELF)** Internal projects can be cloned by any signed-in user except -[external users](permissions.md#external-users). +[external users](admin_area/external_users.md). They are also listed in the public access directory (`/public`), but only for signed-in users. Internal groups can have internal or private subgroups. -Any signed-in users except [external users](permissions.md#external-users) have the +Any signed-in users except [external users](admin_area/external_users.md) have the Guest role on the repository. NOTE: @@ -66,6 +67,8 @@ Private groups can only have private subgroups. ## Change project visibility +You can change the visibility of a project. + Prerequisite: - You must have the Owner role for a project. @@ -78,6 +81,8 @@ Prerequisite: ## Change group visibility +You can change the visibility of all projects in a group. + Prerequisites: - You must have the Owner role for a group. diff --git a/doc/user/read_only_namespaces.md b/doc/user/read_only_namespaces.md new file mode 100644 index 00000000000..b11aac9b00c --- /dev/null +++ b/doc/user/read_only_namespaces.md @@ -0,0 +1,48 @@ +--- +stage: Growth +group: Acquisition +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +noindex: true +--- + +# Read-only namespaces **(FREE SAAS)** + +In GitLab SaaS, a top-level namespace is placed in a read-only state when it either: + +- Exceeds the [free user limit](free_user_limit.md) when the namespace visibility is private. +- Exceeds the [storage usage quota](usage_quotas.md), regardless of namespace visibility. + +While a namespace is in a read-only state, a banner appears at the +top of the page. + +Your ability to write new data to read-only namespaces is restricted. For more +information, see [Restricted actions](#restricted-actions). + +## Remove the read-only state + +To restore a namespace to its normal state, you can: + +- For exceeded free user limits: + - [Reduce the number of members](free_user_limit.md#manage-members-in-your-namespace) in your namespace. + - [Start a free trial](https://gitlab.com/-/trial_registrations/new), which includes an unlimited number of members. + - [Purchase a paid tier](https://about.gitlab.com/pricing/). +- For exceeded storage quota: + - [Purchase more storage for the namespace](../subscriptions/gitlab_com/index.md#purchase-more-storage-and-transfer). + - [Manage your storage usage](usage_quotas.md#manage-your-storage-usage). + +## Restricted actions + +| Feature | Action restricted | +|---------|-------------------| +| Container Registry | Create, edit, and delete cleanup policies <br> Push an image to the container registry | +| Merge Requests | Create and update an MR | +| Package Registry | Publish a package | +| Repositories | Add tags <br> Create new branches <br> Create and update commit status <br> Push and force push to non-protected branches <br> Push and force push to protected branches <br> Upload files <br> Create merge requests | +| CI/CD | Create, edit, admin, and run pipelines <br> Create, edit, admin, and run builds <br> Create and edit admin environments <br> Create and edit admin deployments <br> Create and edit admin clusters <br> Create and edit admin releases | +| Namespaces | **For exceeded free user limits:** Invite new users | + +## Related topics + +- [Frequently Asked Questions - GitLab SaaS Free Tier](https://about.gitlab.com/pricing/faq-efficient-free-tier/) +- [Free user limit](free_user_limit.md) +- [Storage usage quotas](usage_quotas.md) diff --git a/doc/user/report_abuse.md b/doc/user/report_abuse.md index fab01973104..fde839c3ba4 100644 --- a/doc/user/report_abuse.md +++ b/doc/user/report_abuse.md @@ -1,6 +1,6 @@ --- -stage: none -group: unassigned +stage: Anti-Abuse +group: Anti-Abuse info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -27,7 +27,7 @@ You can report a user through their: To report abuse from a user's profile page: 1. Anywhere in GitLab, select the name of the user. -1. In the top right corner of the user's profile, select **Report abuse** (**{information-o}**). +1. In the top right corner of the user's profile, select **Report abuse to administrator** (**{information-o}**). 1. Complete an abuse report. 1. Select **Send report**. @@ -36,7 +36,7 @@ To report abuse from a user's profile page: To report abuse from a user's comment: 1. In the comment, in the top right corner, select **More actions** (**{ellipsis_v}**). -1. Select **Report abuse to admin**. +1. Select **Report abuse to administrator**. 1. Complete an abuse report. 1. Select **Send report**. @@ -47,14 +47,14 @@ A URL to the reported user's comment is pre-filled in the abuse report's ## Report abuse from an issue 1. On the issue, in the top right corner, select the vertical ellipsis (**{ellipsis_v}**). -1. Select **Report abuse**. +1. Select **Report abuse to administrator**. 1. Submit an abuse report. 1. Select **Send report**. ## Report abuse from a merge request 1. On the merge request, in the top right corner, select the vertical ellipsis (**{ellipsis_v}**). -1. Select **Report abuse**. +1. Select **Report abuse to administrator**. 1. Submit an abuse report. 1. Select **Send report**. diff --git a/doc/user/reserved_names.md b/doc/user/reserved_names.md index 1ab22fb846d..3eaf3178f3a 100644 --- a/doc/user/reserved_names.md +++ b/doc/user/reserved_names.md @@ -19,10 +19,13 @@ under the `TOP_LEVEL_ROUTES`, `PROJECT_WILDCARD_ROUTES` and `GROUP_ROUTES` lists ## Limitations on project and group names -- Special characters are not permitted at the start or end of project or group names. They are permitted in any other location of the name. -- Project or group names cannot end in `.git` or `.atom`. +- Project or group names must start with a letter, digit, emoji, or "_". - Project or group names can only contain letters, digits, emojis, "_", ".", "+", dashes, or spaces. -- Paths can only contain letters, digits, "_", "-", and "." +- Project or group slugs must start with a letter or digit. +- Project or group slugs can only contain letters, digits, '_', '.', '+', or dashes. +- Project or group slugs must not contain consecutive special characters. +- Project or group slugs cannot end with a special character. +- Project or group slugs cannot end in `.git` or `.atom`. ## Reserved project names diff --git a/doc/user/search/advanced_search.md b/doc/user/search/advanced_search.md index 925fc7e6036..ed1d3b1d290 100644 --- a/doc/user/search/advanced_search.md +++ b/doc/user/search/advanced_search.md @@ -29,11 +29,27 @@ You can use Advanced Search in: - Commits - Project wikis (not [group wikis](../project/wiki/group.md)) -## Configure Advanced Search +For Advanced Search: + +- You can only search files smaller than 1 MB. + For self-managed GitLab instances, an administrator can + [change this limit](../../integration/advanced_search/elasticsearch.md#advanced-search-configuration). +- You can't use any of the following characters in the search query: + + ```plaintext + . , : ; / ` ' = ? $ & ^ | ~ < > ( ) { } [ ] @ + ``` + +- Only the default branch of a project is indexed for code search. + In a non-default branch, basic search is used. +- Search results show only the first match in a file, + but there might be more results in that file. + +## Enable Advanced Search - On GitLab.com, Advanced Search is enabled for groups with paid subscriptions. - For self-managed GitLab instances, an administrator must - [configure Advanced Search](../../integration/advanced_search/elasticsearch.md). + [enable Advanced Search](../../integration/advanced_search/elasticsearch.md#enable-advanced-search). ## Syntax diff --git a/doc/user/search/img/basic_search_results_v15_1.png b/doc/user/search/img/basic_search_results_v15_1.png Binary files differdeleted file mode 100644 index 0de0b976d7d..00000000000 --- a/doc/user/search/img/basic_search_results_v15_1.png +++ /dev/null diff --git a/doc/user/search/img/basic_search_v15_1.png b/doc/user/search/img/basic_search_v15_1.png Binary files differdeleted file mode 100644 index 069d62ca80c..00000000000 --- a/doc/user/search/img/basic_search_v15_1.png +++ /dev/null diff --git a/doc/user/search/img/search_navbar_v15_7.png b/doc/user/search/img/search_navbar_v15_7.png Binary files differnew file mode 100644 index 00000000000..9175347b36f --- /dev/null +++ b/doc/user/search/img/search_navbar_v15_7.png diff --git a/doc/user/search/img/search_scope_v15_7.png b/doc/user/search/img/search_scope_v15_7.png Binary files differnew file mode 100644 index 00000000000..6395b5f1cda --- /dev/null +++ b/doc/user/search/img/search_scope_v15_7.png diff --git a/doc/user/search/index.md b/doc/user/search/index.md index 9bff2a91ec8..9536f7fe40a 100644 --- a/doc/user/search/index.md +++ b/doc/user/search/index.md @@ -53,12 +53,12 @@ Global search only flags with an error any search that includes more than: To start a search, type your search query in the search bar on the top-right of the screen. You must type at least two characters. - + After the results are displayed, you can modify the search, select a different type of data to search, or choose a specific group or project. - + ## Search in code @@ -150,6 +150,7 @@ To delete filter tokens one at a time, the <kbd>⌥</kbd> (Mac) / <kbd>Control</ In the search bar, you can view autocomplete suggestions for: - Projects and groups +- Users - Various help pages (try and type **API help**) - Project feature pages (try and type **milestones**) - Various settings pages (try and type **user settings**) diff --git a/doc/user/shortcuts.md b/doc/user/shortcuts.md index f9e61ad78ad..64f9b53f891 100644 --- a/doc/user/shortcuts.md +++ b/doc/user/shortcuts.md @@ -37,6 +37,7 @@ These shortcuts are available in most areas of GitLab: | <kbd>f</kbd> | Put cursor in the filter bar. | | <kbd>Shift</kbd> + <kbd>i</kbd> | Go to your Issues page. | | <kbd>Shift</kbd> + <kbd>m</kbd> | Go to your [Merge requests](project/merge_requests/index.md) page. | +| <kbd>Shift</kbd> + <kbd>r</kbd> | Go to your Review requests page. | | <kbd>Shift</kbd> + <kbd>t</kbd> | Go to your To-Do List page. | | <kbd>p</kbd>, then <kbd>b</kbd> | Show or hide the Performance Bar. | | <kbd>Escape</kbd> | Hide tooltips or popovers. | diff --git a/doc/user/snippets.md b/doc/user/snippets.md index 6a8184e9ca1..ee84f8a4169 100644 --- a/doc/user/snippets.md +++ b/doc/user/snippets.md @@ -267,3 +267,7 @@ creating a new snippet, use this workaround: 1. Enter any string into the text area for the second file. 1. Scroll back to the first filename, and select **Delete file**. 1. Create the rest of your file, and select **Create snippet** when done. + +## Related topics + +- [Configure snippet settings](../administration/snippets/index.md) on a self-managed GitLab instance diff --git a/doc/user/ssh.md b/doc/user/ssh.md index 85332446334..1d82d301e9a 100644 --- a/doc/user/ssh.md +++ b/doc/user/ssh.md @@ -270,6 +270,7 @@ You can use [1Password](https://1password.com/) and the [1Password browser exten 1. You can then select **Create SSH Key** or select an existing SSH key to fill in the public key. 1. In the **Title** box, type a description, like `Work Laptop` or `Home Workstation`. +1. Optional. Select the **Usage type** of the key. It can be used either for `Authentication` or `Signing` or both. `Authentication & Signing` is the default value. 1. Optional. Update **Expiration date** to modify the default expiration date. 1. Select **Add key**. @@ -314,6 +315,7 @@ To use SSH with GitLab, copy your public key to your GitLab account: `ssh-ed25519`, `sk-ecdsa-sha2-nistp256@openssh.com`, or `sk-ssh-ed25519@openssh.com`, and may end with a comment. 1. In the **Title** box, type a description, like `Work Laptop` or `Home Workstation`. +1. Optional. Select the **Usage type** of the key. It can be used either for `Authentication` or `Signing` or both. `Authentication & Signing` is the default value. 1. Optional. Update **Expiration date** to modify the default expiration date. In: - GitLab 13.12 and earlier, the expiration date is informational only. It doesn't prevent diff --git a/doc/user/tasks.md b/doc/user/tasks.md index 5fed887ca74..ffe7777fac0 100644 --- a/doc/user/tasks.md +++ b/doc/user/tasks.md @@ -49,7 +49,7 @@ the task opens in a full-page view. Prerequisites: -- You must have at least the Guest role for the project, or the project must be public. +- You must have at least the Reporter role for the project, or the project must be public. To create a task: @@ -207,10 +207,9 @@ To set a start date: ## Add a task to a milestone -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367463) in GitLab 15.5 [with a flag](../administration/feature_flags.md) named `work_items_mvc_2`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available per group, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `work_items_mvc_2`. On GitLab.com, this feature is not available. The feature is not ready for production use. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367463) in GitLab 15.5 [with a flag](../administration/feature_flags.md) named `work_items_mvc_2`. Disabled by default. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/367463) to feature flag named `work_items_mvc` in GitLab 15.7. Disabled by default. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/367463) in GitLab 15.7. You can add a task to a [milestone](project/milestones/index.md). You can see the milestone title when you view a task. @@ -249,10 +248,13 @@ To set issue weight of a task: ## Add a task to an iteration **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367456) in GitLab 15.5 [with a flag](../administration/feature_flags.md) named `work_items_mvc_2`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367456) in GitLab 15.5 [with a flag](../administration/feature_flags.md) named `work_items_mvc_2`. Disabled by default. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/367456) to feature flag named `work_items_mvc` in GitLab 15.7. Disabled by default. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/367456) in GitLab 15.7. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available per group, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `work_items_mvc_2`. On GitLab.com, this feature is not available. The feature is not ready for production use. +On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../administration/feature_flags.md) named `work_items_mvc`. +On GitLab.com, this feature is available. You can add a task to an [iteration](group/iterations/index.md). You can see the iteration title and period only when you view a task. diff --git a/doc/user/todos.md b/doc/user/todos.md index d7deda33a65..4102d31cfc4 100644 --- a/doc/user/todos.md +++ b/doc/user/todos.md @@ -87,6 +87,7 @@ You can manually add an item to your To-Do List. - [Merge request](project/merge_requests/index.md) - [Epic](group/epics/index.md) - [Design](project/issues/design_management.md) + - [Incident](../operations/incident_management/incidents.md) 1. On the right sidebar, at the top, select **Add a to do**. @@ -135,7 +136,7 @@ You can manually mark a to-do item as done. There are two ways to do this: -- In the To-Do List, to the right of the to-do item, select **Done**. +- In the To-Do List, to the right of the to-do item, select **Mark as done** (**{check}**). - In the sidebar of an issue, merge request, or epic, select **Mark as done**.  diff --git a/doc/user/upgrade_email_bypass.md b/doc/user/upgrade_email_bypass.md index ef58d8aec66..afbdcbcdf09 100644 --- a/doc/user/upgrade_email_bypass.md +++ b/doc/user/upgrade_email_bypass.md @@ -76,7 +76,7 @@ Your primary email address is not confirmed. You can assure your users that they have not been [Blocked](admin_area/moderate_users.md#block-and-unblock-users) by an administrator. When affected users see this message, they must confirm their email address before they can commit code. -## What do I need to know as an administrator of a GitLab self-managed Instance? +## What do you need to know as an administrator of a GitLab self-managed Instance? You have the following options to help your users: @@ -85,7 +85,7 @@ You have the following options to help your users: As an administrator, you may also confirm a user in the [Admin Area](admin_area/index.md#administering-users). -## What do I do if I am an administrator and I am locked out? +## What do you do if you are an administrator and you're locked out? If you are an administrator and cannot otherwise verify your email address, sign in to your GitLab instance with a [Rails console session](../administration/operations/rails_console.md#starting-a-rails-console-session). @@ -97,7 +97,7 @@ admin.confirmed_at = Time.zone.now admin.save! ``` -## How do I force-confirm all users on my self-managed instance? +## How do you force-confirm all users on your self-managed instance? If you are an administrator and would like to force-confirm all users on your system, sign in to your GitLab instance with a [Rails console session](../administration/operations/rails_console.md#starting-a-rails-console-session). diff --git a/doc/user/usage_quotas.md b/doc/user/usage_quotas.md index 7df5ade215d..0e5703caffc 100644 --- a/doc/user/usage_quotas.md +++ b/doc/user/usage_quotas.md @@ -126,8 +126,7 @@ Storage types that add to the total namespace storage are: - Wiki - Snippets -If your total namespace storage exceeds the available namespace storage quota, all projects under the namespace are locked. -A locked project cannot push to the repository, run pipelines and jobs, or build and push packages. +If your total namespace storage exceeds the available namespace storage quota, all projects under the namespace become read-only. Your ability to write new data is restricted until the read-only state is removed. For more information, see [Restricted actions](../user/read_only_namespaces.md#restricted-actions). To prevent exceeding the namespace storage quota, you can: diff --git a/doc/user/workspace/index.md b/doc/user/workspace/index.md index 5bcd96cd4a5..4a8b083e3a2 100644 --- a/doc/user/workspace/index.md +++ b/doc/user/workspace/index.md @@ -26,12 +26,12 @@ everything you do as a GitLab administrator, including: Our goal is to reach feature parity between SaaS and self-managed installations, with all [Admin Area settings](/ee/user/admin_area/settings/index.md) moving to either: -- Groups. Available in the Workspace, top-level group namespaces, and subgroups. +- Groups. Available in the Workspace, top-level groups, and subgroups. - Hardware Controls. For functionality that does not apply to groups, Hardware Controls are only applicable to self-managed installations. There is one Hardware Controls section per installation. -To learn about the current state of workspace development, -see [epic 4257](https://gitlab.com/groups/gitlab-org/-/epics/4257). +For more information about the state of workspace development, +see [epic 9265](https://gitlab.com/groups/gitlab-org/-/epics/9265). <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For a video introduction to the new hierarchy concept for groups and projects for epics, see |
