diff options
61 files changed, 789 insertions, 849 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21dfd6563e2..d9b0185a8d8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,7 @@ stages: # in cases where jobs require Docker-in-Docker, the job # definition must be extended with `.use-docker-in-docker` default: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-81.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34" tags: - gitlab-org # All jobs are interruptible by default diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 5be4d4eaf1f..2df24eedc1f 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -15,7 +15,7 @@ - .default-retry - .default-before_script - .assets-compile-cache - image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-graphicsmagick-1.3.34-docker-19.03.1 + image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-git-2.26-lfs-2.9-chrome-81.0-node-12.x-yarn-1.21-graphicsmagick-1.3.34-docker-19.03.1 stage: prepare variables: NODE_ENV: "production" diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 0c26273ef04..21d085c5ca5 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -30,7 +30,7 @@ policy: pull .use-pg9: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-81.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34" services: - name: postgres:9.6.17 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -41,7 +41,7 @@ key: "debian-stretch-ruby-2.6.6-pg9-node-12.x" .use-pg10: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-10-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-81.0-node-12.x-yarn-1.21-postgresql-10-graphicsmagick-1.3.34" services: - name: postgres:10.12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -52,7 +52,7 @@ key: "debian-stretch-ruby-2.6.6-pg10-node-12.x" .use-pg11: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-81.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -63,7 +63,7 @@ key: "debian-stretch-ruby-2.6.6-pg11-node-12.x" .use-pg9-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-81.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34" services: - name: postgres:9.6.17 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -75,7 +75,7 @@ key: "debian-stretch-ruby-2.6.6-pg9-node-12.x" .use-pg10-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-10-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-81.0-node-12.x-yarn-1.21-postgresql-10-graphicsmagick-1.3.34" services: - name: postgres:10.12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -87,7 +87,7 @@ key: "debian-stretch-ruby-2.6.6-pg10-node-12.x" .use-pg11-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-81.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index acf047fe0a8..f2de98cbfab 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -141,7 +141,9 @@ db:migrate:reset: - bundle exec rake db:migrate:reset db:check-schema: - extends: .db-job-base + extends: + - .db-job-base + - .rails:rules:ee-mr-and-master-only script: - scripts/regenerate-schema - source scripts/schema_changed.sh @@ -445,7 +445,7 @@ gem 'sys-filesystem', '~> 1.1.6' gem 'net-ntp' # SSH host key support -gem 'net-ssh', '~> 5.2' +gem 'net-ssh', '~> 6.0' gem 'sshkey', '~> 2.0' # Required for ED25519 SSH host key support diff --git a/Gemfile.lock b/Gemfile.lock index eb1a0eb5a27..edd8b9c73c5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -653,7 +653,7 @@ GEM nenv (0.3.0) net-ldap (0.16.2) net-ntp (2.1.3) - net-ssh (5.2.0) + net-ssh (6.0.0) netrc (0.11.0) nio4r (2.5.2) no_proxy_fix (0.1.2) @@ -1301,7 +1301,7 @@ DEPENDENCIES nakayoshi_fork (~> 0.0.4) net-ldap net-ntp - net-ssh (~> 5.2) + net-ssh (~> 6.0) nokogiri (~> 1.10.9) oauth2 (~> 1.4) octokit (~> 4.15) diff --git a/app/assets/javascripts/users_select/constants.js b/app/assets/javascripts/users_select/constants.js new file mode 100644 index 00000000000..64df1e1748c --- /dev/null +++ b/app/assets/javascripts/users_select/constants.js @@ -0,0 +1,18 @@ +export const AJAX_USERS_SELECT_OPTIONS_MAP = { + projectId: 'projectId', + groupId: 'groupId', + showCurrentUser: 'currentUser', + authorId: 'authorId', + skipUsers: 'skipUsers', +}; + +export const AJAX_USERS_SELECT_PARAMS_MAP = { + project_id: 'projectId', + group_id: 'groupId', + skip_ldap: 'skipLdap', + todo_filter: 'todoFilter', + todo_state_filter: 'todoStateFilter', + current_user: 'showCurrentUser', + author_id: 'authorId', + skip_users: 'skipUsers', +}; diff --git a/app/assets/javascripts/users_select.js b/app/assets/javascripts/users_select/index.js index 8efca67ef19..2dbe5a8171e 100644 --- a/app/assets/javascripts/users_select.js +++ b/app/assets/javascripts/users_select/index.js @@ -4,10 +4,15 @@ import $ from 'jquery'; import { escape, template, uniqBy } from 'lodash'; -import axios from './lib/utils/axios_utils'; -import { s__, __, sprintf } from './locale'; -import ModalStore from './boards/stores/modal_store'; -import { parseBoolean } from './lib/utils/common_utils'; +import axios from '../lib/utils/axios_utils'; +import { s__, __, sprintf } from '../locale'; +import ModalStore from '../boards/stores/modal_store'; +import { parseBoolean } from '../lib/utils/common_utils'; +import { + AJAX_USERS_SELECT_OPTIONS_MAP, + AJAX_USERS_SELECT_PARAMS_MAP, +} from 'ee_else_ce/users_select/constants'; +import { getAjaxUsersSelectOptions, getAjaxUsersSelectParams } from './utils'; // TODO: remove eventHub hack after code splitting refactor window.emitSidebarEvent = window.emitSidebarEvent || $.noop; @@ -555,13 +560,8 @@ function UsersSelect(currentUser, els, options = {}) { import(/* webpackChunkName: 'select2' */ 'select2/select2') .then(() => { $('.ajax-users-select').each((i, select) => { - const options = {}; + const options = getAjaxUsersSelectOptions($(select), AJAX_USERS_SELECT_OPTIONS_MAP); options.skipLdap = $(select).hasClass('skip_ldap'); - options.projectId = $(select).data('projectId'); - options.groupId = $(select).data('groupId'); - options.showCurrentUser = $(select).data('currentUser'); - options.authorId = $(select).data('authorId'); - options.skipUsers = $(select).data('skipUsers'); const showNullUser = $(select).data('nullUser'); const showAnyUser = $(select).data('anyUser'); const showEmailUser = $(select).data('emailUser'); @@ -702,14 +702,7 @@ UsersSelect.prototype.users = function(query, options, callback) { const params = { search: query, active: true, - project_id: options.projectId || null, - group_id: options.groupId || null, - skip_ldap: options.skipLdap || null, - todo_filter: options.todoFilter || null, - todo_state_filter: options.todoStateFilter || null, - current_user: options.showCurrentUser || null, - author_id: options.authorId || null, - skip_users: options.skipUsers || null, + ...getAjaxUsersSelectParams(options, AJAX_USERS_SELECT_PARAMS_MAP), }; if (options.issuableType === 'merge_request') { diff --git a/app/assets/javascripts/users_select/utils.js b/app/assets/javascripts/users_select/utils.js new file mode 100644 index 00000000000..b46fd15fb77 --- /dev/null +++ b/app/assets/javascripts/users_select/utils.js @@ -0,0 +1,27 @@ +/** + * Get options from data attributes on passed `$select`. + * @param {jQuery} $select + * @param {Object} optionsMap e.g. { optionKeyName: 'dataAttributeName' } + */ +export const getAjaxUsersSelectOptions = ($select, optionsMap) => { + return Object.keys(optionsMap).reduce((accumulator, optionKey) => { + const dataKey = optionsMap[optionKey]; + accumulator[optionKey] = $select.data(dataKey); + + return accumulator; + }, {}); +}; + +/** + * Get query parameters used for users request from passed `options` parameter + * @param {Object} options e.g. { currentUserId: 1, fooBar: 'baz' } + * @param {Object} paramsMap e.g. { user_id: 'currentUserId', foo_bar: 'fooBar' } + */ +export const getAjaxUsersSelectParams = (options, paramsMap) => { + return Object.keys(paramsMap).reduce((accumulator, paramKey) => { + const optionKey = paramsMap[paramKey]; + accumulator[paramKey] = options[optionKey] || null; + + return accumulator; + }, {}); +}; diff --git a/app/graphql/types/snippets/blob_type.rb b/app/graphql/types/snippets/blob_type.rb index feff5d20874..e4300dcb367 100644 --- a/app/graphql/types/snippets/blob_type.rb +++ b/app/graphql/types/snippets/blob_type.rb @@ -14,6 +14,7 @@ module Types field :plain_data, GraphQL::STRING_TYPE, description: 'Blob plain highlighted data', + calls_gitaly: true, null: true field :raw_path, GraphQL::STRING_TYPE, diff --git a/app/graphql/types/snippets/blob_viewer_type.rb b/app/graphql/types/snippets/blob_viewer_type.rb index 3e653576d07..50d0b0522d6 100644 --- a/app/graphql/types/snippets/blob_viewer_type.rb +++ b/app/graphql/types/snippets/blob_viewer_type.rb @@ -17,12 +17,14 @@ module Types field :collapsed, GraphQL::BOOLEAN_TYPE, description: 'Shows whether the blob should be displayed collapsed', method: :collapsed?, - null: false + null: false, + resolve: -> (viewer, _args, _ctx) { !!viewer&.collapsed? } field :too_large, GraphQL::BOOLEAN_TYPE, description: 'Shows whether the blob too large to be displayed', method: :too_large?, - null: false + null: false, + resolve: -> (viewer, _args, _ctx) { !!viewer&.too_large? } field :render_error, GraphQL::STRING_TYPE, description: 'Error rendering the blob content', diff --git a/changelogs/unreleased/214678-add-an-api-for-flipper-percentage-of-actors-rollout.yml b/changelogs/unreleased/214678-add-an-api-for-flipper-percentage-of-actors-rollout.yml new file mode 100644 index 00000000000..e6c47a67beb --- /dev/null +++ b/changelogs/unreleased/214678-add-an-api-for-flipper-percentage-of-actors-rollout.yml @@ -0,0 +1,5 @@ +--- +title: Add percentage of actors feature flag rollout +merge_request: 29698 +author: +type: added diff --git a/changelogs/unreleased/216345-add-project-group-to-group-list-when-specified-as-owner.yml b/changelogs/unreleased/216345-add-project-group-to-group-list-when-specified-as-owner.yml new file mode 100644 index 00000000000..80c2c555f51 --- /dev/null +++ b/changelogs/unreleased/216345-add-project-group-to-group-list-when-specified-as-owner.yml @@ -0,0 +1,5 @@ +--- +title: Add a Project's group to list of groups when parsing for codeowner entries +merge_request: 30934 +author: +type: fixed diff --git a/changelogs/unreleased/fj-fix-bug-snippet-blob-viewer-graphql.yml b/changelogs/unreleased/fj-fix-bug-snippet-blob-viewer-graphql.yml new file mode 100644 index 00000000000..ab6285d3cf6 --- /dev/null +++ b/changelogs/unreleased/fj-fix-bug-snippet-blob-viewer-graphql.yml @@ -0,0 +1,5 @@ +--- +title: Fix bug in Snippet BlobViewer GraphQL definition +merge_request: 30927 +author: +type: fixed diff --git a/changelogs/unreleased/move_daily_users_statistics_cronjob_to_ce.yml b/changelogs/unreleased/move_daily_users_statistics_cronjob_to_ce.yml new file mode 100644 index 00000000000..58c3130429d --- /dev/null +++ b/changelogs/unreleased/move_daily_users_statistics_cronjob_to_ce.yml @@ -0,0 +1,5 @@ +--- +title: Move daily create users statistics cronjob to CE +merge_request: 30843 +author: +type: fixed diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 3ebe9476ace..a4461e07861 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -490,6 +490,9 @@ Settings.cron_jobs['container_expiration_policy_worker']['job_class'] = 'Contain Settings.cron_jobs['x509_issuer_crl_check_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['x509_issuer_crl_check_worker']['cron'] ||= '30 1 * * *' Settings.cron_jobs['x509_issuer_crl_check_worker']['job_class'] = 'X509IssuerCrlCheckWorker' +Settings.cron_jobs['users_create_statistics_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['users_create_statistics_worker']['cron'] ||= '2 15 * * *' +Settings.cron_jobs['users_create_statistics_worker']['job_class'] = 'Users::CreateStatisticsWorker' Gitlab.ee do Settings.cron_jobs['adjourned_group_deletion_worker'] ||= Settingslogic.new({}) @@ -552,9 +555,6 @@ Gitlab.ee do Settings.cron_jobs['sync_seat_link_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['sync_seat_link_worker']['cron'] ||= "#{rand(60)} 0 * * *" Settings.cron_jobs['sync_seat_link_worker']['job_class'] = 'SyncSeatLinkWorker' - Settings.cron_jobs['users_create_statistics_worker'] ||= Settingslogic.new({}) - Settings.cron_jobs['users_create_statistics_worker']['cron'] ||= '2 15 * * *' - Settings.cron_jobs['users_create_statistics_worker']['job_class'] = 'Users::CreateStatisticsWorker' end # diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index 0e088cb26b8..995cbf9ab62 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -90,6 +90,7 @@ dotenv downvoted downvotes Dpl +Dreamweaver Elasticsearch enablement enqueued @@ -121,6 +122,8 @@ Gzip hardcode hardcoded hardcodes +heatmap +heatmaps Helm Heroku Herokuish @@ -133,6 +136,7 @@ hotfixes hotfixing http https +idempotence Ingress initializer initializers @@ -177,6 +181,7 @@ mergeable Microsoft middleware middlewares +Minikube MinIO mitmproxy misconfigure @@ -242,6 +247,7 @@ rebase rebased rebases rebasing +Redcarpet Redis Redmine reCAPTCHA @@ -283,6 +289,7 @@ Sentry serverless Sidekiq sharding +shfmt Shibboleth sanitization serializer @@ -296,9 +303,13 @@ Splunk SSH storable strace +strikethrough +strikethroughs subpath subfolder subfolders +subgraph +subgraphs sublicense sublicensed sublicenses @@ -340,6 +351,7 @@ unchecking unchecks uncomment uncommented +uncommenting unencode unencoded unencoder @@ -358,12 +370,14 @@ unoptimize unoptimized unoptimizes unoptimizing +unprioritized unprotect unprotects unprotected unpublish unpublished unpublishes +unpublishing unreferenced unresolve unresolved @@ -373,6 +387,9 @@ unstage unstaged unstages unstaging +unstash +unstashed +unstashing untarred untracked untrusted diff --git a/doc/README.md b/doc/README.md index 44a3b594ab3..4cf20e48492 100644 --- a/doc/README.md +++ b/doc/README.md @@ -23,10 +23,11 @@ No matter how you use GitLab, we have documentation for you. | Essential Documentation | Essential Documentation | |:-------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------| | [**User Documentation**](user/index.md)<br/>Discover features and concepts for GitLab users. | [**Administrator documentation**](administration/index.md)<br/>Everything GitLab self-managed administrators need to know. | -| [**Contributing to GitLab**](#contributing-to-gitlab)<br/>At GitLab, everyone can contribute! | [**New to Git and GitLab?**](#new-to-git-and-gitlab)<br/>We have the resources to get you started. | +| [**Contributing to GitLab**](#contributing-to-gitlab)<br/>At GitLab, everyone can contribute! | [**New to Git and GitLab?**](#new-to-git-and-gitlab)<br/>We have the resources to get you started. | | [**Building an integration with GitLab?**](#building-an-integration-with-gitlab)<br/>Consult our automation and integration documentation. | [**Coming to GitLab from another platform?**](#coming-to-gitlab-from-another-platform)<br/>Consult our handy guides. | | [**Install GitLab**](https://about.gitlab.com/install/)<br/>Installation options for different platforms. | [**Customers**](subscriptions/index.md)<br/>Information for new and existing customers. | -| [**Update GitLab**](update/README.md)<br/>Update your GitLab self-managed instance to the latest version. | [**GitLab Releases**](https://about.gitlab.com/releases/)<br/>What's new in GitLab. | +| [**Update GitLab**](update/README.md)<br/>Update your GitLab self-managed instance to the latest version. | [**Reference Architectures**](administration/reference_architectures/index.md)<br/>GitLab's reference architectures | +| [**GitLab Releases**](https://about.gitlab.com/releases/)<br/>What's new in GitLab. | | ## Popular Documentation diff --git a/doc/administration/availability/index.md b/doc/administration/availability/index.md index d567a21aa16..748373c8941 100644 --- a/doc/administration/availability/index.md +++ b/doc/administration/availability/index.md @@ -1,13 +1,5 @@ --- -type: reference, concepts +redirect_to: ../reference_architectures/index.md --- -# Availability - -GitLab offers a number of options to manage availability and resiliency. Below are the options to consider with trade-offs. - -| Event | GitLab Feature | Recovery Point Objective (RPO) | Recovery Time Objective (RTO) | Cost | -| ----- | -------------- | --- | --- | ---- | -| Availability Zone failure | "GitLab HA" | No loss | No loss | 2x Git storage, multiple nodes balanced across AZ's | -| Region failure | [GitLab Geo Disaster Recovery](../geo/disaster_recovery/index.md) | 5-10 minutes | 30 minutes | 2x primary cost | -| All failures | Backup/Restore | Last backup | Hours to Days | Cost of storing the backups | +This document was moved to [another location](../reference_architectures/index.md). diff --git a/doc/administration/file_hooks.md b/doc/administration/file_hooks.md index 043264519a8..506d0f6a70b 100644 --- a/doc/administration/file_hooks.md +++ b/doc/administration/file_hooks.md @@ -35,7 +35,7 @@ Follow the steps below to set up a custom hook: `/home/git/gitlab/file_hooks/`. For Omnibus installs the path is usually `/opt/gitlab/embedded/service/gitlab-rails/file_hooks`. - For [highly available](availability/index.md) configurations, your hook file should exist on each + For [highly available](reference_architectures/index.md) configurations, your hook file should exist on each application server. 1. Inside the `file_hooks` directory, create a file with a name of your choice, diff --git a/doc/administration/geo/replication/external_database.md b/doc/administration/geo/replication/external_database.md index e305718580e..c26b9ab9c97 100644 --- a/doc/administration/geo/replication/external_database.md +++ b/doc/administration/geo/replication/external_database.md @@ -143,7 +143,7 @@ To configure the connection to the external read-replica database and enable Log database to keep track of replication status and automatically recover from potential replication issues. Omnibus automatically configures a tracking database when `roles ['geo_secondary_role']` is set. For high availability, -refer to [Geo High Availability](../../availability/index.md). +refer to [Geo High Availability](../../reference_architectures/index.md). If you want to run this database external to Omnibus, please follow the instructions below. The tracking database requires an [FDW](https://www.postgresql.org/docs/9.6/postgres-fdw.html) diff --git a/doc/administration/geo/replication/high_availability.md b/doc/administration/geo/replication/high_availability.md index 5099e73d5e8..6f5ab9fdfb2 100644 --- a/doc/administration/geo/replication/high_availability.md +++ b/doc/administration/geo/replication/high_availability.md @@ -47,12 +47,12 @@ It is possible to use cloud hosted services for PostgreSQL and Redis, but this i ## Prerequisites: Two working GitLab HA clusters One cluster will serve as the **primary** node. Use the -[GitLab HA documentation](../../availability/index.md) to set this up. If +[GitLab HA documentation](../../reference_architectures/index.md) to set this up. If you already have a working GitLab instance that is in-use, it can be used as a **primary**. The second cluster will serve as the **secondary** node. Again, use the -[GitLab HA documentation](../../availability/index.md) to set this up. +[GitLab HA documentation](../../reference_architectures/index.md) to set this up. It's a good idea to log in and test it, however, note that its data will be wiped out as part of the process of replicating from the **primary**. @@ -371,7 +371,7 @@ more information. The minimal reference architecture diagram above shows all application services running together on the same machines. However, for high availability we -[strongly recommend running all services separately](../../availability/index.md). +[strongly recommend running all services separately](../../reference_architectures/index.md). For example, a Sidekiq server could be configured similarly to the frontend application servers above, with some changes to run only the `sidekiq` service: diff --git a/doc/administration/geo/replication/index.md b/doc/administration/geo/replication/index.md index 5a9befc2232..728e96cb605 100644 --- a/doc/administration/geo/replication/index.md +++ b/doc/administration/geo/replication/index.md @@ -2,7 +2,7 @@ > - Introduced in GitLab Enterprise Edition 8.9. > - Using Geo in combination with -> [High Availability](../../availability/index.md) +> [High Availability](../../reference_architectures/index.md) > is considered **Generally Available** (GA) in > [GitLab Premium](https://about.gitlab.com/pricing/) 10.4. diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index ac337ed8da8..50a8255a792 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -460,12 +460,12 @@ documentation](index.md#3-gitaly-server-configuration). 1. Configure the storage location for Git data by setting `git_data_dirs` in `/etc/gitlab/gitlab.rb`. Each Gitaly node should have a unique storage name - (eg `gitaly-1`). + (such as `gitaly-1`). Instead of configuring `git_data_dirs` uniquely for each Gitaly node, it is often easier to have include the configuration for all Gitaly nodes on every Gitaly node. This is supported because the Praefect `virtual_storages` - configuration maps each storage name (eg `gitaly-1`) to a specific node, and + configuration maps each storage name (such as `gitaly-1`) to a specific node, and requests are routed accordingly. This means every Gitaly node in your fleet can share the same configuration. @@ -573,7 +573,7 @@ Particular attention should be shown to: }) ``` -1. Allow Gitaly to listen on a tcp port by editing +1. Allow Gitaly to listen on a TCP port by editing `/etc/gitlab/gitlab.rb` ```ruby @@ -742,13 +742,13 @@ strategy in the future. ## Identifying Impact of a Primary Node Failure -When a primary Gitaly node fails, there is a chance of dataloss. Dataloss can occur if there were outstanding replication jobs the secondaries did not manage to process before the failure. The Praefect `dataloss` subcommand helps identify these cases by counting the number of dead replication jobs for each repository within a given timeframe. +When a primary Gitaly node fails, there is a chance of data loss. Data loss can occur if there were outstanding replication jobs the secondaries did not manage to process before the failure. The Praefect `dataloss` sub-command helps identify these cases by counting the number of dead replication jobs for each repository within a given time frame. ```shell sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss -from <rfc3339-time> -to <rfc3339-time> ``` -If the timeframe is not specified, dead replication jobs from the last six hours are counted: +If the time frame is not specified, dead replication jobs from the last six hours are counted: ```shell sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss @@ -759,7 +759,7 @@ example/repository-2: 4 jobs example/repository-3: 2 jobs ``` -To specify a timeframe in UTC, run: +To specify a time frame in UTC, run: ```shell sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dataloss -from 2020-01-02T00:00:00+00:00 -to 2020-01-02T00:02:00+00:00 @@ -779,7 +779,7 @@ When a Praefect backend node fails and is no longer able to replicate changes, the backend node will start to drift from the primary. If that node eventually recovers, it will need to be reconciled with the current primary. The primary node is considered the single source of truth for the -state of a shard. The Praefect `reconcile` subcommand allows for the manual +state of a shard. The Praefect `reconcile` sub-command allows for the manual reconciliation between a backend node and the current primary. Run the following command on the Praefect server after all placeholders diff --git a/doc/administration/high_availability/README.md b/doc/administration/high_availability/README.md index c84456e17a8..d36b029cbb3 100644 --- a/doc/administration/high_availability/README.md +++ b/doc/administration/high_availability/README.md @@ -1,10 +1,7 @@ --- -type: reference, concepts +redirect_to: ../reference_architectures/index.md --- -The page have been deprecated, please see: +# Reference Architectures -1. [Availability page](../availability/index.md) -1. [Scaling page](../scaling/index.md) -1. [Docs page for high availability](./gitlab.md) -1. [High availability solutions page](https://about.gitlab.com/solutions/high-availability/) +This document was moved to [another location](../reference_architectures/index.md). diff --git a/doc/administration/high_availability/database.md b/doc/administration/high_availability/database.md index 32af4574daa..4e62facdd2c 100644 --- a/doc/administration/high_availability/database.md +++ b/doc/administration/high_availability/database.md @@ -24,7 +24,7 @@ If you use a cloud-managed service, or provide your own PostgreSQL: ## PostgreSQL in a Scaled and Highly Available Environment -This section is relevant for [Scalable and Highly Available Setups](../scaling/index.md). +This section is relevant for [Scalable and Highly Available Setups](../reference_architectures/index.md). ### Provide your own PostgreSQL instance **(CORE ONLY)** diff --git a/doc/administration/high_availability/gitaly.md b/doc/administration/high_availability/gitaly.md index 0b5bf8d0c4c..f1ebc730e5b 100644 --- a/doc/administration/high_availability/gitaly.md +++ b/doc/administration/high_availability/gitaly.md @@ -11,7 +11,7 @@ should consider using Gitaly on a separate node. See the [Gitaly HA Epic](https://gitlab.com/groups/gitlab-org/-/epics/289) to track plans and progress toward high availability support. -This document is relevant for [Scalable and Highly Available Setups](../scaling/index.md). +This document is relevant for [scalable and highly available setups](../reference_architectures/index.md). ## Running Gitaly on its own server @@ -19,7 +19,7 @@ See [Running Gitaly on its own server](../gitaly/index.md#running-gitaly-on-its- in Gitaly documentation. Continue configuration of other components by going back to the -[Scaling](../scaling/index.md#components-provided-by-omnibus-gitlab) page. +[reference architecture](../reference_architectures/index.md#configure-gitlab-to-scale) page. ## Enable Monitoring diff --git a/doc/administration/high_availability/gitlab.md b/doc/administration/high_availability/gitlab.md index cb5910decaa..c2860f5452b 100644 --- a/doc/administration/high_availability/gitlab.md +++ b/doc/administration/high_availability/gitlab.md @@ -2,7 +2,9 @@ type: reference --- -# Configuring GitLab for Scaling and High Availability +# Configuring GitLab application (Rails) + +This section describes how to configure the GitLab application (Rails) component. NOTE: **Note:** There is some additional configuration near the bottom for additional GitLab application servers. It's important to read and understand diff --git a/doc/administration/high_availability/monitoring_node.md b/doc/administration/high_availability/monitoring_node.md index 09e9620064a..3e6fdbf9b1c 100644 --- a/doc/administration/high_availability/monitoring_node.md +++ b/doc/administration/high_availability/monitoring_node.md @@ -11,7 +11,7 @@ You can configure a Prometheus node to monitor GitLab. ## Standalone Monitoring node using Omnibus GitLab The Omnibus GitLab package can be used to configure a standalone Monitoring node running [Prometheus](../monitoring/prometheus/index.md) and [Grafana](../monitoring/performance/grafana_configuration.md). -The monitoring node is not highly available. See [Scaling and High Availability](../scaling/index.md) +The monitoring node is not highly available. See [Scaling and High Availability](../reference_architectures/index.md) for an overview of GitLab scaling and high availability options. The steps below are the minimum necessary to configure a Monitoring node running Prometheus and Grafana with diff --git a/doc/administration/high_availability/redis.md b/doc/administration/high_availability/redis.md index a880ad77271..3af175b8539 100644 --- a/doc/administration/high_availability/redis.md +++ b/doc/administration/high_availability/redis.md @@ -27,7 +27,7 @@ These will be necessary when configuring the GitLab application servers later. ## Redis in a Scaled and Highly Available Environment -This section is relevant for [Scalable and Highly Available Setups](../scaling/index.md). +This section is relevant for [scalable and highly available setups](../reference_architectures/index.md). ### Provide your own Redis instance **(CORE ONLY)** @@ -43,8 +43,8 @@ In this configuration Redis is not highly available, and represents a single point of failure. However, in a scaled environment the objective is to allow the environment to handle more users or to increase throughput. Redis itself is generally stable and can handle many requests so it is an acceptable -trade off to have only a single instance. See [High Availability](../availability/index.md) -for an overview of GitLab scaling and high availability options. +trade off to have only a single instance. See the [reference architectures](../reference_architectures/index.md) +page for an overview of GitLab scaling and high availability options. The steps below are the minimum necessary to configure a Redis server with Omnibus: @@ -89,7 +89,7 @@ Advanced configuration options are supported and can be added if needed. Continue configuration of other components by going back to the -[Scaling](../scaling/index.md#components-provided-by-omnibus-gitlab) page. +[reference architectures](../reference_architectures/index.md#configure-gitlab-to-scale) page. ### High Availability with Omnibus GitLab **(PREMIUM ONLY)** diff --git a/doc/administration/index.md b/doc/administration/index.md index 74a4d592d6f..fdae970a716 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -34,7 +34,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Install](../install/README.md): Requirements, directory structures, and installation methods. - [Database load balancing](database_load_balancing.md): Distribute database queries among multiple database servers. **(STARTER ONLY)** - [Omnibus support for log forwarding](https://docs.gitlab.com/omnibus/settings/logs.html#udp-log-shipping-gitlab-enterprise-edition-only) **(STARTER ONLY)** -- [High Availability](availability/index.md): Configure multiple servers for scaling or high availability. +- [High Availability](reference_architectures/index.md): Configure multiple servers for scaling or high availability. - [Installing GitLab HA on Amazon Web Services (AWS)](../install/aws/index.md): Set up GitLab High Availability on Amazon AWS. - [Geo](geo/replication/index.md): Replicate your GitLab instance to other geographic locations as a read-only fully operational version. **(PREMIUM ONLY)** - [Disaster Recovery](geo/disaster_recovery/index.md): Quickly fail-over to a different site with minimal effort in a disaster situation. **(PREMIUM ONLY)** diff --git a/doc/administration/logs.md b/doc/administration/logs.md index 456e0bbc292..fd9556278a1 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -428,7 +428,7 @@ installations from source. This file contains logging information about jobs before they are start being processed by Sidekiq, for example before being enqueued. -This logfile follows the same structure as +This log file follows the same structure as [`sidekiq.log`](#sidekiqlog), so it will be structured as JSON if you've configured this for Sidekiq as mentioned above. @@ -523,7 +523,7 @@ User clone/fetch activity using SSH transport appears in this log as `executing ## Gitaly Logs -This file lives in `/var/log/gitlab/gitaly/current` and is produced by [runit](http://smarden.org/runit/). `runit` is packaged with Omnibus and a brief explanation of its purpose is available [in the omnibus documentation](https://docs.gitlab.com/omnibus/architecture/#runit). [Log files are rotated](http://smarden.org/runit/svlogd.8.html), renamed in unix timestamp format and `gzip`-compressed (e.g. `@1584057562.s`). +This file lives in `/var/log/gitlab/gitaly/current` and is produced by [runit](http://smarden.org/runit/). `runit` is packaged with Omnibus and a brief explanation of its purpose is available [in the omnibus documentation](https://docs.gitlab.com/omnibus/architecture/#runit). [Log files are rotated](http://smarden.org/runit/svlogd.8.html), renamed in Unix timestamp format and `gzip`-compressed (e.g. `@1584057562.s`). ### `grpc.log` @@ -761,7 +761,7 @@ For Omnibus installations, NGINX logs reside in: - `/var/log/gitlab/nginx/gitlab_pages_access.log` contains a log of requests made to Pages static sites. - `/var/log/gitlab/nginx/gitlab_pages_error.log` contains a log of NGINX errors for Pages static sites. - `/var/log/gitlab/nginx/gitlab_registry_access.log` contains a log of requests made to the Container Registry. -- `/var/log/gitlab/nginx/gitlab_registry_error.log` contains a log of NGINX errors for the Container Regsitry. +- `/var/log/gitlab/nginx/gitlab_registry_error.log` contains a log of NGINX errors for the Container Registry. - `/var/log/gitlab/nginx/gitlab_mattermost_access.log` contains a log of requests made to Mattermost. - `/var/log/gitlab/nginx/gitlab_mattermost_error.log` contains a log of NGINX errors for Mattermost. diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index aec5f7f5566..0e03051ac24 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -37,7 +37,7 @@ For configuring GitLab to use Object Storage refer to the following guides: ### Other alternatives to filesystem storage -If you're working to [scale out](scaling/index.md) your GitLab implementation, +If you're working to [scale out](reference_architectures/index.md) your GitLab implementation, or add [fault tolerance and redundancy](high_availability/README.md) you may be looking at removing dependencies on block or network filesystems. See the following guides and @@ -77,7 +77,7 @@ with the Fog library that GitLab uses. Symptoms include: ### GitLab Pages requires NFS -If you're working to add more GitLab servers for [scaling or fault tolerance](scaling/index.md) +If you're working to add more GitLab servers for [scaling or fault tolerance](reference_architectures/index.md) and one of your requirements is [GitLab Pages](../user/project/pages/index.md) this currently requires NFS. There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/issues/196) to remove this dependency. In the future, GitLab Pages may use diff --git a/doc/administration/reference_architectures/img/reference-architectures.png b/doc/administration/reference_architectures/img/reference-architectures.png Binary files differnew file mode 100644 index 00000000000..e15609e78e1 --- /dev/null +++ b/doc/administration/reference_architectures/img/reference-architectures.png diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md new file mode 100644 index 00000000000..88a4f2c97a6 --- /dev/null +++ b/doc/administration/reference_architectures/index.md @@ -0,0 +1,334 @@ +--- +type: reference, concepts +--- +# Reference architectures + +<!-- TBD to be reviewed by Eric --> + +You can set up GitLab on a single server or scale it up to serve many users. +This page details the recommended Reference Architectures that were built and verified by GitLab's Quality and Support teams. + +Below is a chart representing each architecture tier and the number of users they can handle. As your number of users grow with time, it’s recommended that you scale GitLab accordingly. + + +<!-- Internal link: https://docs.google.com/spreadsheets/d/1obYP4fLKkVVDOljaI3-ozhmCiPtEeMblbBKkf2OADKs/edit#gid=1403207183 --> + +Testing on these reference architectures were performed with [GitLab's Performance Tool](https://gitlab.com/gitlab-org/quality/performance) +at specific coded workloads, and the throughputs used for testing were calculated based on sample customer data. +After selecting the reference architecture that matches your scale, refer to +[Configure GitLab to Scale](#configure-gitlab-to-scale) to see the components +involved, and how to configure them. + +Each endpoint type is tested with the following number of requests per second (RPS) per 1000 users: + +- API: 20 RPS +- Web: 2 RPS +- Git: 2 RPS + +For GitLab instances with less than 2,000 users, it's recommended that you use the [default setup](#automated-backups-core-only) +by [installing GitLab](../../install/README.md) on a single machine to minimize maintenance and resource costs. + +If your organization has more than 2,000 users, the recommendation is to scale GitLab's components to multiple +machine nodes. The machine nodes are grouped by component(s). The addition of these +nodes increases the performance and scalability of to your GitLab instance. +As long as there is at least one of each component online and capable of handling +the instance's usage load, your team's productivity will not be interrupted. +Scaling GitLab in this manner also enables you to perform [zero-downtime updates](https://docs.gitlab.com/omnibus/update/#zero-downtime-updates). + +When scaling GitLab, there are several factors to consider: + +- Multiple application nodes to handle frontend traffic. +- A load balancer is added in front to distribute traffic across the application nodes. +- The application nodes connects to a shared file server and PostgreSQL and Redis services on the backend. + +NOTE: **Note:** Depending on your workflow, the following recommended +reference architectures may need to be adapted accordingly. Your workload +is influenced by factors including how active your users are, +how much automation you use, mirroring, and repository/change size. Additionally the +displayed memory values are provided by [GCP machine types](https://cloud.google.com/compute/docs/machine-types). +For different cloud vendors, attempt to select options that best match the provided architecture. + +## Up to 1,000 users + +From 1 to 1,000 users, a [single-node setup with frequent backups](#automated-backups-core-only) is adequate. + +| Users | Configuration([8](#footnotes)) | GCP type | AWS type([9](#footnotes)) | +|-------|--------------------------------|---------------|---------------------------| +| 100 | 2 vCPU, 7.2GB Memory | n1-standard-2 | c5.2xlarge | +| 500 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | +| 1000 | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | + +This solution is appropriate for many teams that have a single server at their disposal. With automatic backup of the GitLab repositories, configuration, and the database, this can be an optimal solution if you don't have strict availability requirements. + +You can also optionally configure GitLab to use an [external PostgreSQL service](../external_database.md) or an [external object storage service](../high_availability/object_storage.md) for added performance and reliability at a relatively low complexity cost. + +<!-- +## Up to 2,000 users + +For up to 2,000 users, defining the reference architecture is [being worked on](https://gitlab.com/gitlab-org/quality/performance/-/issues/223). +--> + +## Up to 3,000 users + +NOTE: **Note:** The 3,000-user reference architecture documented below is +designed to help your organization achieve a highly-available GitLab deployment. +If you do not have the expertise or need to maintain a highly-available +environment, you can have a simpler and less costly-to-operate environment by +deploying two or more GitLab Rails servers, external load balancing, an NFS +server, a PostgreSQL server and a Redis server. A reference architecture with +this alternative in mind is [being worked on](https://gitlab.com/gitlab-org/quality/performance/-/issues/223). + +> - **Supported users (approximate):** 3,000 +> - **Test RPS rates:** API: 40 RPS, Web: 4 RPS, Git: 4 RPS + +| Service | Nodes | Configuration ([8](#footnotes)) | GCP type | AWS type ([9](#footnotes)) | +|--------------------------------------------------------------|-------|---------------------------------|---------------|----------------------------| +| GitLab Rails ([1](#footnotes)) | 3 | 8 vCPU, 7.2GB Memory | n1-highcpu-8 | c5.2xlarge | +| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | +| Redis ([3](#footnotes)) | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | +| Consul + Sentinel ([3](#footnotes)) | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | +| Cloud Object Storage ([4](#footnotes)) | - | - | - | - | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | +| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | + +## Up to 5,000 users + +> - **Supported users (approximate):** 5,000 +> - **Test RPS rates:** API: 100 RPS, Web: 10 RPS, Git: 10 RPS + +| Service | Nodes | Configuration ([8](#footnotes)) | GCP type | AWS type ([9](#footnotes)) | +|--------------------------------------------------------------|-------|---------------------------------|---------------|----------------------------| +| GitLab Rails ([1](#footnotes)) | 3 | 16 vCPU, 14.4GB Memory | n1-highcpu-16 | c5.4xlarge | +| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | +| Redis ([3](#footnotes)) | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | +| Consul + Sentinel ([3](#footnotes)) | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | +| Cloud Object Storage ([4](#footnotes)) | - | - | - | - | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | +| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | + +## Up to 10,000 users + +> - **Supported users (approximate):** 10,000 +> - **Test RPS rates:** API: 200 RPS, Web: 20 RPS, Git: 20 RPS + +| Service | Nodes | GCP Configuration ([8](#footnotes)) | GCP type | AWS type ([9](#footnotes)) | +|--------------------------------------------------------------|-------|-------------------------------------|----------------|----------------------------| +| GitLab Rails ([1](#footnotes)) | 3 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | +| PostgreSQL | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 16 vCPU, 60GB Memory | n1-standard-16 | m5.4xlarge | +| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | +| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | +| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | +| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | +| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | +| Cloud Object Storage ([4](#footnotes)) | - | - | - | - | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | +| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | +| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | + +## Up to 25,000 users + +> - **Supported users (approximate):** 25,000 +> - **Test RPS rates:** API: 500 RPS, Web: 50 RPS, Git: 50 RPS + +| Service | Nodes | Configuration ([8](#footnotes)) | GCP type | AWS type ([9](#footnotes)) | +|--------------------------------------------------------------|-------|---------------------------------|----------------|----------------------------| +| GitLab Rails ([1](#footnotes)) | 5 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | +| PostgreSQL | 3 | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 32 vCPU, 120GB Memory | n1-standard-32 | m5.8xlarge | +| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | +| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | +| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | +| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | +| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | +| Cloud Object Storage ([4](#footnotes)) | - | - | - | - | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | +| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | +| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Internal load balancing node ([6](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | + +## Up to 50,000 users + +> - **Supported users (approximate):** 50,000 +> - **Test RPS rates:** API: 1000 RPS, Web: 100 RPS, Git: 100 RPS + +| Service | Nodes | Configuration ([8](#footnotes)) | GCP type | AWS type ([9](#footnotes)) | +|--------------------------------------------------------------|-------|---------------------------------|----------------|----------------------------| +| GitLab Rails ([1](#footnotes)) | 12 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | +| PostgreSQL | 3 | 16 vCPU, 60GB Memory | n1-standard-16 | m5.4xlarge | +| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 64 vCPU, 240GB Memory | n1-standard-64 | m5.16xlarge | +| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | +| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | +| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | +| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | +| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | +| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | +| Cloud Object Storage ([4](#footnotes)) | - | - | - | - | +| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | +| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | +| Internal load balancing node ([6](#footnotes)) | 1 | 8 vCPU, 7.2GB Memory | n1-highcpu-8 | c5.2xlarge | + +## Availability complexity + +GitLab comes with the following availability complexity for your use, listed from +least to most complex: + +1. [Automated backups](#automated-backups-core-only) +1. [Traffic Load Balancer](#Traffic-load-balancer-starter-only) +1. [Automated database failover](#automated-database-failover-premium-only) +1. [Instance level replication with GitLab Geo](#instance-level-replication-with-gitlab-geo-premium-only) + +As you get started implementing HA, begin with a single server and then do +backups. Only after completing the first server should you proceed to the next. + +Also, not implementing HA for GitLab doesn't necessarily mean that you'll have +more downtime. Depending on your needs and experience level, non-HA servers can +have more actual perceived uptime for your users. + +### Automated backups **(CORE ONLY)** + +> - Level of complexity: **Low** +> - Required domain knowledge: PostgreSQL, GitLab configurations, Git +> - Supported tiers: [GitLab Core, Starter, Premium, and Ultimate](https://about.gitlab.com/pricing/) + +This solution is appropriate for many teams that have the default GitLab installation. +With automatic backups of the GitLab repositories, configuration, and the database, +this can be an optimal solution if you don't have strict availability requirements. +[Automated backups](../../raketasks/backup_restore.md#configuring-cron-to-make-daily-backups) +is the least complex to setup. This provides a point-in-time recovery of a predetermined schedule. + +### Traffic load balancer **(STARTER ONLY)** + +> - Level of complexity: **Medium** +> - Required domain knowledge: HAProxy, shared storage, distributed systems +> - Supported tiers: [GitLab Starter, Premium, and Ultimate](https://about.gitlab.com/pricing/) + +This requires separating out GitLab into multiple application nodes with an added +[load balancer](../high_availability/load_balancer.md). The load balancer will distribute traffic +across GitLab application nodes. Meanwhile, each application node connects to a +shared file server and database systems on the back end. This way, if one of the +application servers fails, the workflow is not interrupted. +[HAProxy](https://www.haproxy.org/) is recommended as the load balancer. + +With this added availability component you have a number of advantages compared +to the default installation: + +- Increase the number of users. +- Enable zero-downtime upgrades. +- Increase availability. + +### Automated database failover **(PREMIUM ONLY)** + +> - Level of complexity: **High** +> - Required domain knowledge: PgBouncer, Repmgr, shared storage, distributed systems +> - Supported tiers: [GitLab Premium and Ultimate](https://about.gitlab.com/pricing/) + +By adding automatic failover for database systems, you can enable higher uptime +with an additional database nodes. This extends the default database with a +cluster management and failover policies. +[PgBouncer](../../development/architecture.md#pgbouncer) in conjunction with +[Repmgr](../high_availability/database.md) is recommended. + +### Instance level replication with GitLab Geo **(PREMIUM ONLY)** + +> - Level of complexity: **Very High** +> - Required domain knowledge: Storage replication +> - Supported tiers: [GitLab Premium and Ultimate](https://about.gitlab.com/pricing/) + +[GitLab Geo](../geo/replication/index.md) allows you to replicate your GitLab +instance to other geographical locations as a read-only fully operational instance +that can also be promoted in case of disaster. + +## Configure GitLab to scale + +The following components are the ones you need to configure in order to scale +GitLab. They are listed in the order you'll typically configure them if they are +required by your [reference architecture](#reference-architectures) of choice. + +Most of them are bundled in the GitLab deb/rpm package (called Omnibus GitLab), +but depending on your system architecture, you may require some components which are +not included in it. If required, those should be configured before +setting up components provided by GitLab. Advice on how to select the right +solution for your organization is provided in the configuration instructions +column. + +| Component | Description | Configuration instructions | Bundled with Omnibus GitLab | +|-----------|-------------|----------------------------| +| Load balancer(s) ([6](#footnotes)) | Handles load balancing, typically when you have multiple GitLab application services nodes | [Load balancer configuration](../high_availability/load_balancer.md) ([6](#footnotes)) | No | +| Object storage service ([4](#footnotes)) | Recommended store for shared data objects | [Cloud Object Storage configuration](../object_storage.md) | No | +| NFS ([5](#footnotes)) ([7](#footnotes)) | Shared disk storage service. Can be used as an alternative for Gitaly or Object Storage. Required for GitLab Pages | [NFS configuration](../high_availability/nfs.md) | No | +| [Consul](../../development/architecture.md#consul) ([3](#footnotes)) | Service discovery and health checks/failover | [Consul HA configuration](../high_availability/consul.md) **(PREMIUM ONLY)** | Yes | +| [PostgreSQL](../../development/architecture.md#postgresql) | Database | [PostgreSQL configuration](https://docs.gitlab.com/omnibus/settings/database.html) | Yes | +| [PgBouncer](../../development/architecture.md#pgbouncer) | Database connection pooler | [PgBouncer configuration](../high_availability/pgbouncer.md#running-pgbouncer-as-part-of-a-non-ha-gitlab-installation) **(PREMIUM ONLY)** | Yes | +| Repmgr | PostgreSQL cluster management and failover | [PostgreSQL and Repmgr configuration](../high_availability/database.md) | Yes | +| [Redis](../../development/architecture.md#redis) ([3](#footnotes)) | Key/value store for fast data lookup and caching | [Redis configuration](../high_availability/redis.md) | Yes | +| Redis Sentinel | High availability for Redis | [Redis Sentinel configuration](../high_availability/redis.md) | Yes | +| [Gitaly](../../development/architecture.md#gitaly) ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | Provides access to Git repositories | [Gitaly configuration](../gitaly/index.md#running-gitaly-on-its-own-server) | Yes | +| [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/background jobs | [Sidekiq configuration](../high_availability/sidekiq.md) | Yes | +| [GitLab application services](../../development/architecture.md#unicorn)([1](#footnotes)) | Unicorn/Puma, Workhorse, GitLab Shell - serves front-end requests (UI, API, Git over HTTP/SSH) | [GitLab app scaling configuration](../high_availability/gitlab.md) | Yes | +| [Prometheus](../../development/architecture.md#prometheus) and [Grafana](../../development/architecture.md#grafana) | GitLab environment monitoring | [Monitoring node for scaling](../high_availability/monitoring_node.md) | Yes | + +## Footnotes + +1. In our architectures we run each GitLab Rails node using the Puma webserver + and have its number of workers set to 90% of available CPUs along with four threads. + +1. Gitaly node requirements are dependent on customer data, specifically the number of + projects and their sizes. We recommend two nodes as an absolute minimum for HA environments + and at least four nodes should be used when supporting 50,000 or more users. + We also recommend that each Gitaly node should store no more than 5TB of data + and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) + set to 20% of available CPUs. Additional nodes should be considered in conjunction + with a review of expected data size and spread based on the recommendations above. + +1. Recommended Redis setup differs depending on the size of the architecture. + For smaller architectures (less than 5,000 users), we suggest one Redis cluster for all + classes and that Redis Sentinel is hosted alongside Consul. + For larger architectures (10,000 users or more) we suggest running a separate + [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + and another for the Queues and Shared State classes respectively. We also recommend + that you run the Redis Sentinel clusters separately for each Redis Cluster. + +1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend a [Cloud Object Storage service](../object_storage.md) + over NFS where possible, due to better performance and availability. + +1. NFS can be used as an alternative for both repository data (replacing Gitaly) and + object storage but this isn't typically recommended for performance reasons. Note however it is required for + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + +1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) + as the load balancer. Although other load balancers with similar feature sets + could also be used, those load balancers have not been validated. + +1. We strongly recommend that any Gitaly or NFS nodes be set up with SSD disks over + HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write + as these components have heavy I/O. These IOPS values are recommended only as a starter + as with time they may be adjusted higher or lower depending on the scale of your + environment's workload. If you're running the environment on a Cloud provider + you may need to refer to their documentation on how configure IOPS correctly. + +1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) + CPU platform on GCP. On different hardware you may find that adjustments, either lower + or higher, are required for your CPU or Node counts accordingly. For more information, a + [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found + [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). + +1. AWS-equivalent configurations are rough suggestions and may change in the + future. They have not yet been tested and validated. diff --git a/doc/administration/scaling/index.md b/doc/administration/scaling/index.md index 23dadca7a9f..748373c8941 100644 --- a/doc/administration/scaling/index.md +++ b/doc/administration/scaling/index.md @@ -1,278 +1,5 @@ --- -type: reference, concepts +redirect_to: ../reference_architectures/index.md --- -# Scaling - -GitLab supports a number of scaling options to ensure that your self-managed -instance is able to scale to meet your organization's needs. - -On this page, we present examples of self-managed instances which demonstrate -how GitLab can be scaled up, scaled out or made highly available. These -examples progress from simple to complex as scaling or highly-available -components are added. - -For detailed insight into how GitLab scales and configures GitLab.com, you can -watch [this 1 hour Q&A](https://www.youtube.com/watch?v=uCU8jdYzpac) -with [John Northrup](https://gitlab.com/northrup), and live questions coming -in from some of our customers. - -## Reference architectures - -GitLab can be set up on a single machine or scaled out to handle large number of users. In this section we'll detail the Reference Architectures that were built and verified by our Quality and Support teams. -Testing was done with our [GitLab Performance Tool](https://gitlab.com/gitlab-org/quality/performance) at specific coded workloads, and the throughputs used for testing were calculated based on sample customer data. - -We test each endpoint type with the following number of requests per second (RPS) per 1000 users: - -- API: 20 RPS -- Web: 2 RPS -- Git: 2 RPS - -For up to 2,000 users we recommend going with a simple setup. Going above 2,000 users, we recommend scaling GitLab components to multiple machine nodes. -The machine nodes are grouped by component(s). The addition of these nodes adds limited fault tolerance to your GitLab instance. -As long as there is at least one of each component online and capable of handling the instance's usage load, your team's productivity will not be interrupted. -The same is true if you are looking to perform [zero-downtime updates](https://docs.gitlab.com/omnibus/update/#zero-downtime-updates). - -When scaling GitLab there's a few factors to consider: - -- Multiple application nodes to handle frontend traffic. -- A load balancer is added in front to distribute traffic across the application nodes. -- The application nodes connects to a shared file server and PostgreSQL and Redis services on the backend. - -References: - -- [Configure your load balancer for GitLab](../high_availability/load_balancer.md) -- [Configure your NFS server to work with GitLab](../high_availability/nfs.md) -- [Configure packaged PostgreSQL server to listen on TCP/IP](https://docs.gitlab.com/omnibus/settings/database.html#configure-packaged-postgresql-server-to-listen-on-tcpip) -- [Setting up a Redis-only server](https://docs.gitlab.com/omnibus/settings/redis.html#setting-up-a-redis-only-server) - -NOTE: **Note:** Note that depending on your workflow the below recommended -reference architectures may need to be adapted accordingly. Your workload -is influenced by factors such as - but not limited to - how active your users are, -how much automation you use, mirroring, and repository/change size. Additionally the -shown memory values are given directly by [GCP machine types](https://cloud.google.com/compute/docs/machine-types). -On different cloud vendors a best effort like for like can be used. - -### Up to 1,000 users - -From 1 to 1,000 users, a single-node [Omnibus](https://docs.gitlab.com/omnibus/) setup with frequent backups is adequate. -Please refer to the [installation documentation](../../install/README.md) and [backup/restore documentation](https://docs.gitlab.com/omnibus/settings/backups.html#backup-and-restore-omnibus-gitlab-configuration). - -| Users | Configuration([8](#footnotes)) | GCP type | AWS type([9](#footnotes)) | -|-------|--------------------------------|---------------|---------------------------| -| 100 | 2 vCPU, 7.2GB Memory | n1-standard-2 | c5.2xlarge | -| 500 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| 1000 | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | - -This solution is appropriate for many teams that have a single server at their disposal. With automatic backup of the GitLab repositories, configuration, and the database, this can be an optimal solution if you don't have strict availability requirements. - -You can also optionally configure GitLab to use an [external PostgreSQL service](../external_database.md) or an [external object storage service](../high_availability/object_storage.md) for added performance and reliability at a relatively low complexity cost. - -### Up to 2,000 users - -For up to 2,000 users, defining the reference architecture is [being worked on](https://gitlab.com/gitlab-org/quality/performance/-/issues/223). - -### Up to 3,000 users - -NOTE: **Note:** The 3,000-user reference architecture documented below is -designed to help your organization achieve a highly-available GitLab deployment. -If you do not have the expertise or need to maintain a highly-available -environment, you can have a simpler and less costly-to-operate environment by -deploying two or more GitLab Rails servers, external load balancing, an NFS -server, a PostgreSQL server and a Redis server. A reference architecture with -this alternative in mind is [being worked on](https://gitlab.com/gitlab-org/quality/performance/-/issues/223). - -- **Supported users (approximate):** 2,000 -- **Test RPS rates:** API: 40 RPS, Web: 4 RPS, Git: 4 RPS -- **Known issues:** [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues) - -| Service | Nodes | Configuration ([8](#footnotes)) | GCP type | AWS type ([9](#footnotes)) | -|--------------------------------------------------------------|-------|---------------------------------|---------------|----------------------------| -| GitLab Rails ([1](#footnotes)) | 3 | 8 vCPU, 7.2GB Memory | n1-highcpu-8 | c5.2xlarge | -| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis ([3](#footnotes)) | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | -| Consul + Sentinel ([3](#footnotes)) | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | -| Cloud Object Storage ([4](#footnotes)) | - | - | - | - | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | - -### Up to 5,000 users - -- **Supported users (approximate):** 5,000 -- **Test RPS rates:** API: 100 RPS, Web: 10 RPS, Git: 10 RPS -- **Known issues:** [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues) - -| Service | Nodes | Configuration ([8](#footnotes)) | GCP type | AWS type ([9](#footnotes)) | -|--------------------------------------------------------------|-------|---------------------------------|---------------|----------------------------| -| GitLab Rails ([1](#footnotes)) | 3 | 16 vCPU, 14.4GB Memory | n1-highcpu-16 | c5.4xlarge | -| PostgreSQL | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | -| Redis ([3](#footnotes)) | 3 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | -| Consul + Sentinel ([3](#footnotes)) | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Sidekiq | 4 | 2 vCPU, 7.5GB Memory | n1-standard-2 | m5.large | -| Cloud Object Storage ([4](#footnotes)) | - | - | - | - | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| Monitoring node | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | - -### Up to 10,000 users - -- **Supported users (approximate):** 10,000 -- **Test RPS rates:** API: 200 RPS, Web: 20 RPS, Git: 20 RPS -- **Known issues:** [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues) - -| Service | Nodes | GCP Configuration ([8](#footnotes)) | GCP type | AWS type ([9](#footnotes)) | -|--------------------------------------------------------------|-------|-------------------------------------|----------------|----------------------------| -| GitLab Rails ([1](#footnotes)) | 3 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | -| PostgreSQL | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 16 vCPU, 60GB Memory | n1-standard-16 | m5.4xlarge | -| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | -| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | -| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Cloud Object Storage ([4](#footnotes)) | - | - | - | - | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Internal load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | - -### Up to 25,000 users - -- **Supported users (approximate):** 25,000 -- **Test RPS rates:** API: 500 RPS, Web: 50 RPS, Git: 50 RPS -- **Known issues:** [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues) - -| Service | Nodes | Configuration ([8](#footnotes)) | GCP type | AWS type ([9](#footnotes)) | -|--------------------------------------------------------------|-------|---------------------------------|----------------|----------------------------| -| GitLab Rails ([1](#footnotes)) | 5 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | -| PostgreSQL | 3 | 8 vCPU, 30GB Memory | n1-standard-8 | m5.2xlarge | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 32 vCPU, 120GB Memory | n1-standard-32 | m5.8xlarge | -| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | -| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | -| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Cloud Object Storage ([4](#footnotes)) | - | - | - | - | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Internal load balancing node ([6](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | - -### Up to 50,000 users - -- **Supported users (approximate):** 50,000 -- **Test RPS rates:** API: 1000 RPS, Web: 100 RPS, Git: 100 RPS -- **Known issues:** [List of known performance issues](https://gitlab.com/gitlab-org/gitlab/issues?label_name%5B%5D=Quality%3Aperformance-issues) - -| Service | Nodes | Configuration ([8](#footnotes)) | GCP type | AWS type ([9](#footnotes)) | -|--------------------------------------------------------------|-------|---------------------------------|----------------|----------------------------| -| GitLab Rails ([1](#footnotes)) | 12 | 32 vCPU, 28.8GB Memory | n1-highcpu-32 | c5.9xlarge | -| PostgreSQL | 3 | 16 vCPU, 60GB Memory | n1-standard-16 | m5.4xlarge | -| PgBouncer | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Gitaly ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | X | 64 vCPU, 240GB Memory | n1-standard-64 | m5.16xlarge | -| Redis ([3](#footnotes)) - Cache | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis ([3](#footnotes)) - Queues / Shared State | 3 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| Redis Sentinel ([3](#footnotes)) - Cache | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | -| Redis Sentinel ([3](#footnotes)) - Queues / Shared State | 3 | 1 vCPU, 1.7GB Memory | g1-small | t2.small | -| Consul | 3 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Sidekiq | 4 | 4 vCPU, 15GB Memory | n1-standard-4 | m5.xlarge | -| NFS Server ([5](#footnotes)) ([7](#footnotes)) | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| Cloud Object Storage ([4](#footnotes)) | - | - | - | - | -| Monitoring node | 1 | 4 vCPU, 3.6GB Memory | n1-highcpu-4 | c5.xlarge | -| External load balancing node ([6](#footnotes)) | 1 | 2 vCPU, 1.8GB Memory | n1-highcpu-2 | c5.large | -| Internal load balancing node ([6](#footnotes)) | 1 | 8 vCPU, 7.2GB Memory | n1-highcpu-8 | c5.2xlarge | - -## Configuring GitLab to scale - -### Components not provided by Omnibus GitLab - -Depending on your system architecture, you may require some components which are -not provided in Omnibus GitLab. If required, these should be configured before -setting up components provided by GitLab. Advice on how to select the right -solution for your organization is provided in the configuration instructions -listed below. - -| Component | Description | Configuration instructions | -|-----------|-------------|----------------------------| -| Load balancer(s) ([6](#footnotes)) | Handles load balancing, typically when you have multiple GitLab application services nodes | [Load balancer configuration](../high_availability/load_balancer.md) ([6](#footnotes)) | -| Object storage service ([4](#footnotes)) | Recommended store for shared data objects | [Cloud Object Storage configuration](../object_storage.md) | -| NFS ([5](#footnotes)) ([7](#footnotes)) | Shared disk storage service. Can be used as an alternative for Gitaly or Object Storage. Required for GitLab Pages | [NFS configuration](../high_availability/nfs.md) | - -### Components provided by Omnibus GitLab - -The following components are provided by Omnibus GitLab. They are listed in the -order you'll typically configure them if they are required by your -[reference architecture](#reference-architectures) of choice. - -| Component | Description | Configuration instructions | -|-----------|-------------|----------------------------| -| [Consul](../../development/architecture.md#consul) ([3](#footnotes)) | Service discovery and health checks/failover | [Consul HA configuration](../high_availability/consul.md) **(PREMIUM ONLY)** | -| [PostgreSQL](../../development/architecture.md#postgresql) | Database | [PostgreSQL configuration](https://docs.gitlab.com/omnibus/settings/database.html) | -| [PgBouncer](../../development/architecture.md#pgbouncer) | Database connection pooler | [PgBouncer configuration](../high_availability/pgbouncer.md#running-pgbouncer-as-part-of-a-non-ha-gitlab-installation) **(PREMIUM ONLY)** | -| Repmgr | PostgreSQL cluster management and failover | [PostgreSQL and Repmgr configuration](../high_availability/database.md) | -| [Redis](../../development/architecture.md#redis) ([3](#footnotes)) | Key/value store for fast data lookup and caching | [Redis configuration](../high_availability/redis.md) | -| Redis Sentinel | High availability for Redis | [Redis Sentinel configuration](../high_availability/redis.md) | -| [Gitaly](../../development/architecture.md#gitaly) ([2](#footnotes)) ([5](#footnotes)) ([7](#footnotes)) | Provides access to Git repositories | [Gitaly configuration](../gitaly/index.md#running-gitaly-on-its-own-server) | -| [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/background jobs | [Sidekiq configuration](../high_availability/sidekiq.md) | -| [GitLab application services](../../development/architecture.md#unicorn)([1](#footnotes)) | Unicorn/Puma, Workhorse, GitLab Shell - serves front-end requests (UI, API, Git over HTTP/SSH) | [GitLab app scaling configuration](../high_availability/gitlab.md) | -| [Prometheus](../../development/architecture.md#prometheus) and [Grafana](../../development/architecture.md#grafana) | GitLab environment monitoring | [Monitoring node for scaling](../high_availability/monitoring_node.md) | - -## Footnotes - -1. In our architectures we run each GitLab Rails node using the Puma webserver - and have its number of workers set to 90% of available CPUs along with 4 threads. - -1. Gitaly node requirements are dependent on customer data, specifically the number of - projects and their sizes. We recommend 2 nodes as an absolute minimum for HA environments - and at least 4 nodes should be used when supporting 50,000 or more users. - We also recommend that each Gitaly node should store no more than 5TB of data - and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) - set to 20% of available CPUs. Additional nodes should be considered in conjunction - with a review of expected data size and spread based on the recommendations above. - -1. Recommended Redis setup differs depending on the size of the architecture. - For smaller architectures (up to 5,000 users) we suggest one Redis cluster for all - classes and that Redis Sentinel is hosted alongside Consul. - For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class - and another for the Queues and Shared State classes respectively. We also recommend - that you run the Redis Sentinel clusters separately as well for each Redis Cluster. - -1. For data objects such as LFS, Uploads, Artifacts, etc. We recommend a [Cloud Object Storage service](../object_storage.md) - over NFS where possible, due to better performance and availability. - -1. NFS can be used as an alternative for both repository data (replacing Gitaly) and - object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). - -1. Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) - as the load balancer. However other reputable load balancers with similar feature sets - should also work instead but be aware these aren't validated. - -1. We strongly recommend that any Gitaly and / or NFS nodes are set up with SSD disks over - HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write - as these components have heavy I/O. These IOPS values are recommended only as a starter - as with time they may be adjusted higher or lower depending on the scale of your - environment's workload. If you're running the environment on a Cloud provider - you may need to refer to their documentation on how configure IOPS correctly. - -1. The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) - CPU platform on GCP. On different hardware you may find that adjustments, either lower - or higher, are required for your CPU or Node counts accordingly. For more information, a - [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found - [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). - -1. AWS-equivalent configurations are rough suggestions and may change in the - future. They have not yet been tested and validated. +This document was moved to [another location](../reference_architectures/index.md). diff --git a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md index 662b6a7d50c..adc4b020331 100644 --- a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md +++ b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md @@ -72,7 +72,7 @@ and they will assist you with any issues you are having. This is the principle of Kubernetes, read [Twelve-factor app](https://12factor.net/) for details. -- How to get cronjobs configured on a cluster +- How to get cron jobs configured on a cluster ```shell kubectl get cronjobs @@ -206,7 +206,7 @@ all Kubernetes resources and dependent charts: helm get manifest <release name> ``` -## Installation of minimal GitLab config via Minukube on macOS +## Installation of minimal GitLab config via Minikube on macOS This section is based on [Developing for Kubernetes with Minikube](https://docs.gitlab.com/charts/development/minikube/index.html) and [Helm](https://docs.gitlab.com/charts/installation/tools.html#helm). Refer diff --git a/doc/ci/review_apps/index.md b/doc/ci/review_apps/index.md index eb333bb66ed..1ce895f3ce2 100644 --- a/doc/ci/review_apps/index.md +++ b/doc/ci/review_apps/index.md @@ -267,7 +267,7 @@ to your review app. ​After determining the ID for the merge request to link to a visual review app, you can supply the ID by either:​​ -- Hardcoding it in the script tag via the data attribute `data-merge-request-id` of the app. +- Hard-coding it in the script tag via the data attribute `data-merge-request-id` of the app. - Dynamically adding the `data-merge-request-id` value during the build of the app. - Supplying it manually through the visual review form in the app. diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index cee90dd087f..f300839e2d9 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -382,7 +382,7 @@ use for storing things like passwords, SSH keys, and credentials. Group-level variables can be added by: 1. Navigating to your group's **Settings > CI/CD** page. -1. Inputing variable types, keys, and values in the **Variables** section. +1. Inputting variable types, keys, and values in the **Variables** section. Any variables of [subgroups](../../user/group/subgroups/index.md) will be inherited recursively. Once you set them, they will be available for all subsequent pipelines. Any group-level user defined variables can be viewed in projects by: diff --git a/doc/development/documentation/workflow.md b/doc/development/documentation/workflow.md index 01f528dcfa4..ab6200155bf 100644 --- a/doc/development/documentation/workflow.md +++ b/doc/development/documentation/workflow.md @@ -1,19 +1,20 @@ # Documentation process -The process for creating and maintaining GitLab product documentation depends on whether the -documentation is associated with: +The process for creating and maintaining GitLab product documentation allows +anyone to contribute a merge request or create an issue for GitLab's +documentation. -- [A new feature or feature enhancement](#for-a-product-change). - - Delivered for a specific milestone and associated with specific code changes. This documentation - has the highest priority. +NOTE: **Note:** +Documentation updates relating to new features or feature enhancements must +use the [feature workflow process](https://about.gitlab.com/handbook/engineering/ux/technical-writing/workflow/#for-a-product-change) described in the GitLab Handbook. -- [Changes outside a specific milestone](#for-all-other-documentation). +## Who updates the docs? - It is usually not associated with a specific code change and has a lower priority. +*Anyone* can contribute! You can create a merge request for documentation when: -Documentation is not usually required when a "backstage feature" is added or changed, and does not -directly affect the way that any user or administrator interacts with GitLab. +- You find errors or other room for improvement in existing documentation. +- You have an idea for all-new documentation that would help a GitLab user or administrator to + accomplish their work with GitLab. ## Documentation labels @@ -32,372 +33,17 @@ The following are also added by members of the Technical Writing team: `docs::` prefix. For example, `~docs::improvement`. - The `~Technical Writing` [team label](../contributing/issue_workflow.md#team-labels). -## For a product change - -This documentation is required for any new or changed feature and is: - -- Created or updated as part of feature development, almost always in the same merge request as the - feature code. Including documentation in the same merge request as the code eliminates the - possibility that code and documentation get out of sync. -- Required with the delivery of a feature for a specific milestone as part of GitLab's - [definition of done](../contributing/merge_request_workflow.md#definition-of-done). -- Often linked from the release post. - -### Roles and responsibilities - -Documentation for specific milestones involves the: - -- Developer of a feature or enhancement. -- Product Manager for the group delivering the new feature or feature enhancement. -- Technical Writer assigned to the group. - -Each role is described below. - -#### Developers - -Developers are the primary author of documentation for a feature or feature enhancement. They are -responsible for: - -- Developing initial content required for a feature. -- Liaising with their Product Manager to understand what documentation must be delivered, and when. -- Requesting technical reviews from other developers within their group. -- Requesting documentation reviews from the Technical Writer - [assigned to the DevOps stage group](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments) - that is delivering the new feature or feature enhancements. - -TIP: **Tip:** -Community Contributors can ask for additional help from GitLab team members. - -##### Authoring - -Because the documentation is an essential part of the product, if a ~feature issue also contains the -~documentation label, you must ship the new or updated documentation with the code of the feature. - -Technical Writers are happy to help, as requested and planned on an issue-by-issue basis. - -For feature issues requiring documentation, follow the process below unless otherwise agreed with -the Product Manager and Technical Writer for a given issue: - -- Include any new and edited documentation, either in: - - The merge request introducing the code. - - A separate merge request raised around the same time. -- Use the [documentation requirements](#documentation-requirements) developed by the Product Manager - in the issue and discuss any further documentation plans or ideas as needed. - - If the new or changed documentation requires extensive collaboration or conversation, a - separate, linked issue can be used for the planning process. - -- Use the [Documentation guidelines](index.md), as well as other resources linked from there, - including: - - Documentation [Structure and template](structure.md) page. - - [Style Guide](styleguide.md). - - [Markdown Guide](https://about.gitlab.com/handbook/engineering/ux/technical-writing/markdown-guide/). -- Contact the Technical Writer for the relevant [DevOps stage](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments) - in your issue or merge request, or within `#docs` on GitLab Slack, if you: - - Need any help to choose the correct place for documentation. - - Want to discuss a documentation idea or outline. - - Want to request any other help. -- If you are working on documentation in a separate merge request, ensure the documentation is - merged as close as possible to the code merge. -- A policy for documenting [feature-flagged](../feature_flags/index.md) issues is forthcoming and you - are welcome to join the [discussion](https://gitlab.com/gitlab-org/gitlab/issues/26347). - -##### Reviews and merging - -Reviewers help ensure: - -- Accuracy. -- Clarity. -- Completeness. -- Adherence to: - - [Documentation requirements](#documentation-requirements) in the issue. - - [Documentation guidelines](index.md). - - [Style guide](styleguide.md). - -Prior to merging, documentation changes committed by the developer must be reviewed by: - -- The code reviewer for the merge request. This is known as a technical review. -- Optionally, others involved in the work such as other developers or the Product Manager. -- The Technical Writer for the DevOps stage group, except in exceptional circumstances where a - [post-merge review](#post-merge-reviews) can be requested. -- A maintainer of the project. - -#### Product Managers - -Product Managers are responsible for the [documentation requirements](#documentation-requirements) -for a feature or feature enhancement. They can also: - -- Liaise with the Technical Writer for discussion and collaboration. -- Review documentation themselves. - -For issues requiring any new or updated documentation, the Product Manager must: - -- Add the ~documentation label. -- Confirm or add the [documentation requirements](#documentation-requirements). -- Ensure the issue contains: - - Any new or updated feature name. - - Overview, description, and use cases when applicable (as required by the - [documentation structure and template](structure.md)). - -Everyone is encouraged to draft the documentation requirements in the issue. However, a Product -Manager will: - -- When the issue is assigned a release milestone, review and update the Documentation details. -- By the kickoff, finalize the documentation details. - -#### Technical Writers - -Technical Writers are responsible for: - -- Participating in issues discussions and reviewing MRs for the upcoming milestone. -- Reviewing documentation requirements in issues when called upon. -- Answering questions, and helping and providing advice throughout the authoring and editing - process. -- Reviewing all significant new and updated documentation content, whether before merge or after it - is merged. -- Assisting the developer and Product Manager with feature documentation delivery. - -##### Planning - -The Technical Writer: - -- Reviews their group's `~feature` issues that are part of the next milestone to get a sense of the - scope of content likely to be authored. -- Recommends the `~documentation` label on issues from that list which don't have it but should, or - inquires with the PM to determine if documentation is truly required. -- For `~direction` issues from that list, reads the full issue and reviews its Documentation - requirements section. Addresses any recommendations or questions with the PMs and others - collaborating on the issue in order to refine or expand the Documentation requirements. - -##### Collaboration - -By default, the developer will work on documentation changes independently, but -the developer, Product Manager, or Technical Writer can propose a broader collaboration for -any given issue. - -Additionally, Technical Writers are available for questions at any time. - -##### Review - -Technical Writers: - -- Provide non-blocking reviews of all documentation changes, before or after the change is merged. -- Confirm that the documentation is: - - Clear. - - Grammatically correct. - - Discoverable. - - Navigable. -- Ensures that the documentation avoids: - - Redundancy. - - Bad file locations. - - Typos. - - Broken links. - -The Technical Writer will review the documentation to check that the developer and -code reviewer have ensured: - -- Clarity. -- Appropriate location, making sure the documentation is in the correct directories (often - reflecting how the product is structured) and has the correct name. -- Syntax, typos, and broken links. -- Improvements to the content. -- Accordance with the: - - [Documentation Style Guide](styleguide.md). - - [Structure and Template](structure.md) doc. - -### When documentation is required - -Documentation [is required](../contributing/merge_request_workflow.md#definition-of-done) for a -milestone when: - -- A new or enhanced feature is shipped that impacts the user or administrator experience. -- There are changes to the UI or API. -- A process, workflow, or previously documented feature is changed. -- A feature is deprecated or removed. - -NOTE: **Note:** -Documentation refactoring unrelated to a feature change is covered in the -[other process](#for-all-other-documentation), so that time-sensitive documentation updates are -prioritized. - -### Documentation requirements - -Requirements for the documentation of a feature should be included as part of the -issue for planning that feature in a **Documentation** section within the issue description. Issues -created using the [**Feature Proposal** template](https://gitlab.com/gitlab-org/gitlab/raw/master/.gitlab/issue_templates/Feature%20proposal.md) -have this section by default. - -Anyone can add these details, but the Product Manager who assigns the issue to a specific release -milestone will ensure these details are present and finalized by the time of that milestone's kickoff. - -Developers, Technical Writers, and others may help further refine this plan at any time on request. - -The following details should be included: - -- What concepts and procedures should the documentation guide and enable the user to understand or - accomplish? -- To this end, what new page(s) are needed, if any? What pages or subsections need updates? - Consider changes and additions to user, admin, and API documentation. -- For any guide or instruction set, should it help address a single use case, or be flexible to - address a certain range of use cases? -- Do we need to update a previously recommended workflow? Should we link the new feature from - various relevant locations? Consider all ways documentation should be affected. -- Are there any key terms or task descriptions that should be included so that the documentation is - found in relevant searches? -- Include suggested titles of any pages or subsection headings, if applicable. -- List any documentation that should be cross-linked, if applicable. - -### Including docs with code - -Currently, the Technical Writing team strongly encourages including documentation in -the same merge request as the code that it relates to, but this is not strictly mandatory. -It's still common for documentation to be added in an MR separate from the feature MR. - -Engineering teams may elect to adopt a workflow where it is **mandatory** that docs -are included in the code MR, as part of their [definition of done](../contributing/merge_request_workflow.md#definition-of-done). -When a team adopts this workflow, that team's engineers must include their docs in the **same** -MR as their feature code, at all times. - -#### Downsides of separate docs MRs - -A workflow that has documentation separated into its own MR has many downsides. - -If the documentation merges **before** the feature: - -- GitLab.com users might try to use the feature before it's released, driving support tickets. -- If the feature is delayed, the documentation might not be pulled/reverted in time and could be - accidentally included in the self-managed package for that release. - -If the documentation merges **after** the feature: - -- The feature might be included in the self-managed package, but without any documentation - if the docs MR misses the cutoff. -- A feature might show up in the GitLab.com UI before any documentation exists for it. - Users surprised by this feature will search for documentation and won't find it, possibly driving - support tickets. - -Having two separate MRs means: - -- Two different people might be responsible for merging one feature, which is not workable - with an asynchronous work style. The feature might merge while the technical writer is asleep, - creating a potentially lengthy delay between the two merges. -- If the docs MR is assigned to the same maintainer as responsible for the feature - code MR, they will have to review and juggle two MRs instead of dealing with just one. - -Documentation quality might be lower, because: - -- Having docs in a separate MR will mean far fewer people will see and verify them, - increasing the likelihood that issues will be missed. -- In a "split" workflow, engineers might only create the documentation MR once the - feature MR is ready, or almost ready. This gives the technical writer little time - to learn about the feature in order to do a good review. It also increases pressure - on them to review and merge faster than desired, letting problems slip in due to haste. - -#### Benefits of always including docs with code - -Including docs with code (and doing it early in the development process) has many benefits: - -- There are no timing issues connected to releases: - - If a feature slips to the next release, the documentation slips too. - - If the feature *just* makes it into a release, the docs *just* make it in too. - - If a feature makes it to GitLab.com early, the documentation will be ready for - our early adopters. -- Only a single person will be responsible for merging the feature (the code maintainer). -- The technical writer will have more time to gain an understanding of the feature - and will be better able to verify the content of the docs in the Review App or GDK. - They will also be able to offer advice for improving the UI text or offer additional use cases. -- The documentation will have increased visibility: - - Everyone involved in the merge request will see the docs. This could include product - managers, multiple engineers with deep domain knowledge, as well as the code reviewers - and maintainer. They will be more likely to catch issues with examples, as well - as background or concepts that the technical writer may not be aware of. - - Increasing visibility of the documentation also has the side effect of improving - *other* engineers' documentation. By reviewing each other's MRs, each engineer's - own documentation skills will improve. -- Thinking about the documentation early can help engineers generate better examples, - as they will need to think about what examples a user will want, and will need to - make sure the code they write implements that example properly. - -#### Docs with code as a workflow - -In order to have docs included with code as a mandatory workflow, some changes might -need to happen to a team's current workflow: - -- The engineers must strive to include the docs early in the development process, - to give ample time for review, not just from the technical writer, but also the - code reviewer and maintainer. -- Reviewers and maintainers must also review the docs during code reviews, to make - sure the described processes match the expected use of the feature, and that examples - are correct. They do *not* need to worry about style, grammar, and so on. -- The technical writer must be assigned the MR directly and not only pinged. Thanks - to the ability to have [multiple assignees for any MR](../../user/project/merge_requests/getting_started.md#multiple-assignees-starter), - this can be done at any time, but must be before the code maintainer review. It's - common to have both the docs and code reviews happening at the same time, with the - author, reviewer and technical writer discussing the docs together. -- When the docs are ready, the technical writer will click **Approve** and usually - will no longer be involved in the MR. If the feature changes during code review and - the docs are updated, the technical writer must be reassigned the MR to verify the - update. -- Maintainers are allowed to merge features with the docs "as-is", even if the technical - writer has not given final approval yet. The **docs reviews must not be blockers**. Therefore - it's important to get the docs included and assigned to the technical writers early. - If the feature is merged before final docs approval, the maintainer must create - a [post-merge follow-up issue](#post-merge-reviews), and assign it to both the engineer - and technical writer. - -Maintainers are allowed to merge features with the docs "as-is" even if the -technical writer has not given final approval yet but the merge request has -all other required approvals. - -You can visualize the parallel workflow for code and docs reviews as: - -```mermaid -graph TD - A("Feature MR Created (Engineer)") --> |Assign| B("Code Review (reviewer)") - B --> |"Approve / Reassign"| C("Code Review (maintainer)") - C --> |Approve| F("Merge (maintainer)") - A --> D("Docs Added (Engineer)") - D --> |Assign| E("Docs Review (Tech Writer)") - E --> |Approve| F -``` - -For complex features split over multiple merge requests: - -- If a merge request is implementing components for a future feature, but the components - are not accessible to users yet, then no documentation should be included. -- If a merge request will expose a feature to users in any way, such as an enabled - UI element, an API endpoint, or anything similar, then that MR **must** have docs. - Note that this may mean multiple docs additions could happen in the buildup to the - implementation of a single large feature, for example API docs and feature usage docs. -- If it's unclear which engineer should add the feature documentation into their - MR, the engineering manager should decide during planning, and tie the documentation - to the last MR that must be merged before a feature is considered released. - This is often, but not always, a frontend MR. - -## For all other documentation - Documentation changes that are not associated with the release of a new or updated feature do not take the `~feature` label, but still need the `~documentation` label. They may include: - Documentation created or updated to improve accuracy, completeness, ease of use, or any reason - other than a [feature change](#for-a-product-change). + other than a [feature change](https://about.gitlab.com/handbook/engineering/ux/technical-writing/workflow/#for-a-product-change). - Addressing gaps in existing documentation, or making improvements to existing documentation. - Work on special projects related to the documentation. -TIP: **Tip:** -Anyone can contribute a merge request or create an issue for GitLab's documentation. - -### Who updates the docs - -Anyone can contribute! You can create a merge request for documentation when: - -- You find errors or other room for improvement in existing documentation. -- You have an idea for all-new documentation that would help a GitLab user or administrator to - accomplish their work with GitLab. - -### How to update the docs +## How to update the docs To update GitLab documentation: @@ -464,7 +110,7 @@ The process involves the following: The process is reflected in the **Documentation** [merge request template](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/merge_request_templates/Documentation.md). -### Other ways to help +## Other ways to help If you have ideas for further documentation resources please [create an issue](https://gitlab.com/gitlab-org/gitlab/issues/new?issuable_template=Documentation) diff --git a/doc/development/packages.md b/doc/development/packages.md index 2d93343a7dd..8614f78d836 100644 --- a/doc/development/packages.md +++ b/doc/development/packages.md @@ -159,7 +159,7 @@ During this phase, the idea is to collect as much information as possible about The analysis usually takes a full milestone to complete, though it's not impossible to start the implementation in the same milestone. -In particular, the upload request can have some [requirements in the GitLab Workhorse project](#file-uploads). This project has a different release cycle than the rails backend. It's **strongly** recommended that you open an issue there as soon as the upload request analysis is done. This way GitLab Worhorse is already ready when the upload request is implemented on the rails backend. +In particular, the upload request can have some [requirements in the GitLab Workhorse project](#file-uploads). This project has a different release cycle than the rails backend. It's **strongly** recommended that you open an issue there as soon as the upload request analysis is done. This way GitLab Workhorse is already ready when the upload request is implemented on the rails backend. ### Implementation diff --git a/doc/development/shell_scripting_guide/index.md b/doc/development/shell_scripting_guide/index.md index 387ef01bdcf..99cd1b9d67f 100644 --- a/doc/development/shell_scripting_guide/index.md +++ b/doc/development/shell_scripting_guide/index.md @@ -80,7 +80,20 @@ We format shell scripts according to the [Google Shell Style Guide](https://goog so the following `shfmt` invocation should be applied to the project's script files: ```shell -shfmt -i 2 -ci scripts/**/*.sh +shfmt -i 2 -ci -w scripts/**/*.sh +``` + +In addition to the [Linting](#linting) GitLab CI/CD job, all projects with shell scripts should also +use this job: + +```yaml +shfmt: + image: mvdan/shfmt:v3.1.0-alpine + stage: test + before_script: + - shfmt -version + script: + - shfmt -i 2 -ci -d scripts # path to your shell scripts ``` TIP: **Tip:** @@ -88,11 +101,6 @@ By default, shfmt will use the [shell detection](https://github.com/mvdan/sh#shf and ignore files starting with a period. To override this, use `-ln` flag to specify the shell dialect: `-ln posix` or `-ln bash`. -NOTE: **Note:** -Currently, the `shfmt` tool [is not shipped](https://github.com/mvdan/sh/issues/68) as a Docker image containing -a Linux shell. This makes it impossible to use the [official Docker image](https://hub.docker.com/r/mvdan/shfmt) -in GitLab Runner. This [may change](https://github.com/mvdan/sh/issues/68#issuecomment-507721371) in future. - ## Testing NOTE: **Note:** diff --git a/doc/install/README.md b/doc/install/README.md index 9169a01af67..f1ef368685e 100644 --- a/doc/install/README.md +++ b/doc/install/README.md @@ -14,15 +14,15 @@ and cost of hosting. There are many ways you can install GitLab depending on your platform: 1. **Omnibus GitLab**: The official deb/rpm packages that contain a bundle of GitLab - and the various components it depends on like PostgreSQL, Redis, Sidekiq, etc. + and the various components it depends on, like PostgreSQL, Redis, Sidekiq, etc. 1. **GitLab Helm chart**: The cloud native Helm chart for installing GitLab and all its components on Kubernetes. 1. **Docker**: The Omnibus GitLab packages dockerized. 1. **Source**: Install GitLab and all its components from scratch. TIP: **If in doubt, choose Omnibus:** -The Omnibus GitLab packages are mature, scalable, support -[high availability](../administration/availability/index.md) and are used +The Omnibus GitLab packages are mature, +[scalable](../administration/reference_architectures/index.md) and are used today on GitLab.com. The Helm charts are recommended for those who are familiar with Kubernetes. @@ -36,7 +36,7 @@ The Omnibus GitLab package uses our official deb/rpm repositories. This is recommended for most users. If you need additional flexibility and resilience, we recommend deploying -GitLab as described in our [High Availability documentation](../administration/availability/index.md). +GitLab as described in our [reference architecture documentation](../administration/reference_architectures/index.md). [**> Install GitLab using the Omnibus GitLab package.**](https://about.gitlab.com/install/) @@ -67,7 +67,7 @@ GitLab maintains a set of official Docker images based on the Omnibus GitLab pac ## Installing GitLab from source If the Omnibus GitLab package is not available in your distribution, you can -install GitLab from source: Useful for unsupported systems like *BSD. For an +install GitLab from source: Useful for unsupported systems like \*BSD. For an overview of the directory structure, read the [structure documentation](structure.md). [**> Install GitLab from source.**](installation.md) diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index 17238bf301b..8daa9b40111 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -739,7 +739,7 @@ Have a read through these other resources and feel free to [open an issue](https://gitlab.com/gitlab-org/gitlab/issues/new) to request additional material: -- [Scaling GitLab](../../administration/scaling/index.md): +- [Scaling GitLab](../../administration/reference_architectures/index.md): GitLab supports several different types of clustering and high-availability. - [Geo replication](../../administration/geo/replication/index.md): Geo is the solution for widely distributed development teams. diff --git a/doc/install/requirements.md b/doc/install/requirements.md index a842c827727..e3981b6b92b 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -95,7 +95,7 @@ This is the recommended minimum hardware for a handful of example GitLab user ba - 4 cores supports up to 500 users - 8 cores supports up to 1,000 users - 32 cores supports up to 5,000 users -- More users? Run it high-availability on [multiple application servers](https://about.gitlab.com/solutions/high-availability/) +- More users? Consult the [reference architectures page](../administration/reference_architectures/index.md) ### Memory @@ -112,7 +112,7 @@ errors during usage. - 16GB RAM supports up to 500 users - 32GB RAM supports up to 1,000 users - 128GB RAM supports up to 5,000 users -- More users? Run it high-availability on [multiple application servers](https://about.gitlab.com/solutions/high-availability/) +- More users? Consult the [reference architectures page](../administration/reference_architectures/index.md) We recommend having at least [2GB of swap on your server](https://askubuntu.com/a/505344/310789), even if you currently have enough available RAM. Having swap will help reduce the chance of errors occurring diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index 1233662d743..e46d8a0e036 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -624,7 +624,7 @@ Here are some common pitfalls and how to overcome them: Gitlab::Elastic::Indexer::Error: time="2020-01-23T09:13:00Z" level=fatal msg="health check timeout: no Elasticsearch node available" ``` - You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticseach Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be accepted as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a). + You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticsearch Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be accepted as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a). Once you have corrected the formatting of the URL, delete the index (via the [dedicated Rake task](#gitlab-elasticsearch-rake-tasks)) and [reindex the content of your instance](#adding-gitlabs-data-to-the-elasticsearch-index). ### Low level troubleshooting diff --git a/doc/integration/sourcegraph.md b/doc/integration/sourcegraph.md index 5da9dd1fbc9..3f5ea04f491 100644 --- a/doc/integration/sourcegraph.md +++ b/doc/integration/sourcegraph.md @@ -107,7 +107,7 @@ When visiting one of these views, you can now hover over a code reference to see ## Sourcegraph for GitLab.com -Sourcegraph powered code intelligence is avaialable for all public projects on GitLab.com. +Sourcegraph powered code intelligence is available for all public projects on GitLab.com. Support for private projects is currently not available for GitLab.com; follow the epic [&2201](https://gitlab.com/groups/gitlab-org/-/epics/2201) diff --git a/doc/topics/git/how_to_install_git/index.md b/doc/topics/git/how_to_install_git/index.md index 01bf3f5f5c5..75ea6183a32 100644 --- a/doc/topics/git/how_to_install_git/index.md +++ b/doc/topics/git/how_to_install_git/index.md @@ -39,7 +39,7 @@ it by running in your terminal: xcode-select --install ``` -Click **Install** to download and install it. Alternativelly, you can install +Click **Install** to download and install it. Alternatively, you can install the entire [XCode](https://developer.apple.com/xcode/) package through the macOS App Store. diff --git a/doc/user/markdown.md b/doc/user/markdown.md index e7c75c7b8cd..38b496e638b 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -105,7 +105,7 @@ changing how standard Markdown is used: | Standard Markdown | Extended Markdown in GitLab | | ------------------------------------- | ------------------------- | -| [blockquotes](#blockquotes) | [multiline blockquotes](#multiline-blockquote) | +| [blockquotes](#blockquotes) | [multi-line blockquotes](#multiline-blockquote) | | [code blocks](#code-spans-and-blocks) | [colored code and syntax highlighting](#colored-code-and-syntax-highlighting) | | [emphasis](#emphasis) | [multiple underscores in words](#multiple-underscores-in-words-and-mid-word-emphasis) | [headers](#headers) | [linkable Header IDs](#header-ids-and-links) | @@ -353,7 +353,7 @@ However, the wrapping tags can't be mixed: - [- deletion -} ``` -If your diff includes words in `` `code` `` font, make sure to escape each bactick `` ` `` with a +If your diff includes words in `` `code` `` font, make sure to escape each backtick `` ` `` with a backslash `\`, otherwise the diff highlight won't render correctly: ```markdown @@ -396,8 +396,8 @@ a^2+b^2=c^2 _Be advised that KaTeX only supports a [subset](https://katex.org/docs/supported.html) of LaTeX._ -NOTE: **Note:** This also works for the asciidoctor `:stem: latexmath`. For details see -the [asciidoctor user manual](https://asciidoctor.org/docs/user-manual/#activating-stem-support). +NOTE: **Note:** This also works for the Asciidoctor `:stem: latexmath`. For details see +the [Asciidoctor user manual](https://asciidoctor.org/docs/user-manual/#activating-stem-support). ### Special GitLab references @@ -608,7 +608,7 @@ Quote break. > If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/markdown.md#multiline-blockquote). -GFM extends the standard Markdown standard by also supporting multiline blockquotes +GFM extends the standard Markdown standard by also supporting multi-line blockquotes fenced by `>>>`: ```markdown @@ -1216,7 +1216,7 @@ Some text to show that the reference links can follow later. - This is an [inline-style link](https://www.google.com) - This is a [link to a repository file in the same directory](index.md) -- This is a [relative link to a readme one directory higher](../README.md) +- This is a [relative link to a README one directory higher](../README.md) - This is a [link that also has title text](https://www.google.com "This link takes you to Google!") Using header ID anchors: @@ -1244,7 +1244,7 @@ will point the link to `wikis/style` only when the link is inside of a wiki Mark #### URL auto-linking -GFM will autolink almost any URL you put into your text: +GFM will auto-link almost any URL you put into your text: ```markdown - https://www.google.com diff --git a/doc/user/profile/personal_access_tokens.md b/doc/user/profile/personal_access_tokens.md index 1223f7b801a..e05528f5696 100644 --- a/doc/user/profile/personal_access_tokens.md +++ b/doc/user/profile/personal_access_tokens.md @@ -4,7 +4,9 @@ type: concepts, howto # Personal access tokens -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3749) in GitLab 8.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/3749) in GitLab 8.8. +> - [Notifications about expiring tokens](https://gitlab.com/gitlab-org/gitlab/-/issues/3649) added in GitLab 12.6. +> - [Token lifetime limits](https://gitlab.com/gitlab-org/gitlab/-/issues/3649) added in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.6. If you're unable to use [OAuth2](../../api/oauth2.md), you can use a personal access token to authenticate with the [GitLab API](../../api/README.md#personal-access-tokens). @@ -12,8 +14,13 @@ You can also use personal access tokens with Git to authenticate over HTTP or SS Personal access tokens expire on the date you define, at midnight UTC. +- GitLab runs a check at 01:00 AM UTC every day to identify personal access tokens that will expire in under seven days. The owners of these tokens are notified by email. +- In GitLab Ultimate, administrators may [limit the lifetime of personal access tokens](../admin_area/settings/account_and_limit_settings.md#limiting-lifetime-of-personal-access-tokens-ultimate-only). + For examples of how you can use a personal access token to authenticate with the API, see the following section from our [API Docs](../../api/README.md#personal-access-tokens). +GitLab also offers [impersonation tokens](../../api/README.md#impersonation-tokens) which are created by administrators via the API. They're a great fit for automated authentication as a specific user. + ## Creating a personal access token You can create as many personal access tokens as you like from your GitLab diff --git a/doc/user/project/merge_requests/creating_merge_requests.md b/doc/user/project/merge_requests/creating_merge_requests.md index d8a2b427288..544727380e7 100644 --- a/doc/user/project/merge_requests/creating_merge_requests.md +++ b/doc/user/project/merge_requests/creating_merge_requests.md @@ -171,7 +171,7 @@ When the changes are merged, your changes are added to the upstream repository a the branch as per specification. After your work is merged, if you don't intend to make any other contributions to the upstream project, you can unlink your fork from its upstream project in the **Settings > Advanced Settings** section by -[removing the forking relashionship](../settings/index.md#removing-a-fork-relationship). +[removing the forking relationship](../settings/index.md#removing-a-fork-relationship). For further details, [see the forking workflow documentation](../repository/forking_workflow.md). diff --git a/lib/api/features.rb b/lib/api/features.rb index 69b751e9bdb..f507919b055 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -16,6 +16,15 @@ module API end end + def gate_key(params) + case params[:key] + when 'percentage_of_actors' + :percentage_of_actors + else + :percentage_of_time + end + end + def gate_targets(params) Feature::Target.new(params).targets end @@ -40,15 +49,22 @@ module API end params do requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time' + optional :key, type: String, desc: '`percentage_of_actors` or the default `percentage_of_time`' optional :feature_group, type: String, desc: 'A Feature group name' optional :user, type: String, desc: 'A GitLab username' optional :group, type: String, desc: "A GitLab group's path, such as 'gitlab-org'" optional :project, type: String, desc: 'A projects path, like gitlab-org/gitlab-ce' + + mutually_exclusive :key, :feature_group + mutually_exclusive :key, :user + mutually_exclusive :key, :group + mutually_exclusive :key, :project end post ':name' do feature = Feature.get(params[:name]) targets = gate_targets(params) value = gate_value(params) + key = gate_key(params) case value when true @@ -64,7 +80,11 @@ module API feature.disable end else - feature.enable_percentage_of_time(value) + if key == :percentage_of_actors + feature.enable_percentage_of_actors(value) + else + feature.enable_percentage_of_time(value) + end end present feature, with: Entities::Feature, current_user: current_user diff --git a/spec/factories/identities.rb b/spec/factories/identities.rb index a2615ce30c3..fda4bfa589b 100644 --- a/spec/factories/identities.rb +++ b/spec/factories/identities.rb @@ -3,6 +3,6 @@ FactoryBot.define do factory :identity do provider { 'ldapmain' } - extern_uid { 'my-ldap-id' } + sequence(:extern_uid) { |n| "my-ldap-id-#{n}" } end end diff --git a/spec/features/issues/filtered_search/visual_tokens_spec.rb b/spec/features/issues/filtered_search/visual_tokens_spec.rb index 3c50cb4c997..2b164e2fd97 100644 --- a/spec/features/issues/filtered_search/visual_tokens_spec.rb +++ b/spec/features/issues/filtered_search/visual_tokens_spec.rb @@ -113,7 +113,7 @@ describe 'Visual tokens', :js do describe 'add new token after editing existing token' do before do input_filtered_search('author:=@root assignee:=none', submit: false) - first('.tokens-container .filtered-search-token').double_click + first('.tokens-container .filtered-search-token').click filtered_search.send_keys(' ') end diff --git a/spec/features/projects/graph_spec.rb b/spec/features/projects/graph_spec.rb index 6b2a9a6b852..25efb7b28a7 100644 --- a/spec/features/projects/graph_spec.rb +++ b/spec/features/projects/graph_spec.rb @@ -59,7 +59,7 @@ describe 'Project Graph', :js do it 'HTML escapes branch name' do expect(page.body).to include("Commit statistics for <strong>#{ERB::Util.html_escape(branch_name)}</strong>") - expect(page.body).not_to include(branch_name) + expect(page.find('.dropdown-toggle-text')['innerHTML']).to eq(ERB::Util.html_escape(branch_name)) end end diff --git a/spec/frontend/pipelines/pipelines_table_spec.js b/spec/frontend/pipelines/pipelines_table_spec.js new file mode 100644 index 00000000000..b0ab250dd16 --- /dev/null +++ b/spec/frontend/pipelines/pipelines_table_spec.js @@ -0,0 +1,66 @@ +import { mount } from '@vue/test-utils'; +import PipelinesTable from '~/pipelines/components/pipelines_table.vue'; + +describe('Pipelines Table', () => { + let pipeline; + let wrapper; + + const jsonFixtureName = 'pipelines/pipelines.json'; + + const defaultProps = { + pipelines: [], + autoDevopsHelpPath: 'foo', + viewType: 'root', + }; + + const createComponent = (props = defaultProps) => { + wrapper = mount(PipelinesTable, { + propsData: props, + }); + }; + const findRows = () => wrapper.findAll('.commit.gl-responsive-table-row'); + + preloadFixtures(jsonFixtureName); + + beforeEach(() => { + const { pipelines } = getJSONFixture(jsonFixtureName); + pipeline = pipelines.find(p => p.user !== null && p.commit !== null); + + createComponent(); + }); + + afterEach(() => { + wrapper.destroy(); + wrapper = null; + }); + + describe('table', () => { + it('should render a table', () => { + expect(wrapper.classes()).toContain('ci-table'); + }); + + it('should render table head with correct columns', () => { + expect(wrapper.find('.table-section.js-pipeline-status').text()).toEqual('Status'); + + expect(wrapper.find('.table-section.js-pipeline-info').text()).toEqual('Pipeline'); + + expect(wrapper.find('.table-section.js-pipeline-commit').text()).toEqual('Commit'); + + expect(wrapper.find('.table-section.js-pipeline-stages').text()).toEqual('Stages'); + }); + }); + + describe('without data', () => { + it('should render an empty table', () => { + expect(findRows()).toHaveLength(0); + }); + }); + + describe('with data', () => { + it('should render rows', () => { + createComponent({ pipelines: [pipeline], autoDevopsHelpPath: 'foo', viewType: 'root' }); + + expect(findRows()).toHaveLength(1); + }); + }); +}); diff --git a/spec/frontend/users_select/utils_spec.js b/spec/frontend/users_select/utils_spec.js new file mode 100644 index 00000000000..a09935d8a04 --- /dev/null +++ b/spec/frontend/users_select/utils_spec.js @@ -0,0 +1,33 @@ +import $ from 'jquery'; +import { getAjaxUsersSelectOptions, getAjaxUsersSelectParams } from '~/users_select/utils'; + +const options = { + fooBar: 'baz', + activeUserId: 1, +}; + +describe('getAjaxUsersSelectOptions', () => { + it('returns options built from select data attributes', () => { + const $select = $('<select />', { 'data-foo-bar': 'baz', 'data-user-id': 1 }); + + expect( + getAjaxUsersSelectOptions($select, { fooBar: 'fooBar', activeUserId: 'user-id' }), + ).toEqual(options); + }); +}); + +describe('getAjaxUsersSelectParams', () => { + it('returns query parameters built from provided options', () => { + expect( + getAjaxUsersSelectParams(options, { + foo_bar: 'fooBar', + active_user_id: 'activeUserId', + non_existent_key: 'nonExistentKey', + }), + ).toEqual({ + foo_bar: 'baz', + active_user_id: 1, + non_existent_key: null, + }); + }); +}); diff --git a/spec/graphql/types/snippets/blob_viewer_type_spec.rb b/spec/graphql/types/snippets/blob_viewer_type_spec.rb index a51d09813ab..841e22451db 100644 --- a/spec/graphql/types/snippets/blob_viewer_type_spec.rb +++ b/spec/graphql/types/snippets/blob_viewer_type_spec.rb @@ -3,10 +3,91 @@ require 'spec_helper' describe GitlabSchema.types['SnippetBlobViewer'] do + let_it_be(:snippet) { create(:personal_snippet, :repository) } + let_it_be(:blob) { snippet.repository.blob_at('HEAD', 'files/images/6049019_460s.jpg') } + it 'has the correct fields' do expected_fields = [:type, :load_async, :too_large, :collapsed, :render_error, :file_type, :loading_partial_name] expect(described_class).to have_graphql_fields(*expected_fields) end + + it { expect(described_class.fields['type'].type).to be_non_null } + it { expect(described_class.fields['loadAsync'].type).to be_non_null } + it { expect(described_class.fields['collapsed'].type).to be_non_null } + it { expect(described_class.fields['tooLarge'].type).to be_non_null } + it { expect(described_class.fields['renderError'].type).not_to be_non_null } + it { expect(described_class.fields['fileType'].type).to be_non_null } + it { expect(described_class.fields['loadingPartialName'].type).to be_non_null } + + shared_examples 'nil field converted to false' do + subject { GitlabSchema.execute(query, context: { current_user: snippet.author }).as_json } + + before do + allow_next_instance_of(SnippetPresenter) do |instance| + allow(instance).to receive(:blob).and_return(blob) + end + end + + it 'returns false' do + snippet_blob = subject.dig('data', 'snippets', 'edges')[0].dig('node', 'blob') + + expect(snippet_blob['path']).to eq blob.path + expect(blob_attribute).to be_nil + expect(snippet_blob['simpleViewer'][attribute]).to eq false + end + end + + describe 'collapsed' do + it_behaves_like 'nil field converted to false' do + let(:query) do + %( + query { + snippets(ids:"#{snippet.to_global_id}"){ + edges { + node { + blob { + path + simpleViewer { + collapsed + } + } + } + } + } + } + ) + end + + let(:attribute) { 'collapsed' } + let(:blob_attribute) { blob.simple_viewer.collapsed? } + end + end + + describe 'tooLarge' do + it_behaves_like 'nil field converted to false' do + let(:query) do + %( + query { + snippets(ids:"#{snippet.to_global_id}"){ + edges { + node { + blob { + path + simpleViewer { + tooLarge + } + } + } + } + } + } + ) + end + + let(:attribute) { 'tooLarge' } + let(:blob_attribute) { blob.simple_viewer.too_large? } + end + end end diff --git a/spec/javascripts/pipelines/pipelines_table_spec.js b/spec/javascripts/pipelines/pipelines_table_spec.js deleted file mode 100644 index 5c3387190ab..00000000000 --- a/spec/javascripts/pipelines/pipelines_table_spec.js +++ /dev/null @@ -1,86 +0,0 @@ -import Vue from 'vue'; -import pipelinesTableComp from '~/pipelines/components/pipelines_table.vue'; -import '~/lib/utils/datetime_utility'; - -describe('Pipelines Table', () => { - const jsonFixtureName = 'pipelines/pipelines.json'; - - let pipeline; - let PipelinesTableComponent; - - preloadFixtures(jsonFixtureName); - - beforeEach(() => { - const { pipelines } = getJSONFixture(jsonFixtureName); - - PipelinesTableComponent = Vue.extend(pipelinesTableComp); - pipeline = pipelines.find(p => p.user !== null && p.commit !== null); - }); - - describe('table', () => { - let component; - beforeEach(() => { - component = new PipelinesTableComponent({ - propsData: { - pipelines: [], - autoDevopsHelpPath: 'foo', - viewType: 'root', - }, - }).$mount(); - }); - - afterEach(() => { - component.$destroy(); - }); - - it('should render a table', () => { - expect(component.$el.getAttribute('class')).toContain('ci-table'); - }); - - it('should render table head with correct columns', () => { - expect( - component.$el.querySelector('.table-section.js-pipeline-status').textContent.trim(), - ).toEqual('Status'); - - expect( - component.$el.querySelector('.table-section.js-pipeline-info').textContent.trim(), - ).toEqual('Pipeline'); - - expect( - component.$el.querySelector('.table-section.js-pipeline-commit').textContent.trim(), - ).toEqual('Commit'); - - expect( - component.$el.querySelector('.table-section.js-pipeline-stages').textContent.trim(), - ).toEqual('Stages'); - }); - }); - - describe('without data', () => { - it('should render an empty table', () => { - const component = new PipelinesTableComponent({ - propsData: { - pipelines: [], - autoDevopsHelpPath: 'foo', - viewType: 'root', - }, - }).$mount(); - - expect(component.$el.querySelectorAll('.commit.gl-responsive-table-row').length).toEqual(0); - }); - }); - - describe('with data', () => { - it('should render rows', () => { - const component = new PipelinesTableComponent({ - propsData: { - pipelines: [pipeline], - autoDevopsHelpPath: 'foo', - viewType: 'root', - }, - }).$mount(); - - expect(component.$el.querySelectorAll('.commit.gl-responsive-table-row').length).toEqual(1); - }); - }); -}); diff --git a/spec/requests/api/features_spec.rb b/spec/requests/api/features_spec.rb index ce72a416c33..4ad5b4f9d49 100644 --- a/spec/requests/api/features_spec.rb +++ b/spec/requests/api/features_spec.rb @@ -198,7 +198,7 @@ describe API::Features do end end - it 'creates a feature with the given percentage if passed an integer' do + it 'creates a feature with the given percentage of time if passed an integer' do post api("/features/#{feature_name}", admin), params: { value: '50' } expect(response).to have_gitlab_http_status(:created) @@ -210,6 +210,19 @@ describe API::Features do { 'key' => 'percentage_of_time', 'value' => 50 } ]) end + + it 'creates a feature with the given percentage of actors if passed an integer' do + post api("/features/#{feature_name}", admin), params: { value: '50', key: 'percentage_of_actors' } + + expect(response).to have_gitlab_http_status(:created) + expect(json_response).to eq( + 'name' => 'my_feature', + 'state' => 'conditional', + 'gates' => [ + { 'key' => 'boolean', 'value' => false }, + { 'key' => 'percentage_of_actors', 'value' => 50 } + ]) + end end context 'when the feature exists' do @@ -298,7 +311,7 @@ describe API::Features do end end - context 'with a pre-existing percentage value' do + context 'with a pre-existing percentage of time value' do before do feature.enable_percentage_of_time(50) end @@ -316,6 +329,25 @@ describe API::Features do ]) end end + + context 'with a pre-existing percentage of actors value' do + before do + feature.enable_percentage_of_actors(42) + end + + it 'updates the percentage of actors if passed an integer' do + post api("/features/#{feature_name}", admin), params: { value: '74', key: 'percentage_of_actors' } + + expect(response).to have_gitlab_http_status(:created) + expect(json_response).to eq( + 'name' => 'my_feature', + 'state' => 'conditional', + 'gates' => [ + { 'key' => 'boolean', 'value' => false }, + { 'key' => 'percentage_of_actors', 'value' => 74 } + ]) + end + end end end diff --git a/spec/requests/api/graphql/project/issues_spec.rb b/spec/requests/api/graphql/project/issues_spec.rb index 256b45498f6..31388b3061c 100644 --- a/spec/requests/api/graphql/project/issues_spec.rb +++ b/spec/requests/api/graphql/project/issues_spec.rb @@ -358,7 +358,7 @@ describe 'getting an issue list for a project' do cursored_query = query("sort: LABEL_PRIORITY_ASC, after: \"#{end_cursor}\"") post_graphql(cursored_query, current_user: current_user) - response_data = JSON.parse(response.body)['data']['project']['issues']['edges'] + response_data = Gitlab::Json.parse(response.body)['data']['project']['issues']['edges'] expect(grab_iids(response_data)).to eq [label_issue2.iid, label_issue4.iid] end @@ -380,7 +380,7 @@ describe 'getting an issue list for a project' do cursored_query = query("sort: LABEL_PRIORITY_DESC, after: \"#{end_cursor}\"") post_graphql(cursored_query, current_user: current_user) - response_data = JSON.parse(response.body)['data']['project']['issues']['edges'] + response_data = Gitlab::Json.parse(response.body)['data']['project']['issues']['edges'] expect(grab_iids(response_data)).to eq [label_issue1.iid, label_issue4.iid] end |
