diff options
223 files changed, 2753 insertions, 1388 deletions
diff --git a/.dockerignore b/.dockerignore index b8e239e4049..4f5b33de167 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ # the files we need to build the QA image are in these folders. # Following are the files we need: # - ./config/initializers/0_inject_enterprise_edition_module.rb +# - ./ee/app/models/license.rb # - ./lib/gitlab.rb # - ./qa/ # - ./INSTALLATION_TYPE @@ -23,7 +24,14 @@ /db/ /doc/ /docker/ -/ee/ +/ee/bin/ +/ee/changelogs/ +/ee/config/ +/ee/db/ +/ee/fixtures/ +/ee/lib/ +/ee/locale/ +/ee/spec/ /fixtures/ /templates/ /lint/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f926cbc2939..4c7a8c05b37 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,15 @@ image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-9.6-graphicsmagick-1.3.33" +stages: + - build + - prepare + - quick-test + - test + - review + - qa + - post-test + - pages + variables: RAILS_ENV: "test" NODE_ENV: "test" @@ -11,25 +21,9 @@ variables: FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json BUILD_ASSETS_IMAGE: "false" -before_script: - - date - - source scripts/utils.sh - - source scripts/prepare_build.sh - - date - after_script: - date -stages: - - build - - prepare - - merge - - test - - review - - qa - - post-test - - pages - include: - local: .gitlab/ci/global.gitlab-ci.yml - local: .gitlab/ci/cng.gitlab-ci.yml diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index dae3c349ff4..0f2dd081e9e 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -9,12 +9,13 @@ app/assets/ @ClemMakesApps @fatihacet @filipa @mikegreiling @timzallmann @kushalpandya @pslaughter *.scss @annabeldunstone @ClemMakesApps @fatihacet @filipa @mikegreiling @timzallmann @kushalpandya @pslaughter -# Maintainers from the Database team should review changes in `db/` -db/ @gl-database -lib/gitlab/background_migration/ @gl-database -lib/gitlab/database/ @gl-database -lib/gitlab/sql/ @gl-database -/ee/db/ @gl-database +# Database maintainers should review changes in `db/` +db/ @gitlab-org/maintainers/database +lib/gitlab/background_migration/ @gitlab-org/maintainers/database +lib/gitlab/database/ @gitlab-org/maintainers/database +lib/gitlab/sql/ @gitlab-org/maintainers/database +lib/gitlab/github_import/ @gitlab-org/maintainers/database +/ee/db/ @gitlab-org/maintainers/database # Feature specific owners /ee/lib/gitlab/code_owners/ @reprazent diff --git a/.gitlab/ci/cng.gitlab-ci.yml b/.gitlab/ci/cng.gitlab-ci.yml index d624e8d09f6..a43d3694103 100644 --- a/.gitlab/ci/cng.gitlab-ci.yml +++ b/.gitlab/ci/cng.gitlab-ci.yml @@ -1,16 +1,15 @@ cloud-native-image: image: ruby:2.6-alpine - before_script: [] dependencies: [] stage: post-test allow_failure: true variables: GIT_DEPTH: "1" - cache: {} when: manual script: - install_gitlab_gem - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng only: - - tags@gitlab-org/gitlab-ce - - tags@gitlab-org/gitlab-ee + refs: + - tags@gitlab-org/gitlab-ce + - tags@gitlab-org/gitlab-ee diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index 39ae62a43c9..e77c773824f 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -1,24 +1,34 @@ -.review-docs: &review-docs - extends: .single-script-job-dedicated-runner +.review-docs: + extends: + - .default-tags + - .default-retry + image: ruby:2.6-alpine + stage: review + dependencies: [] variables: - SCRIPT_NAME: trigger-build-docs + GIT_STRATEGY: none environment: name: review-docs/$CI_COMMIT_REF_SLUG # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are CI variables # Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693 url: http://$CI_ENVIRONMENT_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX on_stop: review-docs-cleanup + before_script: + # We don't clone the repo by using GIT_STRATEGY: none and only download the + # single script we need here so it's much faster than cloning. + - apk add --update openssl + - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/trigger-build-docs + - chmod 755 trigger-build-docs # Trigger a manual docs build in gitlab-docs only on non docs-only branches. # Useful to preview the docs changes live. review-docs-deploy-manual: extends: - .review-docs - - .no-docs-and-no-qa - stage: review + - .except-docs-qa script: - gem install gitlab --no-document - - ./$SCRIPT_NAME deploy + - ./trigger-build-docs deploy when: manual only: - branches@gitlab-org/gitlab-ce @@ -27,50 +37,50 @@ review-docs-deploy-manual: # Always trigger a docs build in gitlab-docs only on docs-only branches. # Useful to preview the docs changes live. review-docs-deploy: - <<: *review-docs - stage: review + extends: + - .review-docs + - .except-qa script: - gem install gitlab --no-document - - ./$SCRIPT_NAME deploy + - ./trigger-build-docs deploy only: - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-ce - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-ee - except: - - /(^qa[\/-].*|.*-qa$)/ # Cleanup remote environment of gitlab-docs review-docs-cleanup: - <<: *review-docs - stage: review + extends: + - .review-docs + - .except-qa environment: name: review-docs/$CI_COMMIT_REF_SLUG action: stop script: - gem install gitlab --no-document - - ./$SCRIPT_NAME cleanup + - ./trigger-build-docs cleanup when: manual only: - branches@gitlab-org/gitlab-ce - branches@gitlab-org/gitlab-ee docs lint: - extends: .dedicated-runner + extends: + - .default-tags + - .default-retry + - .except-qa image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint" stage: test - cache: {} dependencies: [] - before_script: [] script: - scripts/lint-doc.sh + # Lint Markdown + - markdownlint --config .markdownlint.json doc/**/*.md + # Prepare docs for build - mv doc/ /tmp/gitlab-docs/content/$DOCS_GITLAB_REPO_SUFFIX - cd /tmp/gitlab-docs - # Lint Markdown - - bundle exec mdl content/$DOCS_GITLAB_REPO_SUFFIX -c $CI_PROJECT_DIR/.mdlrc # Build HTML from Markdown - bundle exec nanoc # Check the internal links - bundle exec nanoc check internal_links # Check the internal anchor links - bundle exec nanoc check internal_anchors - except: - - /(^qa[\/-].*|.*-qa$)/ diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 5c3278fcf53..df38cb4ff8e 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -1,23 +1,19 @@ -.assets-compile-cache: &assets-compile-cache +.assets-compile-cache: cache: - key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6" paths: - vendor/ruby/ - .yarn-cache/ - tmp/cache/assets/sprockets -.use-pg: &use-pg - services: - - name: postgres:9.6.14 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - .gitlab:assets:compile-metadata: - <<: *assets-compile-cache - extends: .dedicated-no-docs-pull-cache-job + extends: + - .default-tags + - .default-retry + - .assets-compile-cache + - .default-before_script + - .except-docs image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1 - dependencies: - - setup-test-env + dependencies: ["setup-test-env"] services: - docker:19.03.0-dind variables: @@ -30,6 +26,14 @@ NODE_OPTIONS: --max_old_space_size=3584 DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 + cache: + key: "assets-compile:production:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6" + artifacts: + name: webpack-report + expire_in: 31d + paths: + - webpack-report/ + - public/assets/ script: - node --version - retry yarn install --frozen-lockfile --production --cache-folder .yarn-cache --prefer-offline @@ -42,43 +46,41 @@ - install_api_client_dependencies_with_apt - play_job "review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played - play_job "schedule:review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played - artifacts: - name: webpack-report - expire_in: 31d - paths: - - webpack-report/ - - public/assets/ only: - /.+/@gitlab-org/gitlab-ce - /.+/@gitlab-org/gitlab-ee - /.+/@gitlab/gitlabhq - /.+/@gitlab/gitlab-ee tags: - - docker - gitlab-org + - docker gitlab:assets:compile: extends: .gitlab:assets:compile-metadata + only: + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee cache: policy: pull-push - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee gitlab:assets:compile pull-cache: extends: .gitlab:assets:compile-metadata - cache: - policy: pull except: refs: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee - /(^docs[\/-].+|.+-docs$)/ + cache: + policy: pull .compile-assets-metadata: - extends: .dedicated-runner - <<: *use-pg - <<: *assets-compile-cache + extends: + - .default-tags + - .default-retry + - .assets-compile-cache + - .default-before_script + - .use-pg stage: prepare script: - node --version @@ -89,6 +91,8 @@ gitlab:assets:compile pull-cache: variables: # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 + cache: + key: "assets-compile:test:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6" artifacts: expire_in: 7d paths: @@ -96,30 +100,34 @@ gitlab:assets:compile pull-cache: - public/assets compile-assets: - extends: .compile-assets-metadata + extends: + - .compile-assets-metadata + only: + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee cache: policy: pull-push - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee compile-assets pull-cache: extends: .compile-assets-metadata - cache: - policy: pull except: refs: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee - /(^docs[\/-].+|.+-docs$)/ + cache: + policy: pull karma: - extends: .dedicated-no-docs-pull-cache-job - <<: *use-pg - dependencies: - - compile-assets - - compile-assets pull-cache - - setup-test-env + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs + dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"] variables: # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 @@ -142,12 +150,14 @@ karma: junit: junit_karma.xml jest: - extends: .dedicated-no-docs-and-no-qa-pull-cache-job - <<: *use-pg - dependencies: - - compile-assets - - compile-assets pull-cache - - setup-test-env + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"] script: - scripts/gitaly-test-spawn - date @@ -170,36 +180,41 @@ jest: - tmp/jest/jest/ policy: pull-push -qa:internal: - extends: .dedicated-no-docs-no-db-pull-cache-job - services: [] - script: +.qa: + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + dependencies: ["setup-test-env"] + variables: + SETUP_DB: "false" + before_script: - cd qa/ - bundle install + +qa:internal: + extends: .qa + script: - bundle exec rspec - dependencies: - - setup-test-env qa:selectors: - extends: .dedicated-no-docs-no-db-pull-cache-job - services: [] + extends: .qa script: - - cd qa/ - - bundle install - bundle exec bin/qa Test::Sanity::Selectors - dependencies: - - setup-test-env -.qa-frontend-node: &qa-frontend-node - extends: .dedicated-no-docs-no-db-pull-cache-job - stage: test +.qa-frontend-node: + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + dependencies: [] cache: key: "$CI_JOB_NAME" paths: - .yarn-cache/ policy: pull-push - dependencies: [] - before_script: [] script: - date - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline @@ -207,23 +222,28 @@ qa:selectors: - yarn run webpack-prod qa-frontend-node:8: - <<: *qa-frontend-node + extends: .qa-frontend-node image: node:carbon qa-frontend-node:10: - <<: *qa-frontend-node + extends: .qa-frontend-node image: node:dubnium qa-frontend-node:latest: - <<: *qa-frontend-node + extends: .qa-frontend-node image: node:latest allow_failure: true lint:javascript:report: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + variables: + SETUP_DB: "false" stage: post-test dependencies: [] - before_script: [] script: - date - yarn run eslint-report || true # ignore exit code @@ -234,12 +254,15 @@ lint:javascript:report: - eslint-report.html jsdoc: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + variables: + SETUP_DB: "false" stage: post-test - dependencies: - - compile-assets - - compile-assets pull-cache - before_script: [] + dependencies: ["compile-assets", "compile-assets pull-cache"] script: - date - yarn run jsdoc || true # ignore exit code diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 56fe9739d5f..04135447ca4 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -1,83 +1,56 @@ -.dedicated-runner: +.default-tags: + tags: + - gitlab-org + +.default-retry: retry: max: 2 # This is confusing but this means "3 runs at max". when: - unknown_failure - api_failure - runner_system_failure - tags: - - gitlab-org - -.default-cache: &default-cache - key: "debian-stretch-ruby-2.6.3-node-12.x" - paths: - - vendor/ruby - - .yarn-cache/ - - vendor/gitaly-ruby -.dedicated-runner-default-cache: - extends: .dedicated-runner - cache: - <<: *default-cache +.default-before_script: + before_script: + - date + - source scripts/utils.sh + - source scripts/prepare_build.sh + - date # Jobs that only need to pull cache -.dedicated-pull-cache-job: - extends: .dedicated-runner +.default-cache: cache: - <<: *default-cache + key: "debian-stretch-ruby-2.6.3-node-12.x" + paths: + - vendor/ruby + - .yarn-cache/ + - vendor/gitaly-ruby policy: pull - stage: test -.no-docs: +.except-docs: except: refs: - /(^docs[\/-].+|.+-docs$)/ -.no-docs-and-no-qa: +.except-qa: except: refs: - - /(^docs[\/-].+|.+-docs$)/ - /(^qa[\/-].*|.*-qa$)/ -.dedicated-no-docs-pull-cache-job: - extends: - - .dedicated-pull-cache-job - - .no-docs - -.dedicated-no-docs-and-no-qa-pull-cache-job: - extends: - - .dedicated-pull-cache-job - - .no-docs-and-no-qa - -# Jobs that do not need a DB -.dedicated-no-docs-no-db-pull-cache-job: - extends: .dedicated-no-docs-pull-cache-job - variables: - SETUP_DB: "false" - -# Jobs that need a dedicated runner, with no cache -.dedicated-no-docs: - extends: - - .dedicated-runner - - .no-docs +.except-docs-qa: + except: + refs: + - /(^docs[\/-].+|.+-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ -.single-script-job-dedicated-runner: - extends: .dedicated-runner - image: ruby:2.6-alpine - stage: test - cache: {} - dependencies: [] - variables: - GIT_STRATEGY: none - before_script: - # We don't clone the repo by using GIT_STRATEGY: none and only download the - # single script we need here so it's much faster than cloning. - - export SCRIPT_NAME="${SCRIPT_NAME:-$CI_JOB_NAME}" - - apk add --update openssl - - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/$SCRIPT_NAME - - chmod 755 $(basename $SCRIPT_NAME) +.except-docs-qa-geo: + except: + refs: + - /(^docs[\/-].+|.+-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ + - /(^geo[\/-].*|.*-geo$)/ -.review-only: &review-only +.review-only: only: refs: - branches@gitlab-org/gitlab-ce @@ -88,3 +61,16 @@ - master - /^\d+-\d+-auto-deploy-\d+$/ - /(^docs[\/-].+|.+-docs$)/ + +.use-pg: + services: + - name: postgres:9.6.14 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:alpine + +.use-pg-10: + image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33" + services: + - name: postgres:10.9 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:alpine diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml index 9923732e587..1936933cca4 100644 --- a/.gitlab/ci/memory.gitlab-ci.yml +++ b/.gitlab/ci/memory.gitlab-ci.yml @@ -1,5 +1,12 @@ memory-static: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs + variables: + SETUP_DB: "false" script: # Uses two different reports from the 'derailed_benchmars' gem. @@ -23,7 +30,13 @@ memory-static: # The application is booted in `production` environment. # All tests are run without a webserver (directly using Rack::Mock by default). memory-on-boot: - extends: .rspec-metadata-pg-10 + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg-10 + - .except-docs-qa variables: NODE_ENV: "production" RAILS_ENV: "production" diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index f7b18b809b4..3247d7c4bce 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -1,13 +1,15 @@ pages: - extends: .dedicated-no-docs-no-db-pull-cache-job - before_script: [] + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + only: + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee stage: pages - dependencies: - - coverage - - karma - - gitlab:assets:compile - - lint:javascript:report - - jsdoc + dependencies: ["coverage", "karma", "gitlab:assets:compile", "lint:javascript:report", "jsdoc"] script: - mv public/ .public/ - mkdir public/ @@ -21,6 +23,3 @@ pages: artifacts: paths: - public - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 144e5392e55..ac2a70dda0b 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -1,9 +1,8 @@ .package-and-qa-base: image: ruby:2.6-alpine - stage: review # So even if review-deploy failed we can still run this - before_script: [] + stage: qa + needs: ["build-qa-image", "gitlab:assets:compile pull-cache"] dependencies: [] - cache: {} variables: GIT_DEPTH: "1" retry: 0 @@ -14,12 +13,18 @@ only: - branches@gitlab-org/gitlab-ce - branches@gitlab-org/gitlab-ee + except: + - master package-and-qa-manual: extends: - .package-and-qa-base - - .no-docs-and-no-qa + - .except-docs-qa when: manual + except: + - master + - /(^docs[\/-].+|.+-docs$)/ + - /(^qa[\/-].*|.*-qa$) package-and-qa: extends: .package-and-qa-base diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 50476b43dd6..2e8b197829b 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -1,52 +1,31 @@ -.use-pg: &use-pg - services: - - name: postgres:9.6.14 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - -.use-pg-10: &use-pg-10 - services: - - name: postgres:10.9 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - -.only-schedules-master: &only-schedules-master +.only-schedules-master: only: - - schedules@gitlab-org/gitlab-ce - - schedules@gitlab-org/gitlab-ee - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - master@gitlab/gitlabhq - - master@gitlab/gitlab-ee + refs: + - schedules@gitlab-org/gitlab-ce + - schedules@gitlab-org/gitlab-ee + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - master@gitlab/gitlabhq + - master@gitlab/gitlab-ee -.gitlab-setup: &gitlab-setup +.rake-exec: extends: - - .dedicated-no-docs-and-no-qa-pull-cache-job - - .use-pg + - .default-tags + - .default-retry + - .default-cache + - .default-before_script variables: SETUP_DB: "false" script: - # Manually clone gitlab-test and only seed this project in - # db/fixtures/development/04_project.rb thanks to SIZE=1 below - - git clone https://gitlab.com/gitlab-org/gitlab-test.git - /home/git/repositories/gitlab-org/gitlab-test.git - - scripts/gitaly-test-spawn - - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup - artifacts: - when: on_failure - expire_in: 1d - paths: - - log/development.log - -.rake-exec: &rake-exec - extends: .dedicated-no-docs-no-db-pull-cache-job - script: - bundle exec rake $CI_JOB_NAME -.rspec-metadata: &rspec-metadata +.rspec-base: extends: - - .dedicated-pull-cache-job - - .no-docs-and-no-qa + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs-qa stage: test script: - JOB_NAME=( $CI_JOB_NAME ) @@ -83,52 +62,24 @@ reports: junit: junit_rspec.xml -.rspec-metadata-pg: &rspec-metadata-pg - <<: *rspec-metadata - <<: *use-pg - -.rspec-metadata-pg-10: &rspec-metadata-pg-10 - <<: *rspec-metadata - <<: *use-pg-10 - image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33" - -# DB migration, rollback, and seed jobs -.db-migrate-reset: &db-migrate-reset - extends: .dedicated-no-docs-and-no-qa-pull-cache-job - script: - - bundle exec rake db:migrate:reset - dependencies: - - setup-test-env +.rspec-base-pg: + extends: + - .rspec-base + - .use-pg -.migration-paths: &migration-paths - extends: .dedicated-no-docs-and-no-qa-pull-cache-job - variables: - SETUP_DB: "false" - script: - - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v11.11.0 - - git checkout -f FETCH_HEAD - - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile - - sed -i "s/gem 'bootsnap', '~> 1.0.0'/gem 'bootsnap'/" Gemfile - - bundle update google-protobuf grpc bootsnap - - bundle install $BUNDLE_INSTALL_FLAGS - - date - - cp config/gitlab.yml.example config/gitlab.yml - - bundle exec rake db:drop db:create db:schema:load db:seed_fu - - date - - git checkout -f $CI_COMMIT_SHA - - bundle install $BUNDLE_INSTALL_FLAGS - - date - - . scripts/prepare_build.sh - - date - - bundle exec rake db:migrate - dependencies: - - setup-test-env +.rspec-base-pg-10: + extends: + - .rspec-base + - .use-pg-10 setup-test-env: extends: - - .dedicated-runner-default-cache - - .no-docs + - .default-tags + - .default-retry + - .default-cache + - .default-before_script - .use-pg + - .except-docs stage: prepare script: - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init' @@ -141,67 +92,72 @@ setup-test-env: - vendor/gitaly-ruby rspec unit pg: - <<: *rspec-metadata-pg + extends: .rspec-base-pg parallel: 20 rspec integration pg: - <<: *rspec-metadata-pg + extends: .rspec-base-pg parallel: 6 rspec system pg: - <<: *rspec-metadata-pg + extends: .rspec-base-pg parallel: 24 rspec unit pg-10: - <<: *rspec-metadata-pg-10 - <<: *only-schedules-master + extends: + - .rspec-base-pg-10 + - .only-schedules-master parallel: 20 rspec integration pg-10: - <<: *rspec-metadata-pg-10 - <<: *only-schedules-master + extends: + - .rspec-base-pg-10 + - .only-schedules-master parallel: 6 rspec system pg-10: - <<: *rspec-metadata-pg-10 - <<: *only-schedules-master + extends: + - .rspec-base-pg-10 + - .only-schedules-master parallel: 24 rspec-fast-spec-helper: - <<: *rspec-metadata-pg + extends: .rspec-base-pg script: - bundle exec rspec spec/fast_spec_helper.rb -.rspec-quarantine: &rspec-quarantine - <<: *only-schedules-master +rspec quarantine pg: + extends: + - .default-before_script + - .rspec-base-pg + - .only-schedules-master script: - - export CACHE_CLASSES=true + - export NO_KNAPSACK=1 CACHE_CLASSES=true - scripts/gitaly-test-spawn - bin/rspec --color --format documentation --tag quarantine -- spec/ - -rspec quarantine pg: - <<: *rspec-metadata-pg - <<: *rspec-quarantine allow_failure: true static-analysis: - extends: .dedicated-no-docs-no-db-pull-cache-job - dependencies: - - compile-assets - - compile-assets pull-cache - - setup-test-env + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs + dependencies: ["setup-test-env", "compile-assets", "compile-assets pull-cache"] + variables: + SETUP_DB: "false" script: - scripts/static-analysis cache: - key: "debian-stretch-ruby-2.6.3-node-12.x-and-rubocop" + key: "debian-stretch-ruby-2.6.3-and-rubocop" paths: - vendor/ruby - - .yarn-cache/ - tmp/rubocop_cache policy: pull-push downtime_check: - <<: *rake-exec + extends: .rake-exec except: refs: - master @@ -209,22 +165,20 @@ downtime_check: - /^[\d-]+-stable(-ee)?$/ - /(^docs[\/-].+|.+-docs$)/ - /(^qa[\/-].*|.*-qa$)/ - dependencies: - - setup-test-env + dependencies: ["setup-test-env"] ee_compat_check: - <<: *rake-exec + extends: .rake-exec dependencies: [] except: refs: - master - tags - - /[\d-]+-stable(-ee)?/ - - /^security-/ - branches@gitlab-org/gitlab-ee - branches@gitlab/gitlab-ee + - /^[\d-]+-stable(-ee)?$/ - /(^docs[\/-].+|.+-docs$)/ - retry: 0 + - /^security-/ artifacts: name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}" when: always @@ -232,44 +186,106 @@ ee_compat_check: paths: - ee_compat_check/patches/*.patch -db:migrate:reset-pg: - <<: *db-migrate-reset - <<: *use-pg +# DB migration, rollback, and seed jobs +db:migrate:reset: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] + script: + - bundle exec rake db:migrate:reset -db:check-schema-pg: - <<: *db-migrate-reset - <<: *use-pg +db:check-schema: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] script: - source scripts/schema_changed.sh -migration:path-pg: - <<: *migration-paths - <<: *use-pg +db:migrate-from-v11.11.0: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] + variables: + SETUP_DB: "false" + script: + - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v11.11.0 + - git checkout -f FETCH_HEAD + - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile + - sed -i "s/gem 'bootsnap', '~> 1.0.0'/gem 'bootsnap'/" Gemfile + - bundle update google-protobuf grpc bootsnap + - bundle install $BUNDLE_INSTALL_FLAGS + - date + - cp config/gitlab.yml.example config/gitlab.yml + - bundle exec rake db:drop db:create db:schema:load db:seed_fu + - date + - git checkout -f $CI_COMMIT_SHA + - bundle install $BUNDLE_INSTALL_FLAGS + - date + - . scripts/prepare_build.sh + - date + - bundle exec rake db:migrate -.db-rollback: &db-rollback - extends: .dedicated-no-docs-and-no-qa-pull-cache-job +db:rollback: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] script: - bundle exec rake db:migrate VERSION=20180101160629 - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true - dependencies: - - setup-test-env - -db:rollback-pg: - <<: *db-rollback - <<: *use-pg -gitlab:setup-pg: - <<: *gitlab-setup - <<: *use-pg - dependencies: - - setup-test-env +gitlab:setup: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] + variables: + SETUP_DB: "false" + script: + # Manually clone gitlab-test and only seed this project in + # db/fixtures/development/04_project.rb thanks to SIZE=1 below + - git clone https://gitlab.com/gitlab-org/gitlab-test.git + /home/git/repositories/gitlab-org/gitlab-test.git + - scripts/gitaly-test-spawn + - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup + artifacts: + when: on_failure + expire_in: 1d + paths: + - log/development.log coverage: # Don't include dedicated-no-docs-no-db-pull-cache-job here since we need to # download artifacts from all the rspec jobs instead of from setup-test-env only extends: - - .dedicated-runner-default-cache - - .no-docs-and-no-qa + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs-qa cache: policy: pull variables: diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index ccd7d6ec84d..905f1ede2c4 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -4,35 +4,29 @@ include: - template: Security/Dependency-Scanning.gitlab-ci.yml - template: Security/DAST.gitlab-ci.yml +.reports: + extends: + - .default-retry + - .except-docs + tags: + - gitlab-org + - docker + code_quality: - extends: .dedicated-no-docs - # gitlab-org runners set `privileged: false` but we need to have it set to true - # since we're using Docker in Docker - tags: [] - before_script: [] - cache: {} + extends: .reports sast: - extends: .dedicated-no-docs - tags: [] - before_script: [] - cache: {} + extends: .reports variables: SAST_BRAKEMAN_LEVEL: 2 + SAST_EXCLUDED_PATHS: qa,spec,doc dependency_scanning: - extends: .dedicated-no-docs - tags: [] - before_script: [] - cache: {} + extends: .reports dast: - extends: - - .dedicated-runner - - .review-only + extends: .reports stage: qa - dependencies: - - review-deploy + dependencies: ["review-deploy"] before_script: - export DAST_WEBSITE="$(cat review_app_url.txt)" - cache: {} diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index beb049c0b3b..3415f1b6ab4 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -1,4 +1,4 @@ -.review-schedules-only: &review-schedules-only +.review-schedules-only: only: refs: - schedules@gitlab-org/gitlab-ce @@ -11,39 +11,39 @@ - tags - /(^docs[\/-].+|.+-docs$)/ -.review-base: &review-base +.review-base: extends: - - .dedicated-runner + - .default-tags + - .default-retry - .review-only image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base - cache: {} dependencies: [] before_script: - source scripts/utils.sh -.review-docker: &review-docker - <<: *review-base +.review-docker: + extends: .review-base image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine services: - docker:19.03.0-dind tags: - gitlab-org - docker - variables: &review-docker-variables + variables: DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 LATEST_QA_IMAGE: "gitlab/${CI_PROJECT_NAME}-qa:nightly" QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/${CI_PROJECT_NAME}-qa:${CI_COMMIT_REF_SLUG}" build-qa-image: - <<: *review-docker + extends: .review-docker stage: test script: - time docker build --cache-from ${LATEST_QA_IMAGE} --tag ${QA_IMAGE} --file ./qa/Dockerfile ./ - echo "${CI_JOB_TOKEN}" | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY} - time docker push ${QA_IMAGE} -.review-build-cng-base: &review-build-cng-base +.review-build-cng-base: image: ruby:2.6-alpine stage: test when: manual @@ -52,20 +52,21 @@ build-qa-image: - install_api_client_dependencies_with_apk - install_gitlab_gem dependencies: [] - cache: {} script: - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng review-build-cng: - extends: .review-only - <<: *review-build-cng-base + extends: + - .review-build-cng-base + - .review-only schedule:review-build-cng: - <<: *review-schedules-only - <<: *review-build-cng-base + extends: + - .review-build-cng-base + - .review-schedules-only -.review-deploy-base: &review-deploy-base - <<: *review-base +review-deploy: + extends: .review-base allow_failure: true retry: 1 stage: review @@ -73,7 +74,7 @@ schedule:review-build-cng: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" GITLAB_HELM_CHART_REF: "master" - environment: &review-environment + environment: name: review/${CI_COMMIT_REF_NAME} url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} on_stop: review-stop @@ -98,50 +99,45 @@ schedule:review-build-cng: expire_in: 2 days when: always -review-deploy: - <<: *review-deploy-base - schedule:review-deploy: - <<: *review-deploy-base - <<: *review-schedules-only + extends: + - review-deploy + - .review-schedules-only review-stop: - extends: - - .single-script-job-dedicated-runner - - .review-only - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base - stage: review + extends: review-deploy when: manual - allow_failure: true - variables: - SCRIPT_NAME: review_apps/review-apps.sh environment: - <<: *review-environment action: stop - script: + variables: + GIT_STRATEGY: none + before_script: + # We don't clone the repo by using GIT_STRATEGY: none and only download the + # single script we need here so it's much faster than cloning. + - apk add --update openssl + - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/review_apps/review-apps.sh - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/utils.sh - source utils.sh - - source $(basename $SCRIPT_NAME) + - source review-apps.sh + script: - delete + artifacts: {} -.review-qa-base: &review-qa-base - <<: *review-docker - allow_failure: true +.review-qa-base: + extends: .review-docker retry: 2 stage: qa variables: - <<: *review-docker-variables QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" QA_CAN_TEST_GIT_PROTOCOL_V2: "false" + QA_DEBUG: "true" GITLAB_USERNAME: "root" GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" GITLAB_ADMIN_USERNAME: "root" GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" - QA_DEBUG: "true" - dependencies: - - review-deploy + dependencies: ["review-deploy"] artifacts: paths: - ./qa/gitlab-qa-run-* @@ -156,12 +152,13 @@ review-stop: - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}} review-qa-smoke: - <<: *review-qa-base + extends: .review-qa-base + allow_failure: true script: - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" review-qa-all: - <<: *review-qa-base + extends: .review-qa-base allow_failure: true when: manual parallel: 5 @@ -172,21 +169,16 @@ review-qa-all: parallel-spec-reports: extends: - - .dedicated-runner - - .no-docs - dependencies: - - review-qa-all + - .default-tags + - .except-docs image: ruby:2.6-alpine - services: [] - before_script: [] + stage: post-test + dependencies: ["review-qa-all"] variables: - SETUP_DB: "false" NEW_PARALLEL_SPECS_REPORT: qa/report-new.html BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/" - stage: post-test allow_failure: true when: manual - retry: 0 artifacts: when: always paths: @@ -196,15 +188,15 @@ parallel-spec-reports: junit: qa/gitlab-qa-run-*/**/rspec-*.xml script: - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/* - - gem install nokogiri + - gem install nokogiri --no-document - cd qa/gitlab-qa-run-*/gitlab-* - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_) - - cd ../../.. + - cd - - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}' - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm -.review-performance-base: &review-performance-base - <<: *review-qa-base +review-performance: + extends: .review-qa-base allow_failure: true before_script: - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)" @@ -222,18 +214,16 @@ parallel-spec-reports: reports: performance: performance.json -review-performance: - <<: *review-performance-base - schedule:review-performance: - <<: *review-performance-base - <<: *review-schedules-only - dependencies: - - schedule:review-deploy + extends: + - review-performance + - .review-schedules-only + dependencies: ["schedule:review-deploy"] schedule:review-cleanup: - <<: *review-base - <<: *review-schedules-only + extends: + - .review-base + - .review-schedules-only stage: build allow_failure: true environment: @@ -246,11 +236,13 @@ schedule:review-cleanup: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb danger-review: - extends: .dedicated-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger stage: test dependencies: [] - before_script: [] only: variables: - $DANGER_GITLAB_API_TOKEN @@ -259,9 +251,8 @@ danger-review: - master - /^\d+-\d+-auto-deploy-\d+$/ - /^[\d-]+-stable(-ee)?$/ - variables: - - $CI_COMMIT_REF_NAME =~ /^ce-to-ee-.*/ - - $CI_COMMIT_REF_NAME =~ /.*-stable(-ee)?-prepare-.*/ + - /^ce-to-ee-.*/ + - /.*-stable(-ee)?-prepare-.*/ script: - git version - node --version diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index c1fc3a893ca..d9384780356 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -1,41 +1,42 @@ # Insurance in case a gem needed by one of our releases gets yanked from # rubygems.org in the future. cache gems: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs + dependencies: ["setup-test-env"] + variables: + SETUP_DB: "false" script: - bundle package --all --all-platforms artifacts: paths: - vendor/cache only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - tags - dependencies: - - setup-test-env + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - tags -gitlab_git_test: +.minimal-job: extends: - - .dedicated-runner - - .no-docs-and-no-qa - variables: - SETUP_DB: "false" - before_script: [] + - .default-tags + - .default-retry + - .except-docs-qa dependencies: [] - cache: {} + +gitlab_git_test: + extends: .minimal-job script: - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes no_ee_check: - extends: - - .dedicated-runner - - .no-docs-and-no-qa - variables: - SETUP_DB: "false" - before_script: [] - dependencies: [] - cache: {} + extends: .minimal-job script: - scripts/no-ee-check only: - - /.+/@gitlab-org/gitlab-ce + refs: + - branches@gitlab-org/gitlab-ce diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index 4c97a4feb18..b9dac64957e 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -1,5 +1,4 @@ -.tests-metadata-state: &tests-metadata-state - extends: .dedicated-runner +.tests-metadata-state: variables: TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache" before_script: @@ -14,7 +13,7 @@ retrieve-tests-metadata: extends: - .tests-metadata-state - - .no-docs-and-no-qa + - .except-docs-qa stage: prepare cache: key: tests_metadata @@ -29,7 +28,7 @@ retrieve-tests-metadata: - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}' update-tests-metadata: - <<: *tests-metadata-state + extends: .tests-metadata-state stage: post-test cache: key: tests_metadata @@ -49,25 +48,24 @@ update-tests-metadata: - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json - scripts/insert-rspec-profiling-data only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - master@gitlab/gitlabhq - - master@gitlab/gitlab-ee + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - master@gitlab/gitlabhq + - master@gitlab/gitlab-ee flaky-examples-check: - extends: .dedicated-runner + extends: + - .default-tags + - .default-retry image: ruby:2.6-alpine - services: [] - before_script: [] + stage: post-test variables: - SETUP_DB: "false" - USE_BUNDLE_INSTALL: "false" NEW_FLAKY_SPECS_REPORT: rspec_flaky/report-new.json - stage: post-test allow_failure: true - retry: 0 only: - - branches + refs: + - branches except: refs: - master diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml index b7aa418d8f7..3e107b475c9 100644 --- a/.gitlab/ci/yaml.gitlab-ci.yml +++ b/.gitlab/ci/yaml.gitlab-ci.yml @@ -1,9 +1,10 @@ # Yamllint of *.yml for .gitlab-ci.yml. # This uses rules from project root `.yamllint`. lint-ci-gitlab: - extends: .dedicated-runner - before_script: [] - dependencies: [] + extends: + - .default-tags + - .default-retry image: sdesbure/yamllint:latest + dependencies: [] script: - yamllint .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates changelogs diff --git a/.markdownlint.json b/.markdownlint.json new file mode 100644 index 00000000000..2c40c0859f0 --- /dev/null +++ b/.markdownlint.json @@ -0,0 +1,31 @@ +{ + "default": true, + "first-header-h1": true, + "header-style": { + "style": "atx" + }, + "ul-style": { + "style": "dash" + }, + "line-length": false, + "commands-show-output": false, + "no-duplicate-header": { + "allow_different_nesting": true + }, + "no-trailing-punctuation": { + "punctuation": ".,;:!。，；：！？" + }, + "ol-prefix": { + "style": "one" + }, + "no-inline-html": false, + "hr-style": { + "style": "---" + }, + "no-emphasis-as-heading": false, + "fenced-code-language": false, + "first-line-h1": false, + "code-block-style": { + "style": "fenced" + } +} diff --git a/.mdlrc b/.mdlrc deleted file mode 100644 index 151c54f7d44..00000000000 --- a/.mdlrc +++ /dev/null @@ -1,7 +0,0 @@ -# This is the options file for mdl, configured in .gitlab/ci/docs.gitlab-ci.yml, -# and related to the style file ./mdlrc.style - -# See https://github.com/markdownlint/markdownlint/blob/master/docs/configuration.md - -ignore_front_matter true -style File.expand_path('.mdlrc.style', __dir__) diff --git a/.mdlrc.style b/.mdlrc.style deleted file mode 100644 index 85bc3aaa99b..00000000000 --- a/.mdlrc.style +++ /dev/null @@ -1,32 +0,0 @@ -# This is the style file for mdl, configured in .gitlab/ci/docs.gitlab-ci.yml, -# and related to the options file ./mdlrc - -# See https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md -# for more detailed information on the rules and styles. - -rule "MD001" -rule "MD002" -rule "MD003", :style => :atx -rule "MD006" -rule "MD010" -rule "MD011" -rule "MD012" -rule "MD019" -rule "MD022" -rule "MD023" -rule "MD025" -rule "MD028" -rule "MD029", :style => :one -rule "MD030" -# rule "MD032" -rule "MD034" -rule "MD037" -rule "MD038" - -# Should not be used currently: - -# rule "MD004", :style => :dash # unordered list style - dash -# False positives, see https://github.com/markdownlint/markdownlint/issues/261 - -# rule "MD039" # Spaces inside link text -# Crashes when link text has certain punctuation diff --git a/.rubocop.yml b/.rubocop.yml index b75c63e1f58..012f4890c33 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -178,6 +178,9 @@ Gitlab/ModuleWithInstanceVariables: Gitlab/HTTParty: Enabled: true + Exclude: + - 'spec/**/*' + - 'ee/spec/**/*' GitlabSecurity/PublicSend: Enabled: true @@ -211,3 +214,54 @@ ActiveRecordAssociationReload: Exclude: - 'spec/**/*' - 'ee/spec/**/*' + +RSpec/FactoriesInMigrationSpecs: + Enabled: true + Include: + - 'spec/migrations/**/*.rb' + - 'ee/spec/migrations/**/*.rb' + - 'spec/lib/gitlab/background_migration/**/*.rb' + - 'ee/spec/lib/gitlab/background_migration/**/*.rb' + +Cop/IncludeActionViewContext: + Enabled: true + Exclude: + - 'spec/**/*' + - 'ee/spec/**/*' + +Cop/IncludeSidekiqWorker: + Enabled: true + Exclude: + - 'spec/**/*' + - 'ee/spec/**/*' + +Gitlab/Union: + Enabled: true + Exclude: + - 'spec/**/*' + - 'ee/spec/**/*' + +Cop/SidekiqOptionsQueue: + Enabled: true + Exclude: + - 'spec/**/*.rb' + - 'ee/spec/**/*.rb' + +Graphql/AuthorizeTypes: + Enabled: true + Exclude: + - 'spec/**/*.rb' + - 'ee/spec/**/*.rb' + +RSpec/EnvAssignment: + Enable: true + Include: + - 'spec/**/*.rb' + - 'ee/spec/**/*.rb' + Exclude: + - 'spec/**/fast_spec_helper.rb' + - 'ee/spec/**/fast_spec_helper.rb' + - 'spec/**/rails_helper.rb' + - 'ee/spec/**/rails_helper.rb' + - 'spec/**/spec_helper.rb' + - 'ee/spec/**/spec_helper.rb' @@ -83,7 +83,7 @@ gem 'grape-entity', '~> 0.7.1' gem 'rack-cors', '~> 1.0.0', require: 'rack/cors' # GraphQL API -gem 'graphql', '~> 1.8.0' +gem 'graphql', '= 1.8.4' gem 'graphiql-rails', '~> 1.4.10' gem 'apollo_upload_server', '~> 2.0.0.beta3' gem 'graphql-docs', '~> 1.6.0', group: [:development, :test] @@ -399,7 +399,7 @@ gem 'mail_room', '~> 0.9.1' gem 'email_reply_trimmer', '~> 0.1' gem 'html2text' -gem 'ruby-prof', '~> 0.17.0' +gem 'ruby-prof', '~> 1.0.0' gem 'rbtrace', '~> 0.4', require: false gem 'memory_profiler', '~> 0.9', require: false gem 'benchmark-memory', '~> 0.1', require: false @@ -438,6 +438,7 @@ gem 'toml-rb', '~> 1.0.0', require: false gem 'flipper', '~> 0.13.0' gem 'flipper-active_record', '~> 0.13.0' gem 'flipper-active_support_cache_store', '~> 0.13.0' +gem 'unleash', '~> 0.1.5' # Structured logging gem 'lograge', '~> 0.5' diff --git a/Gemfile.lock b/Gemfile.lock index d9a32027f3c..ae0d0e78c37 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -375,7 +375,7 @@ GEM graphiql-rails (1.4.10) railties sprockets-rails - graphql (1.8.1) + graphql (1.8.4) graphql-docs (1.6.0) commonmarker (~> 0.16) escape_utils (~> 1.2) @@ -535,6 +535,7 @@ GEM multi_json (1.13.1) multi_xml (0.6.0) multipart-post (2.0.0) + murmurhash3 (0.1.6) mustermann (1.0.3) mustermann-grape (1.0.0) mustermann (~> 1.0.0) @@ -836,7 +837,7 @@ GEM i18n ruby-fogbugz (0.2.1) crack (~> 0.4) - ruby-prof (0.17.0) + ruby-prof (1.0.0) ruby-progressbar (1.10.1) ruby-saml (1.7.2) nokogiri (>= 1.5.10) @@ -972,6 +973,8 @@ GEM get_process_mem (~> 0) unicorn (>= 4, < 6) uniform_notifier (1.10.0) + unleash (0.1.5) + murmurhash3 (~> 0.1.6) unparser (0.4.5) abstract_type (~> 0.0.7) adamantium (~> 0.2.0) @@ -1115,7 +1118,7 @@ DEPENDENCIES grape-path-helpers (~> 1.1) grape_logging (~> 1.7) graphiql-rails (~> 1.4.10) - graphql (~> 1.8.0) + graphql (= 1.8.4) graphql-docs (~> 1.6.0) grpc (~> 1.19.0) haml_lint (~> 0.31.0) @@ -1215,7 +1218,7 @@ DEPENDENCIES rubocop-performance (~> 1.1.0) rubocop-rspec (~> 1.22.1) ruby-fogbugz (~> 0.2.1) - ruby-prof (~> 0.17.0) + ruby-prof (~> 1.0.0) ruby-progressbar ruby_parser (~> 3.8) rubyzip (~> 1.2.2) @@ -1251,6 +1254,7 @@ DEPENDENCIES unf (~> 0.1.4) unicorn (~> 5.4.1) unicorn-worker-killer (~> 0.4.4) + unleash (~> 0.1.5) validates_hostname (~> 1.0.6) version_sorter (~> 2.2.4) vmstat (~> 2.3.0) diff --git a/app/assets/javascripts/lib/utils/http_status.js b/app/assets/javascripts/lib/utils/http_status.js index 37ad1676f7a..5e5d10883a3 100644 --- a/app/assets/javascripts/lib/utils/http_status.js +++ b/app/assets/javascripts/lib/utils/http_status.js @@ -19,6 +19,7 @@ const httpStatusCodes = { UNAUTHORIZED: 401, FORBIDDEN: 403, NOT_FOUND: 404, + GONE: 410, UNPROCESSABLE_ENTITY: 422, }; diff --git a/app/assets/javascripts/notes/components/noteable_note.vue b/app/assets/javascripts/notes/components/noteable_note.vue index 2f201839d45..9019f0542b6 100644 --- a/app/assets/javascripts/notes/components/noteable_note.vue +++ b/app/assets/javascripts/notes/components/noteable_note.vue @@ -14,6 +14,7 @@ import NoteBody from './note_body.vue'; import eventHub from '../event_hub'; import noteable from '../mixins/noteable'; import resolvable from '../mixins/resolvable'; +import httpStatusCodes from '~/lib/utils/http_status'; export default { name: 'NoteableNote', @@ -122,7 +123,13 @@ export default { }, methods: { - ...mapActions(['deleteNote', 'updateNote', 'toggleResolveNote', 'scrollToNoteIfNeeded']), + ...mapActions([ + 'deleteNote', + 'removeNote', + 'updateNote', + 'toggleResolveNote', + 'scrollToNoteIfNeeded', + ]), editHandler() { this.isEditing = true; this.$emit('handleEdit'); @@ -185,15 +192,21 @@ export default { this.updateSuccess(); callback(); }) - .catch(() => { - this.isRequesting = false; - this.isEditing = true; - this.$nextTick(() => { - const msg = __('Something went wrong while editing your comment. Please try again.'); - Flash(msg, 'alert', this.$el); - this.recoverNoteContent(noteText); + .catch(response => { + if (response.status === httpStatusCodes.GONE) { + this.removeNote(this.note); + this.updateSuccess(); callback(); - }); + } else { + this.isRequesting = false; + this.isEditing = true; + this.$nextTick(() => { + const msg = __('Something went wrong while editing your comment. Please try again.'); + Flash(msg, 'alert', this.$el); + this.recoverNoteContent(noteText); + callback(); + }); + } }); }, formCancelHandler(shouldConfirm, isDirty) { diff --git a/app/assets/javascripts/notes/stores/actions.js b/app/assets/javascripts/notes/stores/actions.js index b7857997d42..411bd585672 100644 --- a/app/assets/javascripts/notes/stores/actions.js +++ b/app/assets/javascripts/notes/stores/actions.js @@ -61,18 +61,22 @@ export const updateDiscussion = ({ commit, state }, discussion) => { return utils.findNoteObjectById(state.discussions, discussion.id); }; -export const deleteNote = ({ commit, dispatch, state }, note) => - axios.delete(note.path).then(() => { - const discussion = state.discussions.find(({ id }) => id === note.discussion_id); +export const removeNote = ({ commit, dispatch, state }, note) => { + const discussion = state.discussions.find(({ id }) => id === note.discussion_id); - commit(types.DELETE_NOTE, note); + commit(types.DELETE_NOTE, note); - dispatch('updateMergeRequestWidget'); - dispatch('updateResolvableDiscussionsCounts'); + dispatch('updateMergeRequestWidget'); + dispatch('updateResolvableDiscussionsCounts'); - if (isInMRPage()) { - dispatch('diffs/removeDiscussionsFromDiff', discussion); - } + if (isInMRPage()) { + dispatch('diffs/removeDiscussionsFromDiff', discussion); + } +}; + +export const deleteNote = ({ dispatch }, note) => + axios.delete(note.path).then(() => { + dispatch('removeNote', note); }); export const updateNote = ({ commit, dispatch }, { endpoint, note }) => diff --git a/app/controllers/concerns/notes_actions.rb b/app/controllers/concerns/notes_actions.rb index d2a961efff7..4b7899d469b 100644 --- a/app/controllers/concerns/notes_actions.rb +++ b/app/controllers/concerns/notes_actions.rb @@ -73,6 +73,11 @@ module NotesActions # rubocop:disable Gitlab/ModuleWithInstanceVariables def update @note = Notes::UpdateService.new(project, current_user, update_note_params).execute(note) + unless @note + head :gone + return + end + prepare_notes_for_rendering([@note]) respond_to do |format| diff --git a/app/controllers/dashboard/projects_controller.rb b/app/controllers/dashboard/projects_controller.rb index 71f18694613..1dc89943f7f 100644 --- a/app/controllers/dashboard/projects_controller.rb +++ b/app/controllers/dashboard/projects_controller.rb @@ -70,6 +70,7 @@ class Dashboard::ProjectsController < Dashboard::ApplicationController .new(params: finder_params, current_user: current_user) .execute .includes(:route, :creator, :group, namespace: [:route, :owner]) + .preload(:project_feature) .page(finder_params[:page]) prepare_projects_for_rendering(projects) diff --git a/app/controllers/projects/notes_controller.rb b/app/controllers/projects/notes_controller.rb index 65d9b074eee..13e8453ed00 100644 --- a/app/controllers/projects/notes_controller.rb +++ b/app/controllers/projects/notes_controller.rb @@ -6,7 +6,7 @@ class Projects::NotesController < Projects::ApplicationController include NotesHelper include ToggleAwardEmoji - before_action :whitelist_query_limiting, only: [:create] + before_action :whitelist_query_limiting, only: [:create, :update] before_action :authorize_read_note! before_action :authorize_create_note!, only: [:create] before_action :authorize_resolve_note!, only: [:resolve, :unresolve] diff --git a/app/graphql/functions/base_function.rb b/app/graphql/functions/base_function.rb deleted file mode 100644 index 2512ecbd255..00000000000 --- a/app/graphql/functions/base_function.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -module Functions - class BaseFunction < GraphQL::Function - end -end diff --git a/app/graphql/functions/echo.rb b/app/graphql/functions/echo.rb deleted file mode 100644 index 3104486faac..00000000000 --- a/app/graphql/functions/echo.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Functions - class Echo < BaseFunction - argument :text, GraphQL::STRING_TYPE - - description "Testing endpoint to validate the API with" - - def call(obj, args, ctx) - username = ctx[:current_user]&.username - - "#{username.inspect} says: #{args[:text]}" - end - end -end diff --git a/app/graphql/gitlab_schema.rb b/app/graphql/gitlab_schema.rb index 7edd14e48f7..4c8612c8f2e 100644 --- a/app/graphql/gitlab_schema.rb +++ b/app/graphql/gitlab_schema.rb @@ -49,7 +49,7 @@ class GitlabSchema < GraphQL::Schema def id_from_object(object) unless object.respond_to?(:to_global_id) # This is an error in our schema and needs to be solved. So raise a - # more meaningfull error message + # more meaningful error message raise "#{object} does not implement `to_global_id`. "\ "Include `GlobalID::Identification` into `#{object.class}" end diff --git a/app/graphql/resolvers/echo_resolver.rb b/app/graphql/resolvers/echo_resolver.rb new file mode 100644 index 00000000000..8076e1784ce --- /dev/null +++ b/app/graphql/resolvers/echo_resolver.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Resolvers + class EchoResolver < BaseResolver + argument :text, GraphQL::STRING_TYPE, required: true + description 'Testing endpoint to validate the API with' + + def resolve(**args) + username = context[:current_user]&.username + + "#{username.inspect} says: #{args[:text]}" + end + end +end diff --git a/app/graphql/types/query_type.rb b/app/graphql/types/query_type.rb index 53d36b43576..c686300b25d 100644 --- a/app/graphql/types/query_type.rb +++ b/app/graphql/types/query_type.rb @@ -24,6 +24,6 @@ module Types resolver: Resolvers::MetadataResolver, description: 'Metadata about GitLab' - field :echo, GraphQL::STRING_TYPE, null: false, function: Functions::Echo.new + field :echo, GraphQL::STRING_TYPE, null: false, resolver: Resolvers::EchoResolver end end diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 33f0be91632..85f5a2040c0 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -5,7 +5,7 @@ class DeployToken < ApplicationRecord include TokenAuthenticatable include PolicyActor include Gitlab::Utils::StrongMemoize - add_authentication_token_field :token + add_authentication_token_field :token, encrypted: :optional AVAILABLE_SCOPES = %i(read_repository read_registry).freeze GITLAB_DEPLOY_TOKEN_NAME = 'gitlab-deploy-token'.freeze diff --git a/app/models/project.rb b/app/models/project.rb index 10679fb1f85..66d5286196e 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -2175,8 +2175,7 @@ class Project < ApplicationRecord hashed_storage?(:repository) && public? && repository_exists? && - Gitlab::CurrentSettings.hashed_storage_enabled && - Feature.enabled?(:object_pools, self, default_enabled: true) + Gitlab::CurrentSettings.hashed_storage_enabled end def leave_pool_repository diff --git a/app/models/user.rb b/app/models/user.rb index 6131a8dc710..6107aaa7fca 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -161,6 +161,8 @@ class User < ApplicationRecord # # Note: devise :validatable above adds validations for :email and :password validates :name, presence: true, length: { maximum: 128 } + validates :first_name, length: { maximum: 255 } + validates :last_name, length: { maximum: 255 } validates :email, confirmation: true validates :notification_email, presence: true validates :notification_email, devise_email: true, if: ->(user) { user.notification_email != user.email } @@ -881,7 +883,15 @@ class User < ApplicationRecord end def first_name - name.split.first unless name.blank? + read_attribute(:first_name) || begin + name.split(' ').first unless name.blank? + end + end + + def last_name + read_attribute(:last_name) || begin + name.split(' ').drop(1).join(' ') unless name.blank? + end end def projects_limit_left diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index b8dee1b0789..e2634692dc7 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -502,6 +502,8 @@ class ProjectPolicy < BasePolicy end def feature_available?(feature) + return false unless project.project_feature + case project.project_feature.access_level(feature) when ProjectFeature::DISABLED false diff --git a/app/services/application_settings/update_service.rb b/app/services/application_settings/update_service.rb index 471df6e2d0c..8115585b7a8 100644 --- a/app/services/application_settings/update_service.rb +++ b/app/services/application_settings/update_service.rb @@ -7,7 +7,7 @@ module ApplicationSettings attr_reader :params, :application_setting def execute - validate_classification_label(application_setting, :external_authorization_service_default_label) + validate_classification_label(application_setting, :external_authorization_service_default_label) unless bypass_external_auth? if application_setting.errors.any? return false @@ -59,5 +59,9 @@ module ApplicationSettings Group.find_by_full_path(group_full_path)&.id if group_full_path.present? end + + def bypass_external_auth? + params.key?(:external_authorization_service_enabled) && !Gitlab::Utils.to_boolean(params[:external_authorization_service_enabled]) + end end end diff --git a/app/services/merge_requests/create_service.rb b/app/services/merge_requests/create_service.rb index 06e46595b95..a69678a4422 100644 --- a/app/services/merge_requests/create_service.rb +++ b/app/services/merge_requests/create_service.rb @@ -27,6 +27,7 @@ module MergeRequests issuable.cache_merge_request_closes_issues!(current_user) create_pipeline_for(issuable, current_user) issuable.update_head_pipeline + Gitlab::UsageDataCounters::MergeRequestCounter.count(:create) super end diff --git a/app/services/notes/update_service.rb b/app/services/notes/update_service.rb index 384d1dd2e50..853faed9d85 100644 --- a/app/services/notes/update_service.rb +++ b/app/services/notes/update_service.rb @@ -8,24 +8,70 @@ module Notes old_mentioned_users = note.mentioned_users.to_a note.update(params.merge(updated_by: current_user)) - note.create_new_cross_references!(current_user) - if note.previous_changes.include?('note') - TodoService.new.update_note(note, current_user, old_mentioned_users) + only_commands = false + + quick_actions_service = QuickActionsService.new(project, current_user) + if quick_actions_service.supported?(note) + content, update_params, message = quick_actions_service.execute(note, {}) + + only_commands = content.empty? + + note.note = content + end + + unless only_commands + note.create_new_cross_references!(current_user) + + update_todos(note, old_mentioned_users) + + update_suggestions(note) end - if note.supports_suggestion? - Suggestion.transaction do - note.suggestions.delete_all - Suggestions::CreateService.new(note).execute + if quick_actions_service.commands_executed_count.to_i > 0 + if update_params.present? + quick_actions_service.apply_updates(update_params, note) + note.commands_changes = update_params end - # We need to refresh the previous suggestions call cache - # in order to get the new records. - note.reset + if only_commands + delete_note(note, message) + note = nil + else + note.save + end end note end + + private + + def delete_note(note, message) + # We must add the error after we call #save because errors are reset + # when #save is called + note.errors.add(:commands_only, message.presence || _('Commands did not apply')) + + Notes::DestroyService.new(project, current_user).execute(note) + end + + def update_suggestions(note) + return unless note.supports_suggestion? + + Suggestion.transaction do + note.suggestions.delete_all + Suggestions::CreateService.new(note).execute + end + + # We need to refresh the previous suggestions call cache + # in order to get the new records. + note.reset + end + + def update_todos(note, old_mentioned_users) + return unless note.previous_changes.include?('note') + + TodoService.new.update_note(note, current_user, old_mentioned_users) + end end end diff --git a/app/views/layouts/_piwik.html.haml b/app/views/layouts/_piwik.html.haml index 2cb2e23433d..361a7b03180 100644 --- a/app/views/layouts/_piwik.html.haml +++ b/app/views/layouts/_piwik.html.haml @@ -11,5 +11,5 @@ var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.type='text/javascript'; g.async=true; g.defer=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s); })(); - <noscript><p><img src="//#{extra_config.piwik_url}/piwik.php?idsite=#{extra_config.piwik_site_id}" style="border:0;" alt="" /></p></noscript> - <!-- End Piwik Code --> +<noscript><p><img src="//#{extra_config.piwik_url}/piwik.php?idsite=#{extra_config.piwik_site_id}" style="border:0;" alt="" /></p></noscript> +<!-- End Piwik Code --> diff --git a/app/views/profiles/preferences/show.html.haml b/app/views/profiles/preferences/show.html.haml index d99063e344f..0887e8e64da 100644 --- a/app/views/profiles/preferences/show.html.haml +++ b/app/views/profiles/preferences/show.html.haml @@ -47,7 +47,7 @@ = s_('Preferences|Layout width') = f.select :layout, layout_choices, {}, class: 'select2' .form-text.text-muted - = s_('Preferences|Choose between fixed (max. 1280px) and fluid (100%%) application layout.') + = s_('Preferences|Choose between fixed (max. 1280px) and fluid (%{percentage}) application layout.').html_safe % { percentage: '100%' } .form-group = f.label :dashboard, class: 'label-bold' do = s_('Preferences|Default dashboard') diff --git a/app/views/projects/mirrors/_mirror_repos.html.haml b/app/views/projects/mirrors/_mirror_repos.html.haml index eb100e5cf47..6f8a93fbcf5 100644 --- a/app/views/projects/mirrors/_mirror_repos.html.haml +++ b/app/views/projects/mirrors/_mirror_repos.html.haml @@ -13,8 +13,6 @@ .settings-content = form_for @project, url: project_mirror_path(@project), html: { class: 'gl-show-field-errors js-mirror-form', autocomplete: 'new-password', data: mirrors_form_data_attributes } do |f| .panel.panel-default - .panel-heading - %h3.panel-title= _('Mirror a repository') .panel-body %div= form_errors(@project) diff --git a/changelogs/unreleased/21505-quickactions-update-pd.yml b/changelogs/unreleased/21505-quickactions-update-pd.yml new file mode 100644 index 00000000000..243f8eda4e3 --- /dev/null +++ b/changelogs/unreleased/21505-quickactions-update-pd.yml @@ -0,0 +1,5 @@ +--- +title: Apply quickactions when modifying comments +merge_request: 31136 +author: +type: added diff --git a/changelogs/unreleased/57657-promote-label-to-group-label-via-api-endpoint.yml b/changelogs/unreleased/57657-promote-label-to-group-label-via-api-endpoint.yml new file mode 100644 index 00000000000..572bce34f45 --- /dev/null +++ b/changelogs/unreleased/57657-promote-label-to-group-label-via-api-endpoint.yml @@ -0,0 +1,5 @@ +--- +title: 'API: Promote project labels to group labels' +merge_request: 25218 +author: Robert Schilling +type: added diff --git a/changelogs/unreleased/63502-encrypt-deploy-token.yml b/changelogs/unreleased/63502-encrypt-deploy-token.yml new file mode 100644 index 00000000000..81ce1e6c3dd --- /dev/null +++ b/changelogs/unreleased/63502-encrypt-deploy-token.yml @@ -0,0 +1,5 @@ +--- +title: Encrypt existing and new deploy tokens +merge_request: 30679 +author: +type: other diff --git a/changelogs/unreleased/66443-unrecoverable-configuration-loop-in-external-auth-control.yml b/changelogs/unreleased/66443-unrecoverable-configuration-loop-in-external-auth-control.yml new file mode 100644 index 00000000000..ab52e3e5a2c --- /dev/null +++ b/changelogs/unreleased/66443-unrecoverable-configuration-loop-in-external-auth-control.yml @@ -0,0 +1,5 @@ +--- +title: Don't check external authorization when disabling the service +merge_request: 32102 +author: Robert Schilling +type: fixed diff --git a/changelogs/unreleased/georgekoltsov-54023-fogbugz-visibility-level.yml b/changelogs/unreleased/georgekoltsov-54023-fogbugz-visibility-level.yml new file mode 100644 index 00000000000..d292958c92a --- /dev/null +++ b/changelogs/unreleased/georgekoltsov-54023-fogbugz-visibility-level.yml @@ -0,0 +1,5 @@ +--- +title: Change default visibility level for FogBugz imported projects to Private +merge_request: 32142 +author: +type: fixed diff --git a/changelogs/unreleased/id-code-review-smau.yml b/changelogs/unreleased/id-code-review-smau.yml new file mode 100644 index 00000000000..0bc7bca789b --- /dev/null +++ b/changelogs/unreleased/id-code-review-smau.yml @@ -0,0 +1,5 @@ +--- +title: Add usage pings for merge request creating +merge_request: 32059 +author: +type: added diff --git a/changelogs/unreleased/oauth_bypass_two_factor.yml b/changelogs/unreleased/oauth_bypass_two_factor.yml new file mode 100644 index 00000000000..7261e65a9f3 --- /dev/null +++ b/changelogs/unreleased/oauth_bypass_two_factor.yml @@ -0,0 +1,5 @@ +--- +title: Add option to allow OAuth providers to bypass two factor +merge_request: 31996 +author: Dodocat +type: added diff --git a/changelogs/unreleased/sh-fix-nplusone-issues.yml b/changelogs/unreleased/sh-fix-nplusone-issues.yml new file mode 100644 index 00000000000..f749b5eeb40 --- /dev/null +++ b/changelogs/unreleased/sh-fix-nplusone-issues.yml @@ -0,0 +1,5 @@ +--- +title: Fix N+1 Gitaly calls in /api/v4/projects/:id/issues +merge_request: 32171 +author: +type: performance diff --git a/changelogs/unreleased/sh-fix-piwik-template.yml b/changelogs/unreleased/sh-fix-piwik-template.yml new file mode 100644 index 00000000000..f0baed6a2e0 --- /dev/null +++ b/changelogs/unreleased/sh-fix-piwik-template.yml @@ -0,0 +1,5 @@ +--- +title: Fix Piwik not working +merge_request: 32234 +author: +type: fixed diff --git a/changelogs/unreleased/sh-guard-against-orphaned-project-feature.yml b/changelogs/unreleased/sh-guard-against-orphaned-project-feature.yml new file mode 100644 index 00000000000..99c8732c5b0 --- /dev/null +++ b/changelogs/unreleased/sh-guard-against-orphaned-project-feature.yml @@ -0,0 +1,5 @@ +--- +title: Guard against deleted project feature entry in project permissions +merge_request: 32187 +author: +type: fixed diff --git a/changelogs/unreleased/sh-project-feature-nplus-one.yml b/changelogs/unreleased/sh-project-feature-nplus-one.yml new file mode 100644 index 00000000000..425ae7815c1 --- /dev/null +++ b/changelogs/unreleased/sh-project-feature-nplus-one.yml @@ -0,0 +1,5 @@ +--- +title: Remove N+1 SQL query loading project feature in dashboard +merge_request: 32169 +author: +type: performance diff --git a/changelogs/unreleased/user_name_migration.yml b/changelogs/unreleased/user_name_migration.yml new file mode 100644 index 00000000000..11d6ff2e8b2 --- /dev/null +++ b/changelogs/unreleased/user_name_migration.yml @@ -0,0 +1,5 @@ +--- +title: Add First and Last name columns to User model +merge_request: 31985 +author: +type: added diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 973c2747838..20b1020e025 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -780,6 +780,14 @@ production: &base # (default: []) external_providers: [] + # CAUTION! + # This allows users to login with the specified providers without two factor. Define the allowed providers + # using an array, e.g. ["twitter", 'google_oauth2'], or as true/false to allow all providers or none. + # This option should only be configured for providers which already have two factor. + # This configration dose not apply to SAML. + # (default: false) + allow_bypass_two_factor: ["twitter", 'google_oauth2'] + ## Auth providers # Uncomment the following lines and fill in the data of the auth provider you want to use # If your favorite auth provider is not listed you can use others: diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index fdc6b0a05ab..81433b620bc 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -84,6 +84,7 @@ Settings['omniauth'] ||= Settingslogic.new({}) Settings.omniauth['enabled'] = true if Settings.omniauth['enabled'].nil? Settings.omniauth['auto_sign_in_with_provider'] = false if Settings.omniauth['auto_sign_in_with_provider'].nil? Settings.omniauth['allow_single_sign_on'] = false if Settings.omniauth['allow_single_sign_on'].nil? +Settings.omniauth['allow_bypass_two_factor'] = false if Settings.omniauth['allow_bypass_two_factor'].nil? Settings.omniauth['external_providers'] = [] if Settings.omniauth['external_providers'].nil? Settings.omniauth['block_auto_created_users'] = true if Settings.omniauth['block_auto_created_users'].nil? Settings.omniauth['auto_link_ldap_user'] = false if Settings.omniauth['auto_link_ldap_user'].nil? diff --git a/danger/only_documentation/Dangerfile b/danger/only_documentation/Dangerfile index dad12c0d29c..ce7ede26d9a 100644 --- a/danger/only_documentation/Dangerfile +++ b/danger/only_documentation/Dangerfile @@ -1,7 +1,7 @@ # rubocop:disable Style/SignalException # frozen_string_literal: true -has_only_docs_changes = helper.all_changed_files.all? { |file| file.start_with?('doc/', '.gitlab/ci/docs.gitlab-ci.yml', '.mdlrc') || file.end_with?('.md') } +has_only_docs_changes = helper.all_changed_files.all? { |file| file.start_with?('doc/', '.gitlab/ci/docs.gitlab-ci.yml', '.markdownlint.json') || file.end_with?('.md') } is_docs_only_branch = gitlab.branch_for_head =~ /(^docs[\/-].*|.*-docs$)/ if is_docs_only_branch && !has_only_docs_changes diff --git a/db/migrate/20190711200053_change_deploy_tokens_token_not_null.rb b/db/migrate/20190711200053_change_deploy_tokens_token_not_null.rb new file mode 100644 index 00000000000..14ccf544d0b --- /dev/null +++ b/db/migrate/20190711200053_change_deploy_tokens_token_not_null.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class ChangeDeployTokensTokenNotNull < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + change_column_null :deploy_tokens, :token, true + end +end diff --git a/db/migrate/20190711200508_add_token_encrypted_to_deploy_tokens.rb b/db/migrate/20190711200508_add_token_encrypted_to_deploy_tokens.rb new file mode 100644 index 00000000000..ea0956fdf7f --- /dev/null +++ b/db/migrate/20190711200508_add_token_encrypted_to_deploy_tokens.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddTokenEncryptedToDeployTokens < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :deploy_tokens, :token_encrypted, :string, limit: 255 + end +end diff --git a/db/migrate/20190719174505_add_index_to_deploy_tokens_token_encrypted.rb b/db/migrate/20190719174505_add_index_to_deploy_tokens_token_encrypted.rb new file mode 100644 index 00000000000..d58d1d8348c --- /dev/null +++ b/db/migrate/20190719174505_add_index_to_deploy_tokens_token_encrypted.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddIndexToDeployTokensTokenEncrypted < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :deploy_tokens, :token_encrypted, unique: true, name: "index_deploy_tokens_on_token_encrypted" + end + + def down + remove_concurrent_index_by_name :deploy_tokens, "index_deploy_tokens_on_token_encrypted" + end +end diff --git a/db/migrate/20190726101050_rename_allow_local_requests_from_hooks_and_services_application_setting.rb b/db/migrate/20190726101050_rename_allow_local_requests_from_hooks_and_services_application_setting.rb index ac65e8d745c..cce8942128c 100644 --- a/db/migrate/20190726101050_rename_allow_local_requests_from_hooks_and_services_application_setting.rb +++ b/db/migrate/20190726101050_rename_allow_local_requests_from_hooks_and_services_application_setting.rb @@ -12,6 +12,6 @@ class RenameAllowLocalRequestsFromHooksAndServicesApplicationSetting < ActiveRec end def down - cleanup_concurrent_column_rename :application_settings, :allow_local_requests_from_web_hooks_and_services, :allow_local_requests_from_hooks_and_services + undo_rename_column_concurrently :application_settings, :allow_local_requests_from_hooks_and_services, :allow_local_requests_from_web_hooks_and_services end end diff --git a/db/migrate/20190820163320_add_first_last_name_to_user.rb b/db/migrate/20190820163320_add_first_last_name_to_user.rb new file mode 100644 index 00000000000..0ea465fc2e2 --- /dev/null +++ b/db/migrate/20190820163320_add_first_last_name_to_user.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddFirstLastNameToUser < ActiveRecord::Migration[5.2] + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + def change + add_column(:users, :first_name, :string, null: true, limit: 255) + add_column(:users, :last_name, :string, null: true, limit: 255) + end +end diff --git a/db/post_migrate/20190711201818_encrypt_deploy_tokens_tokens.rb b/db/post_migrate/20190711201818_encrypt_deploy_tokens_tokens.rb new file mode 100644 index 00000000000..2eb8d1ee11c --- /dev/null +++ b/db/post_migrate/20190711201818_encrypt_deploy_tokens_tokens.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class EncryptDeployTokensTokens < ActiveRecord::Migration[5.1] + DOWNTIME = false + + class DeploymentTokens < ActiveRecord::Base + self.table_name = 'deploy_tokens' + end + + def up + say_with_time("Encrypting tokens from deploy_tokens") do + DeploymentTokens.where('token_encrypted is NULL AND token IS NOT NULL').find_each(batch_size: 10000) do |deploy_token| + token_encrypted = Gitlab::CryptoHelper.aes256_gcm_encrypt(deploy_token.token) + deploy_token.update!(token_encrypted: token_encrypted) + end + end + end + + def down + say_with_time("Decrypting tokens from deploy_tokens") do + DeploymentTokens.where('token_encrypted IS NOT NULL AND token IS NULL').find_each(batch_size: 10000) do |deploy_token| + token = Gitlab::CryptoHelper.aes256_gcm_decrypt(deploy_token.token_encrypted) + deploy_token.update!(token: token) + end + end + end +end diff --git a/db/post_migrate/20190801114109_cleanup_allow_local_requests_from_hooks_and_services_application_setting_rename.rb b/db/post_migrate/20190801114109_cleanup_allow_local_requests_from_hooks_and_services_application_setting_rename.rb index 127e44254ac..cb86f843f9c 100644 --- a/db/post_migrate/20190801114109_cleanup_allow_local_requests_from_hooks_and_services_application_setting_rename.rb +++ b/db/post_migrate/20190801114109_cleanup_allow_local_requests_from_hooks_and_services_application_setting_rename.rb @@ -12,6 +12,6 @@ class CleanupAllowLocalRequestsFromHooksAndServicesApplicationSettingRename < Ac end def down - rename_column_concurrently :application_settings, :allow_local_requests_from_web_hooks_and_services, :allow_local_requests_from_hooks_and_services + undo_cleanup_concurrent_column_rename :application_settings, :allow_local_requests_from_hooks_and_services, :allow_local_requests_from_web_hooks_and_services end end diff --git a/db/schema.rb b/db/schema.rb index 944d23e5365..f30dad3d030 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_08_15_093949) do +ActiveRecord::Schema.define(version: 2019_08_20_163320) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -1121,10 +1121,12 @@ ActiveRecord::Schema.define(version: 2019_08_15_093949) do t.datetime_with_timezone "expires_at", null: false t.datetime_with_timezone "created_at", null: false t.string "name", null: false - t.string "token", null: false + t.string "token" t.string "username" + t.string "token_encrypted", limit: 255 t.index ["token", "expires_at", "id"], name: "index_deploy_tokens_on_token_and_expires_at_and_id", where: "(revoked IS FALSE)" t.index ["token"], name: "index_deploy_tokens_on_token", unique: true + t.index ["token_encrypted"], name: "index_deploy_tokens_on_token_encrypted", unique: true end create_table "deployments", id: :serial, force: :cascade do |t| @@ -3511,6 +3513,8 @@ ActiveRecord::Schema.define(version: 2019_08_15_093949) do t.text "note" t.integer "roadmap_layout", limit: 2 t.integer "bot_type", limit: 2 + t.string "first_name", limit: 255 + t.string "last_name", limit: 255 t.index ["accepted_term_id"], name: "index_users_on_accepted_term_id" t.index ["admin"], name: "index_users_on_admin" t.index ["bot_type"], name: "index_users_on_bot_type" diff --git a/doc/administration/database_load_balancing.md b/doc/administration/database_load_balancing.md index dc4cc401fca..64eca0b00f6 100644 --- a/doc/administration/database_load_balancing.md +++ b/doc/administration/database_load_balancing.md @@ -122,6 +122,7 @@ production: discover: nameserver: localhost record: secondary.postgresql.service.consul + record_type: A port: 8600 interval: 60 disconnect_timeout: 120 @@ -137,12 +138,16 @@ The following options can be set: | Option | Description | Default | |----------------------|---------------------------------------------------------------------------------------------------|-----------| | `nameserver` | The nameserver to use for looking up the DNS record. | localhost | -| `record` | The A record to look up. This option is required for service discovery to work. | | +| `record` | The record to look up. This option is required for service discovery to work. | | +| `record_type` | Optional record type to look up, this can be either A or SRV (since GitLab 12.3) | A | | `port` | The port of the nameserver. | 8600 | | `interval` | The minimum time in seconds between checking the DNS record. | 60 | | `disconnect_timeout` | The time in seconds after which an old connection is closed, after the list of hosts was updated. | 120 | | `use_tcp` | Lookup DNS resources using TCP instead of UDP | false | +If `record_type` is set to `SRV`, GitLab will continue to use a round-robin algorithm +and will ignore the `weight` and `priority` in the record. + The `interval` value specifies the _minimum_ time between checks. If the A record has a TTL greater than this value, then service discovery will honor said TTL. For example, if the TTL of the A record is 90 seconds, then service diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index e0a06e71316..daec5f14e36 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -139,7 +139,7 @@ Git operations in GitLab will result in an API error. NOTE: **Note:** In most or all cases, the storage paths below end in `/repositories` which is -not that case with `path` in `git_data_dirs` of Omnibus GitLab installations. +not the case with `path` in `git_data_dirs` of Omnibus GitLab installations. Check the directory layout on your Gitaly server to be sure. **For Omnibus GitLab** @@ -283,10 +283,6 @@ can read and write to `/mnt/gitlab/storage2`. gitlab_rails['gitaly_token'] = 'abc123secret' ``` - NOTE: **Note:** - In some cases, you'll have to set `path` for each `git_data_dirs` in the - format `'path' => '/mnt/gitlab/<storage name>'`. - 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. Tail the logs to see the requests: @@ -304,18 +300,22 @@ can read and write to `/mnt/gitlab/storage2`. storages: default: gitaly_address: tcp://gitaly1.internal:8075 + path: /some/dummy/path storage1: gitaly_address: tcp://gitaly1.internal:8075 + path: /some/dummy/path storage2: gitaly_address: tcp://gitaly2.internal:8075 + path: /some/dummy/path gitaly: token: 'abc123secret' ``` NOTE: **Note:** - In some cases, you'll have to set `path` for each of the `storages` in the - format `path: /mnt/gitlab/<storage name>/repositories`. + `/some/dummy/path` should be set to a local folder that exists, however no + data will be stored in this folder. This will no longer be necessary after + [this issue](https://gitlab.com/gitlab-org/gitaly/issues/1282) is resolved. 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). 1. Tail the logs to see the requests: @@ -416,18 +416,22 @@ To configure Gitaly with TLS: storages: default: gitaly_address: tls://gitaly1.internal:9999 + path: /some/dummy/path storage1: gitaly_address: tls://gitaly1.internal:9999 + path: /some/dummy/path storage2: gitaly_address: tls://gitaly2.internal:9999 + path: /some/dummy/path gitaly: token: 'abc123secret' ``` NOTE: **Note:** - In some cases, you'll have to set `path` for each of the `storages` in the - format `path: /mnt/gitlab/<storage name>/repositories`. + `/some/dummy/path` should be set to a local folder that exists, however no + data will be stored in this folder. This will no longer be necessary after + [this issue](https://gitlab.com/gitlab-org/gitaly/issues/1282) is resolved. 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). 1. On the Gitaly server nodes, edit `/home/git/gitaly/config.toml`: @@ -514,6 +518,47 @@ One current feature of GitLab that still requires a shared directory (NFS) is There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/issues/196) to eliminate the need for NFS to support GitLab Pages. +## Limiting RPC concurrency + +It can happen that CI clone traffic puts a large strain on your Gitaly +service. The bulk of the work gets done in the SSHUploadPack (for Git +SSH) and PostUploadPack (for Git HTTP) RPC's. To prevent such workloads +from overcrowding your Gitaly server you can set concurrency limits in +Gitaly's configuration file. + +```ruby +# in /etc/gitlab/gitlab.rb + +gitaly['concurrency'] = [ + { + 'rpc' => "/gitaly.SmartHTTPService/PostUploadPack", + 'max_per_repo' => 20 + }, + { + 'rpc' => "/gitaly.SSHService/SSHUploadPack", + 'max_per_repo' => 20 + } +] +``` +This will limit the number of in-flight RPC calls for the given RPC's. +The limit is applied per repository. In the example above, each on the +Gitaly server can have at most 20 simultaneous PostUploadPack calls in +flight, and the same for SSHUploadPack. If another request comes in for +a repository that hase used up its 20 slots, that request will get +queued. + +You can observe the behavior of this queue via the Gitaly logs and via +Prometheus. In the Gitaly logs, you can look for the string (or +structured log field) `acquire_ms`. Messages that have this field are +reporting about the concurrency limiter. In Prometheus, look for the +`gitaly_rate_limiting_in_progress`, `gitaly_rate_limiting_queued` and +`gitaly_rate_limiting_seconds` metrics. + +The name of the Prometheus metric is not quite right because this is a +concurrency limiter, not a rate limiter. If a client makes 1000 requests +in a row in a very short timespan, the concurrency will not exceed 1, +and this mechanism (the concurrency limiter) will do nothing. + ## Troubleshooting Gitaly ### Commits, pushes, and clones return a 401 diff --git a/doc/administration/issue_closing_pattern.md b/doc/administration/issue_closing_pattern.md index 66ab3ef8d81..f9be06c6daf 100644 --- a/doc/administration/issue_closing_pattern.md +++ b/doc/administration/issue_closing_pattern.md @@ -1,8 +1,8 @@ # Issue closing pattern **(CORE ONLY)** >**Note:** -This is the administration documentation. -There is a separate [user documentation] on issue closing pattern. +This is the administration documentation. There is a separate [user documentation](../user/project/issues/managing_issues.md#closing-issues-automatically) +on issue closing pattern. When a commit or merge request resolves one or more issues, it is possible to automatically have these issues closed when the commit or merge request lands @@ -13,8 +13,8 @@ in the project's default branch. In order to change the pattern you need to have access to the server that GitLab is installed on. -The default pattern can be located in [`gitlab.yml.example`] under the -"Automatic issue closing" section. +The default pattern can be located in [`gitlab.yml.example`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example) +under the "Automatic issue closing" section. > **Tip:** You are advised to use <http://rubular.com> to test the issue closing pattern. @@ -31,7 +31,7 @@ Because Rubular doesn't understand `%{issue_ref}`, you can replace this by gitlab_rails['gitlab_issue_closing_pattern'] = "\b((?:[Cc]los(?:e[sd]|ing)|\b[Ff]ix(?:e[sd]|ing)?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?))+)" ``` -1. [Reconfigure] GitLab for the changes to take effect. +1. [Reconfigure](restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. **For installations from source** @@ -42,9 +42,4 @@ Because Rubular doesn't understand `%{issue_ref}`, you can replace this by issue_closing_pattern: "\b((?:[Cc]los(?:e[sd]|ing)|\b[Ff]ix(?:e[sd]|ing)?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?))+)" ``` -1. [Restart] GitLab for the changes to take effect. - -[gitlab.yml.example]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example -[reconfigure]: restart_gitlab.md#omnibus-gitlab-reconfigure -[restart]: restart_gitlab.md#installations-from-source -[user documentation]: ../user/project/issues/managing_issues.md#closing-issues-automatically +1. [Restart](restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index 0605fb76e2f..4809bee0df6 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -1,7 +1,7 @@ # GitLab Prometheus metrics >**Note:** -Available since [Omnibus GitLab 9.3][29118]. For +Available since [Omnibus GitLab 9.3](https://gitlab.com/gitlab-org/gitlab-ce/issues/29118). For installations from source you'll have to configure it yourself. To enable the GitLab Prometheus metrics: @@ -9,13 +9,13 @@ To enable the GitLab Prometheus metrics: 1. Log into GitLab as an administrator, and go to the Admin area. 1. Click on the gear, then click on Settings. 1. Find the `Metrics - Prometheus` section, and click `Enable Prometheus Metrics` -1. [Restart GitLab][restart] for the changes to take effect +1. [Restart GitLab](../../restart_gitlab.md#omnibus-gitlab-restart) for the changes to take effect ## Collecting the metrics GitLab monitors its own internal service metrics, and makes them available at the -`/-/metrics` endpoint. Unlike other [Prometheus] exporters, in order to access -it, the client IP needs to be [included in a whitelist][whitelist]. +`/-/metrics` endpoint. Unlike other [Prometheus](https://prometheus.io) exporters, in order to access +it, the client IP needs to be [included in a whitelist](../ip_whitelist.md). For Omnibus and Chart installations, these metrics are automatically enabled and collected as of [GitLab 9.4](https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests/1702). For source installations or earlier versions, these metrics will need to be enabled manually and collected by a Prometheus server. @@ -23,159 +23,159 @@ For Omnibus and Chart installations, these metrics are automatically enabled and The following metrics are available: -| Metric | Type | Since | Description | Labels | -|:-------------------------------------------------------------|:----------|-----------------------:|:----------------------------------------------------------------------------------------------------|:----------------------------------------------------| -| gitlab_banzai_cached_render_real_duration_seconds | Histogram | 9.4 | Duration of rendering markdown into HTML when cached output exists | controller, action | -| gitlab_banzai_cacheless_render_real_duration_seconds | Histogram | 9.4 | Duration of rendering markdown into HTML when cached outupt does not exist | controller, action | -| gitlab_cache_misses_total | Counter | 10.2 | Cache read miss | controller, action | -| gitlab_cache_operation_duration_seconds | Histogram | 10.2 | Cache access time | | -| gitlab_cache_operations_total | Counter | 12.2 | Cache operations by controller/action | controller, action, operation | -| gitlab_database_transaction_seconds | Histogram | 12.1 | Time spent in database transactions, in seconds | | -| gitlab_method_call_duration_seconds | Histogram | 10.2 | Method calls real duration | controller, action, module, method | -| gitlab_rails_queue_duration_seconds | Histogram | 9.4 | Measures latency between gitlab-workhorse forwarding a request to Rails | | -| gitlab_sql_duration_seconds | Histogram | 10.2 | SQL execution time, excluding SCHEMA operations and BEGIN / COMMIT | | -| gitlab_transaction_allocated_memory_bytes | Histogram | 10.2 | Allocated memory for all transactions (gitlab_transaction_* metrics) | | -| gitlab_transaction_cache_<key>_count_total | Counter | 10.2 | Counter for total Rails cache calls (per key) | | -| gitlab_transaction_cache_<key>_duration_total | Counter | 10.2 | Counter for total time (seconds) spent in Rails cache calls (per key) | | -| gitlab_transaction_cache_count_total | Counter | 10.2 | Counter for total Rails cache calls (aggregate) | | -| gitlab_transaction_cache_duration_total | Counter | 10.2 | Counter for total time (seconds) spent in Rails cache calls (aggregate) | | -| gitlab_transaction_cache_read_hit_count_total | Counter | 10.2 | Counter for cache hits for Rails cache calls | controller, action | -| gitlab_transaction_cache_read_miss_count_total | Counter | 10.2 | Counter for cache misses for Rails cache calls | controller, action | -| gitlab_transaction_duration_seconds | Histogram | 10.2 | Duration for all transactions (gitlab_transaction_* metrics) | controller, action | -| gitlab_transaction_event_build_found_total | Counter | 9.4 | Counter for build found for api /jobs/request | | -| gitlab_transaction_event_build_invalid_total | Counter | 9.4 | Counter for build invalid due to concurrency conflict for api /jobs/request | | -| gitlab_transaction_event_build_not_found_cached_total | Counter | 9.4 | Counter for cached response of build not found for api /jobs/request | | -| gitlab_transaction_event_build_not_found_total | Counter | 9.4 | Counter for build not found for api /jobs/request | | -| gitlab_transaction_event_change_default_branch_total | Counter | 9.4 | Counter when default branch is changed for any repository | | -| gitlab_transaction_event_create_repository_total | Counter | 9.4 | Counter when any repository is created | | -| gitlab_transaction_event_etag_caching_cache_hit_total | Counter | 9.4 | Counter for etag cache hit. | endpoint | -| gitlab_transaction_event_etag_caching_header_missing_total | Counter | 9.4 | Counter for etag cache miss - header missing | endpoint | -| gitlab_transaction_event_etag_caching_key_not_found_total | Counter | 9.4 | Counter for etag cache miss - key not found | endpoint | -| gitlab_transaction_event_etag_caching_middleware_used_total | Counter | 9.4 | Counter for etag middleware accessed | endpoint | -| gitlab_transaction_event_etag_caching_resource_changed_total | Counter | 9.4 | Counter for etag cache miss - resource changed | endpoint | -| gitlab_transaction_event_fork_repository_total | Counter | 9.4 | Counter for repository forks (RepositoryForkWorker). Only incremented when source repository exists | | -| gitlab_transaction_event_import_repository_total | Counter | 9.4 | Counter for repository imports (RepositoryImportWorker) | | -| gitlab_transaction_event_push_branch_total | Counter | 9.4 | Counter for all branch pushes | | -| gitlab_transaction_event_push_commit_total | Counter | 9.4 | Counter for commits | branch | -| gitlab_transaction_event_push_tag_total | Counter | 9.4 | Counter for tag pushes | | -| gitlab_transaction_event_rails_exception_total | Counter | 9.4 | Counter for number of rails exceptions | | -| gitlab_transaction_event_receive_email_total | Counter | 9.4 | Counter for recieved emails | handler | -| gitlab_transaction_event_remote_mirrors_failed_total | Counter | 10.8 | Counter for failed remote mirrors | | -| gitlab_transaction_event_remote_mirrors_finished_total | Counter | 10.8 | Counter for finished remote mirrors | | -| gitlab_transaction_event_remote_mirrors_running_total | Counter | 10.8 | Counter for running remote mirrors | | -| gitlab_transaction_event_remove_branch_total | Counter | 9.4 | Counter when a branch is removed for any repository | | -| gitlab_transaction_event_remove_repository_total | Counter | 9.4 | Counter when a repository is removed | | -| gitlab_transaction_event_remove_tag_total | Counter | 9.4 | Counter when a tag is remove for any repository | | -| gitlab_transaction_event_sidekiq_exception_total | Counter | 9.4 | Counter of sidekiq exceptions | | -| gitlab_transaction_event_stuck_import_jobs_total | Counter | 9.4 | Count of stuck import jobs | projects_without_jid_count, projects_with_jid_count | -| gitlab_transaction_event_update_build_total | Counter | 9.4 | Counter for update build for api /jobs/request/:id | | -| gitlab_transaction_new_redis_connections_total | Counter | 9.4 | Counter for new redis connections | | -| gitlab_transaction_queue_duration_total | Counter | 9.4 | Duration jobs were enqueued before processing | | -| gitlab_transaction_rails_queue_duration_total | Counter | 9.4 | Measures latency between gitlab-workhorse forwarding a request to Rails | controller, action | -| gitlab_transaction_view_duration_total | Counter | 9.4 | Duration for views | controller, action, view | -| gitlab_view_rendering_duration_seconds | Histogram | 10.2 | Duration for views (histogram) | controller, action, view | -| http_requests_total | Counter | 9.4 | Rack request count | method | -| http_request_duration_seconds | Histogram | 9.4 | HTTP response time from rack middleware | method, status | -| pipelines_created_total | Counter | 9.4 | Counter of pipelines created | | -| rack_uncaught_errors_total | Counter | 9.4 | Rack connections handling uncaught errors count | | -| user_session_logins_total | Counter | 9.4 | Counter of how many users have logged in | | -| upload_file_does_not_exist | Counter | 10.7 in EE, 11.5 in CE | Number of times an upload record could not find its file | | -| failed_login_captcha_total | Gauge | 11.0 | Counter of failed CAPTCHA attempts during login | | -| successful_login_captcha_total | Gauge | 11.0 | Counter of successful CAPTCHA attempts during login | | +| Metric | Type | Since | Description | Labels | +|:---------------------------------------------------------------|:----------|-----------------------:|:----------------------------------------------------------------------------------------------------|:----------------------------------------------------| +| `gitlab_banzai_cached_render_real_duration_seconds` | Histogram | 9.4 | Duration of rendering markdown into HTML when cached output exists | controller, action | +| `gitlab_banzai_cacheless_render_real_duration_seconds` | Histogram | 9.4 | Duration of rendering markdown into HTML when cached outupt does not exist | controller, action | +| `gitlab_cache_misses_total` | Counter | 10.2 | Cache read miss | controller, action | +| `gitlab_cache_operation_duration_seconds` | Histogram | 10.2 | Cache access time | | +| `gitlab_cache_operations_total` | Counter | 12.2 | Cache operations by controller/action | controller, action, operation | +| `gitlab_database_transaction_seconds` | Histogram | 12.1 | Time spent in database transactions, in seconds | | +| `gitlab_method_call_duration_seconds` | Histogram | 10.2 | Method calls real duration | controller, action, module, method | +| `gitlab_rails_queue_duration_seconds` | Histogram | 9.4 | Measures latency between GitLab Workhorse forwarding a request to Rails | | +| `gitlab_sql_duration_seconds` | Histogram | 10.2 | SQL execution time, excluding SCHEMA operations and BEGIN / COMMIT | | +| `gitlab_transaction_allocated_memory_bytes` | Histogram | 10.2 | Allocated memory for all transactions (gitlab_transaction_* metrics) | | +| `gitlab_transaction_cache_<key>_count_total` | Counter | 10.2 | Counter for total Rails cache calls (per key) | | +| `gitlab_transaction_cache_<key>_duration_total` | Counter | 10.2 | Counter for total time (seconds) spent in Rails cache calls (per key) | | +| `gitlab_transaction_cache_count_total` | Counter | 10.2 | Counter for total Rails cache calls (aggregate) | | +| `gitlab_transaction_cache_duration_total` | Counter | 10.2 | Counter for total time (seconds) spent in Rails cache calls (aggregate) | | +| `gitlab_transaction_cache_read_hit_count_total` | Counter | 10.2 | Counter for cache hits for Rails cache calls | controller, action | +| `gitlab_transaction_cache_read_miss_count_total` | Counter | 10.2 | Counter for cache misses for Rails cache calls | controller, action | +| `gitlab_transaction_duration_seconds` | Histogram | 10.2 | Duration for all transactions (gitlab_transaction_* metrics) | controller, action | +| `gitlab_transaction_event_build_found_total` | Counter | 9.4 | Counter for build found for api /jobs/request | | +| `gitlab_transaction_event_build_invalid_total` | Counter | 9.4 | Counter for build invalid due to concurrency conflict for api /jobs/request | | +| `gitlab_transaction_event_build_not_found_cached_total` | Counter | 9.4 | Counter for cached response of build not found for api /jobs/request | | +| `gitlab_transaction_event_build_not_found_total` | Counter | 9.4 | Counter for build not found for api /jobs/request | | +| `gitlab_transaction_event_change_default_branch_total` | Counter | 9.4 | Counter when default branch is changed for any repository | | +| `gitlab_transaction_event_create_repository_total` | Counter | 9.4 | Counter when any repository is created | | +| `gitlab_transaction_event_etag_caching_cache_hit_total` | Counter | 9.4 | Counter for etag cache hit. | endpoint | +| `gitlab_transaction_event_etag_caching_header_missing_total` | Counter | 9.4 | Counter for etag cache miss - header missing | endpoint | +| `gitlab_transaction_event_etag_caching_key_not_found_total` | Counter | 9.4 | Counter for etag cache miss - key not found | endpoint | +| `gitlab_transaction_event_etag_caching_middleware_used_total` | Counter | 9.4 | Counter for etag middleware accessed | endpoint | +| `gitlab_transaction_event_etag_caching_resource_changed_total` | Counter | 9.4 | Counter for etag cache miss - resource changed | endpoint | +| `gitlab_transaction_event_fork_repository_total` | Counter | 9.4 | Counter for repository forks (RepositoryForkWorker). Only incremented when source repository exists | | +| `gitlab_transaction_event_import_repository_total` | Counter | 9.4 | Counter for repository imports (RepositoryImportWorker) | | +| `gitlab_transaction_event_push_branch_total` | Counter | 9.4 | Counter for all branch pushes | | +| `gitlab_transaction_event_push_commit_total` | Counter | 9.4 | Counter for commits | branch | +| `gitlab_transaction_event_push_tag_total` | Counter | 9.4 | Counter for tag pushes | | +| `gitlab_transaction_event_rails_exception_total` | Counter | 9.4 | Counter for number of rails exceptions | | +| `gitlab_transaction_event_receive_email_total` | Counter | 9.4 | Counter for recieved emails | handler | +| `gitlab_transaction_event_remote_mirrors_failed_total` | Counter | 10.8 | Counter for failed remote mirrors | | +| `gitlab_transaction_event_remote_mirrors_finished_total` | Counter | 10.8 | Counter for finished remote mirrors | | +| `gitlab_transaction_event_remote_mirrors_running_total` | Counter | 10.8 | Counter for running remote mirrors | | +| `gitlab_transaction_event_remove_branch_total` | Counter | 9.4 | Counter when a branch is removed for any repository | | +| `gitlab_transaction_event_remove_repository_total` | Counter | 9.4 | Counter when a repository is removed | | +| `gitlab_transaction_event_remove_tag_total` | Counter | 9.4 | Counter when a tag is remove for any repository | | +| `gitlab_transaction_event_sidekiq_exception_total` | Counter | 9.4 | Counter of sidekiq exceptions | | +| `gitlab_transaction_event_stuck_import_jobs_total` | Counter | 9.4 | Count of stuck import jobs | projects_without_jid_count, projects_with_jid_count | +| `gitlab_transaction_event_update_build_total` | Counter | 9.4 | Counter for update build for api /jobs/request/:id | | +| `gitlab_transaction_new_redis_connections_total` | Counter | 9.4 | Counter for new redis connections | | +| `gitlab_transaction_queue_duration_total` | Counter | 9.4 | Duration jobs were enqueued before processing | | +| `gitlab_transaction_rails_queue_duration_total` | Counter | 9.4 | Measures latency between GitLab Workhorse forwarding a request to Rails | controller, action | +| `gitlab_transaction_view_duration_total` | Counter | 9.4 | Duration for views | controller, action, view | +| `gitlab_view_rendering_duration_seconds` | Histogram | 10.2 | Duration for views (histogram) | controller, action, view | +| `http_requests_total` | Counter | 9.4 | Rack request count | method | +| `http_request_duration_seconds` | Histogram | 9.4 | HTTP response time from rack middleware | method, status | +| `pipelines_created_total` | Counter | 9.4 | Counter of pipelines created | | +| `rack_uncaught_errors_total` | Counter | 9.4 | Rack connections handling uncaught errors count | | +| `user_session_logins_total` | Counter | 9.4 | Counter of how many users have logged in | | +| `upload_file_does_not_exist` | Counter | 10.7 in EE, 11.5 in CE | Number of times an upload record could not find its file | | +| `failed_login_captcha_total` | Gauge | 11.0 | Counter of failed CAPTCHA attempts during login | | +| `successful_login_captcha_total` | Gauge | 11.0 | Counter of successful CAPTCHA attempts during login | | ## Metrics controlled by a feature flag The following metrics can be controlled by feature flags: -| Metric | Feature Flag | -|:-------------------------------------------------------------|:-----------------------------------------------------------------| -| gitlab_method_call_duration_seconds | prometheus_metrics_method_instrumentation | -| gitlab_transaction_allocated_memory_bytes | prometheus_metrics_transaction_allocated_memory | -| gitlab_transaction_event_build_found_total | prometheus_transaction_event_build_found_total | -| gitlab_transaction_event_build_invalid_total | prometheus_transaction_event_build_invalid_total | -| gitlab_transaction_event_build_not_found_cached_total | prometheus_transaction_event_build_not_found_cached_total | -| gitlab_transaction_event_build_not_found_total | prometheus_transaction_event_build_not_found_total | -| gitlab_transaction_event_change_default_branch_total | prometheus_transaction_event_change_default_branch_total | -| gitlab_transaction_event_create_repository_total | prometheus_transaction_event_create_repository_total | -| gitlab_transaction_event_etag_caching_cache_hit_total | prometheus_transaction_event_etag_caching_cache_hit_total | -| gitlab_transaction_event_etag_caching_header_missing_total | prometheus_transaction_event_etag_caching_header_missing_total | -| gitlab_transaction_event_etag_caching_key_not_found_total | prometheus_transaction_event_etag_caching_key_not_found_total | -| gitlab_transaction_event_etag_caching_middleware_used_total | prometheus_transaction_event_etag_caching_middleware_used_total | -| gitlab_transaction_event_etag_caching_resource_changed_total | prometheus_transaction_event_etag_caching_resource_changed_total | -| gitlab_transaction_event_fork_repository_total | prometheus_transaction_event_fork_repository_total | -| gitlab_transaction_event_import_repository_total | prometheus_transaction_event_import_repository_total | -| gitlab_transaction_event_push_branch_total | prometheus_transaction_event_push_branch_total | -| gitlab_transaction_event_push_commit_total | prometheus_transaction_event_push_commit_total | -| gitlab_transaction_event_push_tag_total | prometheus_transaction_event_push_tag_total | -| gitlab_transaction_event_rails_exception_total | prometheus_transaction_event_rails_exception_total | -| gitlab_transaction_event_receive_email_total | prometheus_transaction_event_receive_email_total | -| gitlab_transaction_event_remote_mirrors_failed_total | prometheus_transaction_event_remote_mirrors_failed_total | -| gitlab_transaction_event_remote_mirrors_finished_total | prometheus_transaction_event_remote_mirrors_finished_total | -| gitlab_transaction_event_remote_mirrors_running_total | prometheus_transaction_event_remote_mirrors_running_total | -| gitlab_transaction_event_remove_branch_total | prometheus_transaction_event_remove_branch_total | -| gitlab_transaction_event_remove_repository_total | prometheus_transaction_event_remove_repository_total | -| gitlab_transaction_event_remove_tag_total | prometheus_transaction_event_remove_tag_total | -| gitlab_transaction_event_sidekiq_exception_total | prometheus_transaction_event_sidekiq_exception_total | -| gitlab_transaction_event_stuck_import_jobs_total | prometheus_transaction_event_stuck_import_jobs_total | -| gitlab_transaction_event_update_build_total | prometheus_transaction_event_update_build_total | -| gitlab_view_rendering_duration_seconds | prometheus_metrics_view_instrumentation | +| Metric | Feature Flag | +|:---------------------------------------------------------------|:-------------------------------------------------------------------| +| `gitlab_method_call_duration_seconds` | `prometheus_metrics_method_instrumentation` | +| `gitlab_transaction_allocated_memory_bytes` | `prometheus_metrics_transaction_allocated_memory` | +| `gitlab_transaction_event_build_found_total` | `prometheus_transaction_event_build_found_total` | +| `gitlab_transaction_event_build_invalid_total` | `prometheus_transaction_event_build_invalid_total` | +| `gitlab_transaction_event_build_not_found_cached_total` | `prometheus_transaction_event_build_not_found_cached_total` | +| `gitlab_transaction_event_build_not_found_total` | `prometheus_transaction_event_build_not_found_total` | +| `gitlab_transaction_event_change_default_branch_total` | `prometheus_transaction_event_change_default_branch_total` | +| `gitlab_transaction_event_create_repository_total` | `prometheus_transaction_event_create_repository_total` | +| `gitlab_transaction_event_etag_caching_cache_hit_total` | `prometheus_transaction_event_etag_caching_cache_hit_total` | +| `gitlab_transaction_event_etag_caching_header_missing_total` | `prometheus_transaction_event_etag_caching_header_missing_total` | +| `gitlab_transaction_event_etag_caching_key_not_found_total` | `prometheus_transaction_event_etag_caching_key_not_found_total` | +| `gitlab_transaction_event_etag_caching_middleware_used_total` | `prometheus_transaction_event_etag_caching_middleware_used_total` | +| `gitlab_transaction_event_etag_caching_resource_changed_total` | `prometheus_transaction_event_etag_caching_resource_changed_total` | +| `gitlab_transaction_event_fork_repository_total` | `prometheus_transaction_event_fork_repository_total` | +| `gitlab_transaction_event_import_repository_total` | `prometheus_transaction_event_import_repository_total` | +| `gitlab_transaction_event_push_branch_total` | `prometheus_transaction_event_push_branch_total` | +| `gitlab_transaction_event_push_commit_total` | `prometheus_transaction_event_push_commit_total` | +| `gitlab_transaction_event_push_tag_total` | `prometheus_transaction_event_push_tag_total` | +| `gitlab_transaction_event_rails_exception_total` | `prometheus_transaction_event_rails_exception_total` | +| `gitlab_transaction_event_receive_email_total` | `prometheus_transaction_event_receive_email_total` | +| `gitlab_transaction_event_remote_mirrors_failed_total` | `prometheus_transaction_event_remote_mirrors_failed_total` | +| `gitlab_transaction_event_remote_mirrors_finished_total` | `prometheus_transaction_event_remote_mirrors_finished_total` | +| `gitlab_transaction_event_remote_mirrors_running_total` | `prometheus_transaction_event_remote_mirrors_running_total` | +| `gitlab_transaction_event_remove_branch_total` | `prometheus_transaction_event_remove_branch_total` | +| `gitlab_transaction_event_remove_repository_total` | `prometheus_transaction_event_remove_repository_total` | +| `gitlab_transaction_event_remove_tag_total` | `prometheus_transaction_event_remove_tag_total` | +| `gitlab_transaction_event_sidekiq_exception_total` | `prometheus_transaction_event_sidekiq_exception_total` | +| `gitlab_transaction_event_stuck_import_jobs_total` | `prometheus_transaction_event_stuck_import_jobs_total` | +| `gitlab_transaction_event_update_build_total` | `prometheus_transaction_event_update_build_total` | +| `gitlab_view_rendering_duration_seconds` | `prometheus_metrics_view_instrumentation` | ## Sidekiq Metrics available for Geo **(PREMIUM)** Sidekiq jobs may also gather metrics, and these metrics can be accessed if the Sidekiq exporter is enabled (e.g. via the `monitoring.sidekiq_exporter` configuration option in `gitlab.yml`. -| Metric | Type | Since | Description | Labels | -|:-------------------------------------------- |:------- |:----- |:----------- |:------ | -| geo_db_replication_lag_seconds | Gauge | 10.2 | Database replication lag (seconds) | url -| geo_repositories | Gauge | 10.2 | Total number of repositories available on primary | url -| geo_repositories_synced | Gauge | 10.2 | Number of repositories synced on secondary | url -| geo_repositories_failed | Gauge | 10.2 | Number of repositories failed to sync on secondary | url -| geo_lfs_objects | Gauge | 10.2 | Total number of LFS objects available on primary | url -| geo_lfs_objects_synced | Gauge | 10.2 | Number of LFS objects synced on secondary | url -| geo_lfs_objects_failed | Gauge | 10.2 | Number of LFS objects failed to sync on secondary | url -| geo_attachments | Gauge | 10.2 | Total number of file attachments available on primary | url -| geo_attachments_synced | Gauge | 10.2 | Number of attachments synced on secondary | url -| geo_attachments_failed | Gauge | 10.2 | Number of attachments failed to sync on secondary | url -| geo_last_event_id | Gauge | 10.2 | Database ID of the latest event log entry on the primary | url -| geo_last_event_timestamp | Gauge | 10.2 | UNIX timestamp of the latest event log entry on the primary | url -| geo_cursor_last_event_id | Gauge | 10.2 | Last database ID of the event log processed by the secondary | url -| geo_cursor_last_event_timestamp | Gauge | 10.2 | Last UNIX timestamp of the event log processed by the secondary | url -| geo_status_failed_total | Counter | 10.2 | Number of times retrieving the status from the Geo Node failed | url -| geo_last_successful_status_check_timestamp | Gauge | 10.2 | Last timestamp when the status was successfully updated | url -| geo_lfs_objects_synced_missing_on_primary | Gauge | 10.7 | Number of LFS objects marked as synced due to the file missing on the primary | url -| geo_job_artifacts_synced_missing_on_primary | Gauge | 10.7 | Number of job artifacts marked as synced due to the file missing on the primary | url -| geo_attachments_synced_missing_on_primary | Gauge | 10.7 | Number of attachments marked as synced due to the file missing on the primary | url -| geo_repositories_checksummed_count | Gauge | 10.7 | Number of repositories checksummed on primary | url -| geo_repositories_checksum_failed_count | Gauge | 10.7 | Number of repositories failed to calculate the checksum on primary | url -| geo_wikis_checksummed_count | Gauge | 10.7 | Number of wikis checksummed on primary | url -| geo_wikis_checksum_failed_count | Gauge | 10.7 | Number of wikis failed to calculate the checksum on primary | url -| geo_repositories_verified_count | Gauge | 10.7 | Number of repositories verified on secondary | url -| geo_repositories_verification_failed_count | Gauge | 10.7 | Number of repositories failed to verify on secondary | url -| geo_repositories_checksum_mismatch_count | Gauge | 10.7 | Number of repositories that checksum mismatch on secondary | url -| geo_wikis_verified_count | Gauge | 10.7 | Number of wikis verified on secondary | url -| geo_wikis_verification_failed_count | Gauge | 10.7 | Number of wikis failed to verify on secondary | url -| geo_wikis_checksum_mismatch_count | Gauge | 10.7 | Number of wikis that checksum mismatch on secondary | url -| geo_repositories_checked_count | Gauge | 11.1 | Number of repositories that have been checked via `git fsck` | url -| geo_repositories_checked_failed_count | Gauge | 11.1 | Number of repositories that have a failure from `git fsck` | url -| geo_repositories_retrying_verification_count | Gauge | 11.2 | Number of repositories verification failures that Geo is actively trying to correct on secondary | url -| geo_wikis_retrying_verification_count | Gauge | 11.2 | Number of wikis verification failures that Geo is actively trying to correct on secondary | url +| Metric | Type | Since | Description | Labels | +|:---------------------------------------------- |:------- |:----- |:----------- |:------ | +| `geo_db_replication_lag_seconds` | Gauge | 10.2 | Database replication lag (seconds) | url | +| `geo_repositories` | Gauge | 10.2 | Total number of repositories available on primary | url | +| `geo_repositories_synced` | Gauge | 10.2 | Number of repositories synced on secondary | url | +| `geo_repositories_failed` | Gauge | 10.2 | Number of repositories failed to sync on secondary | url | +| `geo_lfs_objects` | Gauge | 10.2 | Total number of LFS objects available on primary | url | +| `geo_lfs_objects_synced` | Gauge | 10.2 | Number of LFS objects synced on secondary | url | +| `geo_lfs_objects_failed` | Gauge | 10.2 | Number of LFS objects failed to sync on secondary | url | +| `geo_attachments` | Gauge | 10.2 | Total number of file attachments available on primary | url | +| `geo_attachments_synced` | Gauge | 10.2 | Number of attachments synced on secondary | url | +| `geo_attachments_failed` | Gauge | 10.2 | Number of attachments failed to sync on secondary | url | +| `geo_last_event_id` | Gauge | 10.2 | Database ID of the latest event log entry on the primary | url | +| `geo_last_event_timestamp` | Gauge | 10.2 | UNIX timestamp of the latest event log entry on the primary | url | +| `geo_cursor_last_event_id` | Gauge | 10.2 | Last database ID of the event log processed by the secondary | url | +| `geo_cursor_last_event_timestamp` | Gauge | 10.2 | Last UNIX timestamp of the event log processed by the secondary | url | +| `geo_status_failed_total` | Counter | 10.2 | Number of times retrieving the status from the Geo Node failed | url | +| `geo_last_successful_status_check_timestamp` | Gauge | 10.2 | Last timestamp when the status was successfully updated | url | +| `geo_lfs_objects_synced_missing_on_primary` | Gauge | 10.7 | Number of LFS objects marked as synced due to the file missing on the primary | url | +| `geo_job_artifacts_synced_missing_on_primary` | Gauge | 10.7 | Number of job artifacts marked as synced due to the file missing on the primary | url | +| `geo_attachments_synced_missing_on_primary` | Gauge | 10.7 | Number of attachments marked as synced due to the file missing on the primary | url | +| `geo_repositories_checksummed_count` | Gauge | 10.7 | Number of repositories checksummed on primary | url | +| `geo_repositories_checksum_failed_count` | Gauge | 10.7 | Number of repositories failed to calculate the checksum on primary | url | +| `geo_wikis_checksummed_count` | Gauge | 10.7 | Number of wikis checksummed on primary | url | +| `geo_wikis_checksum_failed_count` | Gauge | 10.7 | Number of wikis failed to calculate the checksum on primary | url | +| `geo_repositories_verified_count` | Gauge | 10.7 | Number of repositories verified on secondary | url | +| `geo_repositories_verification_failed_count` | Gauge | 10.7 | Number of repositories failed to verify on secondary | url | +| `geo_repositories_checksum_mismatch_count` | Gauge | 10.7 | Number of repositories that checksum mismatch on secondary | url | +| `geo_wikis_verified_count` | Gauge | 10.7 | Number of wikis verified on secondary | url | +| `geo_wikis_verification_failed_count` | Gauge | 10.7 | Number of wikis failed to verify on secondary | url | +| `geo_wikis_checksum_mismatch_count` | Gauge | 10.7 | Number of wikis that checksum mismatch on secondary | url | +| `geo_repositories_checked_count` | Gauge | 11.1 | Number of repositories that have been checked via `git fsck` | url | +| `geo_repositories_checked_failed_count` | Gauge | 11.1 | Number of repositories that have a failure from `git fsck` | url | +| `geo_repositories_retrying_verification_count` | Gauge | 11.2 | Number of repositories verification failures that Geo is actively trying to correct on secondary | url | +| `geo_wikis_retrying_verification_count` | Gauge | 11.2 | Number of wikis verification failures that Geo is actively trying to correct on secondary | url | ### Ruby metrics Some basic Ruby runtime metrics are available: -| Metric | Type | Since | Description | -|:-------------------------------------- |:--------- |:----- |:----------- | -| ruby_gc_duration_seconds | Counter | 11.1 | Time spent by Ruby in GC | -| ruby_gc_stat_... | Gauge | 11.1 | Various metrics from [GC.stat] | -| ruby_file_descriptors | Gauge | 11.1 | File descriptors per process | -| ruby_memory_bytes | Gauge | 11.1 | Memory usage by process | -| ruby_sampler_duration_seconds | Counter | 11.1 | Time spent collecting stats | -| ruby_process_cpu_seconds_total | Gauge | 12.0 | Total amount of CPU time per process | -| ruby_process_max_fds | Gauge | 12.0 | Maximum number of open file descriptors per process | -| ruby_process_resident_memory_bytes | Gauge | 12.0 | Memory usage by process, measured in bytes | -| ruby_process_start_time_seconds | Gauge | 12.0 | UNIX timestamp of process start time | +| Metric | Type | Since | Description | +|:------------------------------------ |:--------- |:----- |:----------- | +| `ruby_gc_duration_seconds` | Counter | 11.1 | Time spent by Ruby in GC | +| `ruby_gc_stat_...` | Gauge | 11.1 | Various metrics from [GC.stat] | +| `ruby_file_descriptors` | Gauge | 11.1 | File descriptors per process | +| `ruby_memory_bytes` | Gauge | 11.1 | Memory usage by process | +| `ruby_sampler_duration_seconds` | Counter | 11.1 | Time spent collecting stats | +| `ruby_process_cpu_seconds_total` | Gauge | 12.0 | Total amount of CPU time per process | +| `ruby_process_max_fds` | Gauge | 12.0 | Maximum number of open file descriptors per process | +| `ruby_process_resident_memory_bytes` | Gauge | 12.0 | Memory usage by process, measured in bytes | +| `ruby_process_start_time_seconds` | Gauge | 12.0 | UNIX timestamp of process start time | [GC.stat]: https://ruby-doc.org/core-2.6.3/GC.html#method-c-stat @@ -183,28 +183,28 @@ Some basic Ruby runtime metrics are available: Unicorn specific metrics, when Unicorn is used. -| Metric | Type | Since | Description | -|:---------------------------|:------|:------|:---------------------------------------------------| -| unicorn_active_connections | Gauge | 11.0 | The number of active Unicorn connections (workers) | -| unicorn_queued_connections | Gauge | 11.0 | The number of queued Unicorn connections | -| unicorn_workers | Gauge | 12.0 | The number of Unicorn workers | +| Metric | Type | Since | Description | +|:-----------------------------|:------|:------|:---------------------------------------------------| +| `unicorn_active_connections` | Gauge | 11.0 | The number of active Unicorn connections (workers) | +| `unicorn_queued_connections` | Gauge | 11.0 | The number of queued Unicorn connections | +| `unicorn_workers` | Gauge | 12.0 | The number of Unicorn workers | ## Puma Metrics **(EXPERIMENTAL)** -When Puma is used instead of Unicorn, following metrics are available: - -| Metric | Type | Since | Description | -|:-------------------------------------------- |:------- |:----- |:----------- | -| puma_workers | Gauge | 12.0 | Total number of workers | -| puma_running_workers | Gauge | 12.0 | Number of booted workers | -| puma_stale_workers | Gauge | 12.0 | Number of old workers | -| puma_running | Gauge | 12.0 | Number of running threads | -| puma_queued_connections | Gauge | 12.0 | Number of connections in that worker's "todo" set waiting for a worker thread | -| puma_active_connections | Gauge | 12.0 | Number of threads processing a request | -| puma_pool_capacity | Gauge | 12.0 | Number of requests the worker is capable of taking right now | -| puma_max_threads | Gauge | 12.0 | Maximum number of worker threads | -| puma_idle_threads | Gauge | 12.0 | Number of spawned threads which are not processing a request | -| puma_killer_terminations_total | Gauge | 12.0 | Number of workers terminated by PumaWorkerKiller | +When Puma is used instead of Unicorn, the following metrics are available: + +| Metric | Type | Since | Description | +|:---------------------------------------------- |:------- |:----- |:----------- | +| `puma_workers` | Gauge | 12.0 | Total number of workers | +| `puma_running_workers` | Gauge | 12.0 | Number of booted workers | +| `puma_stale_workers` | Gauge | 12.0 | Number of old workers | +| `puma_running` | Gauge | 12.0 | Number of running threads | +| `puma_queued_connections` | Gauge | 12.0 | Number of connections in that worker's "todo" set waiting for a worker thread | +| `puma_active_connections` | Gauge | 12.0 | Number of threads processing a request | +| `puma_pool_capacity` | Gauge | 12.0 | Number of requests the worker is capable of taking right now | +| `puma_max_threads` | Gauge | 12.0 | Maximum number of worker threads | +| `puma_idle_threads` | Gauge | 12.0 | Number of spawned threads which are not processing a request | +| `puma_killer_terminations_total` | Gauge | 12.0 | Number of workers terminated by PumaWorkerKiller | ## Metrics shared directory @@ -221,9 +221,3 @@ If GitLab is installed using Omnibus and `tmpfs` is available then metrics directory will be automatically configured. [← Back to the main Prometheus page](index.md) - -[29118]: https://gitlab.com/gitlab-org/gitlab-ce/issues/29118 -[Prometheus]: https://prometheus.io -[restart]: ../../restart_gitlab.md#omnibus-gitlab-restart -[whitelist]: ../ip_whitelist.md -[reconfigure]: ../../restart_gitlab.md#omnibus-gitlab-reconfigure diff --git a/doc/administration/repository_storage_types.md b/doc/administration/repository_storage_types.md index 142bcc65561..3ee8673b297 100644 --- a/doc/administration/repository_storage_types.md +++ b/doc/administration/repository_storage_types.md @@ -82,19 +82,17 @@ by another folder with the next 2 characters. They are both stored in a special > [Introduced](https://gitlab.com/gitlab-org/gitaly/issues/1606) in GitLab 12.1. -Forks of public projects are deduplicated by creating a third repository, the object pool, containing the objects from the source project. Using `objects/info/alternates`, the source project and forks use the object pool for shared objects. Objects are moved from the source project to the object pool when housekeeping is run on the source project. +Forks of public projects are deduplicated by creating a third repository, the +object pool, containing the objects from the source project. Using +`objects/info/alternates`, the source project and forks use the object pool for +shared objects. Objects are moved from the source project to the object pool +when housekeeping is run on the source project. ```ruby # object pool paths "@pools/#{hash[0..1]}/#{hash[2..3]}/#{hash}.git" ``` -Object pools can be disabled using the `object_pools` feature flag, and can be -disabled for individual projects by executing -`Feature.disable(:object_pools, Project.find(<id>))`. Disabling object pools -will not change existing deduplicated forks, but will prevent new forks from -being deduplicated. - DANGER: **Danger:** Do not run `git prune` or `git gc` in pool repositories! This can cause data loss in "real" repositories that depend on the pool in diff --git a/doc/administration/smime_signing_email.md b/doc/administration/smime_signing_email.md index 9f719088f25..b2e3bf8487b 100644 --- a/doc/administration/smime_signing_email.md +++ b/doc/administration/smime_signing_email.md @@ -22,7 +22,7 @@ email_smime: ``` - Both files must be provided PEM-encoded. -- The key file must be unencrypted so that Gitlab can read it without user +- The key file must be unencrypted so that Gitlab can read it without user intervention. NOTE: **Note:** Be mindful of the access levels for your private keys and visibility to diff --git a/doc/api/events.md b/doc/api/events.md index 6dca8e52f69..1cd7047b867 100644 --- a/doc/api/events.md +++ b/doc/api/events.md @@ -70,7 +70,7 @@ Parameters: Example request: -``` +```bash curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/events?target_type=issue&action=created&after=2017-01-31&before=2017-03-01 ``` @@ -275,7 +275,7 @@ Parameters: Example request: -``` +```bash curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:project_id/events?target_type=issue&action=created&after=2017-01-31&before=2017-03-01 ``` @@ -343,8 +343,8 @@ Example response: "username": "root", "id": 1, "state": "active", - "avatar_url": "http://localhost:3000/uploads/user/avatar/1/fox_avatar.png", - "web_url": "http://localhost:3000/root" + "avatar_url": "https://gitlab.example.com/uploads/user/avatar/1/fox_avatar.png", + "web_url": "https://gitlab.example.com/root" }, "created_at": "2015-12-04T10:33:56.698Z", "system": false, @@ -357,8 +357,8 @@ Example response: "username": "root", "id": 1, "state": "active", - "avatar_url": "http://localhost:3000/uploads/user/avatar/1/fox_avatar.png", - "web_url": "http://localhost:3000/root" + "avatar_url": "https://gitlab.example.com/uploads/user/avatar/1/fox_avatar.png", + "web_url": "https://gitlab.example.com/root" }, "author_username": "root" } diff --git a/doc/api/issues.md b/doc/api/issues.md index 8313dd2c3bd..cadc9291489 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -284,7 +284,6 @@ Example response: "award_emoji":"http://example.com/api/v4/projects/4/issues/41/award_emoji", "project":"http://example.com/api/v4/projects/4" }, - "subscribed": false, "task_completion_status":{ "count":0, "completed_count":0 diff --git a/doc/api/labels.md b/doc/api/labels.md index fde1d861cf6..9692cc8b710 100644 --- a/doc/api/labels.md +++ b/doc/api/labels.md @@ -186,6 +186,40 @@ Example response: } ``` +## Promote a project label to a group label + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/25218) in GitLab 12.3. + +Promotes a project label to a group label. + +``` +PUT /projects/:id/labels/promote +``` + +| Attribute | Type | Required | Description | +| --------------- | ------- | --------------------------------- | ------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | +| `name` | string | yes | The name of the existing label | + +```bash +curl --request PUT --data "name=documentation" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/labels/promote" +``` + +Example response: + +```json +{ + "id" : 8, + "name" : "documentation", + "color" : "#8E44AD", + "description": "Documentation", + "open_issues_count": 1, + "closed_issues_count": 0, + "open_merge_requests_count": 2, + "subscribed": false +} +``` + ## Subscribe to a label Subscribes the authenticated user to a label to receive notifications. diff --git a/doc/api/projects.md b/doc/api/projects.md index 9f392418153..cf28ea84704 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -852,10 +852,10 @@ Get the users list of a project. GET /projects/:id/users ``` -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `search` | string | no | Search for specific users | -| `skip_users` | array[int] | no | Filter out users with the specified IDs | +| Attribute | Type | Required | Description | +| ------------ | ------------- | -------- | ----------- | +| `search` | string | no | Search for specific users | +| `skip_users` | integer array | no | Filter out users with the specified IDs | ```json [ diff --git a/doc/ci/caching/index.md b/doc/ci/caching/index.md index f8151e3e18c..a59a0477b80 100644 --- a/doc/ci/caching/index.md +++ b/doc/ci/caching/index.md @@ -172,6 +172,29 @@ job: cache: {} ``` +### Inherit global config, but override specific settings per job + +You can override cache settings without overwriting the global cache by using +[anchors](../yaml/README.md#anchors). For example, if you want to override the +`policy` for one job: + +```yaml +cache: &global_cache + key: ${CI_COMMIT_REF_SLUG} + paths: + - node_modules/ + - public/ + - vendor/ + policy: pull-push + +job: + cache: + # inherit all global cache settings + <<: *global_cache + # override the policy + policy: pull +``` + For more fine tuning, read also about the [`cache: policy`](../yaml/README.md#cachepolicy). diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 89a61b2a9e3..1368764bcf8 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -100,6 +100,7 @@ The following table lists available parameters for jobs: | [`stage`](#stage) | Defines a job stage (default: `test`). | | [`only`](#onlyexcept-basic) | Limit when jobs are created. Also available: [`only:refs`, `only:kubernetes`, `only:variables`, and `only:changes`](#onlyexcept-advanced). | | [`except`](#onlyexcept-basic) | Limit when jobs are not created. Also available: [`except:refs`, `except:kubernetes`, `except:variables`, and `except:changes`](#onlyexcept-advanced). | +| [`rules`](#rules) | List of coniditions to evaluate and determine selected attributes of a build and whether or not it is created. May not be used alongside `only`/`except`. | [`tags`](#tags) | List of tags which are used to select Runner. | | [`allow_failure`](#allow_failure) | Allow job to fail. Failed job doesn't contribute to commit status. | | [`when`](#when) | When to run job. Also available: `when:manual` and `when:delayed`. | @@ -386,7 +387,7 @@ In addition, `only` and `except` allow the use of special keywords: | `triggers` | For pipelines created using a trigger token. | | `web` | For pipelines created using **Run pipeline** button in GitLab UI (under your project's **Pipelines**). | | `merge_requests` | When a merge request is created or updated (See [pipelines for merge requests](../merge_request_pipelines/index.md)). | -| `chats` | For jobs created using a [GitLab ChatOps](../chatops/README.md) command. | +| `chat` | For jobs created using a [GitLab ChatOps](../chatops/README.md) command. | In the example below, `job` will run only for refs that start with `issue-`, whereas all branches will be skipped: @@ -690,6 +691,125 @@ In the scenario above, if a merge request is created or updated that changes either files in `service-one` directory or the `Dockerfile`, GitLab creates and triggers the `docker build service one` job. +### `rules` + +Using `rules` allows for a list of individual rule objects to be evaluated +*in order*, until one matches and dynamically provides attributes to the job. + +Available rule clauses include: + +- `if` (similar to [`only:variables`](#onlyvariablesexceptvariables)). +- `changes` (same as [`only:changes`](#onlychangesexceptchanges)). + +For example, using `if`: + +```yaml +job: + script: "echo Hello, Rules!" + rules: + - if: '$CI_MERGE_REQUEST_TARGET_BRANCH == "master"' # This rule will be evaluated + when: always + - if: '$VAR =~ /pattern/' # This rule will only be evaluated if the first does not match + when: manual + - when: on_success # A Rule entry with no conditional clauses evaluates to true. If neither of the first two Rules match, this one will and set job:when to "on_success" +``` + +If the first rule does not match, further rules will be evaluated sequentially +until a match is found. The above configuration will specify that `job` should +be built and run for every pipeline on merge requests targeting `master`, +regardless of the status of other builds. + +#### `rules:if` + +`rules:if` differs slightly from `only:variables` by accepting only a single +expression string, rather than an array of them. Any set of expressions to be +evaluated should be conjoined into a single expression using `&&` or `||`. For example: + +```yaml +job: + script: "echo Hello, Rules!" + rules: + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH =~ /^feature/ && $CI_MERGE_REQUEST_TARGET_BRANCH == "master"' # This rule will be evaluated + when: always + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH =~ /^feature/' # This rule will only be evaluated if the target branch is not "master" + when: manual + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH' # If neither of the first two match but the simple presence does, we set to "on_success" by default +``` + +If none of the provided rules match, the job will be set to `when:never`, and +not included in the pipeline. If `rules:when` is not included in the configuration +at all, the behavior defaults to `job:when`, which continues to default to +`on_success`. + +#### `rules:changes` + +`changes` works exactly the same way as [`only`/`except`](#onlychangesexceptchanges), +accepting an array of paths. The following configuration configures a job to be +run manually if `Dockerfile` has changed OR `$VAR == "string value"`. Otherwise +it is set to `when:on_success` by the last rule, where 0 clauses evaluate as +vacuously true. + +```yaml +docker build: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - changes: # Will include the job and set to when:manual if any of the follow paths match a modified file. + - Dockerfile + when: manual + - if: '$VAR == "string value"' + when: manual # Will include the job and set to when:manual if the expression evaluates to true, after the `changes:` rule fails to match. + - when: on_success # If neither of the first rules match, set to on_success + +``` + +#### Complex Rule Clauses + +To conjoin `if` and `changes` clauses with an AND, use them in the same rule. +Here we run the job manually if `Dockerfile` or any file in `docker/scripts/` +has changed AND `$VAR == "string value"`. Otherwise, the job will not be +included in the pipeline. + +```yaml +docker build: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - if: '$VAR == "string value"' + changes: # Will include the job and set to when:manual if any of the follow paths match a modified file. + - Dockerfile + - docker/scripts/* + when: manual + # - when: never would be redundant here, this is implied any time rules are listed. +``` + +The only clauses currently available are `if` and `changes`. Keywords such as +`branches` or `refs` that are currently available for `only`/`except` are not +yet available in `rules` as they are being individually considered for their +usage and behavior in the newer context. + +#### Permitted attributes + +The only job attributes currently set by `rules` are `when` and `start_in`, if +`when` is set to `delayed`. A job will be included in a pipeline if `when` is +evaluated to any value except `never`. + +Delayed jobs require a `start_in` value, so rule objects do as well. For example: + +```yaml +docker build: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - changes: # Will include the job and delay 3 hours when the Dockerfile has changed + - Dockerfile + when: delayed + start_in: '3 hours' + - when: on_success # Otherwise include the job and set to run normally + +``` + +Additional Job configuration may be added to rules in the future, if something +useful isn't available, please open an issue on +[Gitlab CE](https://www.gitlab.com/gitlab-org/gitlab-ce/issues). + ### `tags` `tags` is used to select specific Runners from the list of all Runners that are @@ -1700,19 +1820,19 @@ mac:build: linux:rspec: stage: test - needs: [linux:build] + needs: ["linux:build"] linux:rubocop: stage: test - needs: [linux:build] + needs: ["linux:build"] mac:rspec: stage: test - needs: [mac:build] + needs: ["mac:build"] mac:rubocop: stage: test - needs: [mac:build] + needs: ["mac:build"] production: stage: deploy diff --git a/doc/development/architecture.md b/doc/development/architecture.md index 5cb2ddf6e52..2adca2dae28 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -12,21 +12,21 @@ New versions of GitLab are released in stable branches and the master branch is For information, see the [GitLab Release Process](https://gitlab.com/gitlab-org/release/docs/tree/master#gitlab-release-process). -Both EE and CE require some add-on components called gitlab-shell and Gitaly. These components are available from the [gitlab-shell](https://gitlab.com/gitlab-org/gitlab-shell/tree/master) and [gitaly](https://gitlab.com/gitlab-org/gitaly/tree/master) repositories respectively. New versions are usually tags but staying on the master branch will give you the latest stable version. New releases are generally around the same time as GitLab CE releases with exception for informal security updates deemed critical. +Both EE and CE require some add-on components called GitLab Shell and Gitaly. These components are available from the [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell/tree/master) and [Gitaly](https://gitlab.com/gitlab-org/gitaly/tree/master) repositories respectively. New versions are usually tags but staying on the master branch will give you the latest stable version. New releases are generally around the same time as GitLab CE releases with exception for informal security updates deemed critical. ## Components A typical install of GitLab will be on GNU/Linux. It uses Nginx or Apache as a web front end to proxypass the Unicorn web server. By default, communication between Unicorn and the front end is via a Unix domain socket but forwarding requests via TCP is also supported. The web front end accesses `/home/git/gitlab/public` bypassing the Unicorn server to serve static pages, uploads (e.g. avatar images or attachments), and precompiled assets. GitLab serves web pages and a [GitLab API](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/api) using the Unicorn web server. It uses Sidekiq as a job queue which, in turn, uses redis as a non-persistent database backend for job information, meta data, and incoming jobs. -We also support deploying GitLab on Kubernetes using our [gitlab Helm chart](https://docs.gitlab.com/charts/). +We also support deploying GitLab on Kubernetes using our [GitLab Helm chart](https://docs.gitlab.com/charts/). -The GitLab web app uses PostgreSQL for persistent database information (e.g. users, permissions, issues, other meta data). GitLab stores the bare git repositories it serves in `/home/git/repositories` by default. It also keeps default branch and hook information with the bare repository. +The GitLab web app uses PostgreSQL for persistent database information (e.g. users, permissions, issues, other meta data). GitLab stores the bare Git repositories it serves in `/home/git/repositories` by default. It also keeps default branch and hook information with the bare repository. -When serving repositories over HTTP/HTTPS GitLab utilizes the GitLab API to resolve authorization and access as well as serving git objects. +When serving repositories over HTTP/HTTPS GitLab utilizes the GitLab API to resolve authorization and access as well as serving Git objects. -The add-on component gitlab-shell serves repositories over SSH. It manages the SSH keys within `/home/git/.ssh/authorized_keys` which should not be manually edited. gitlab-shell accesses the bare repositories through Gitaly to serve git objects and communicates with redis to submit jobs to Sidekiq for GitLab to process. gitlab-shell queries the GitLab API to determine authorization and access. +The add-on component GitLab Shell serves repositories over SSH. It manages the SSH keys within `/home/git/.ssh/authorized_keys` which should not be manually edited. GitLab Shell accesses the bare repositories through Gitaly to serve Git objects and communicates with redis to submit jobs to Sidekiq for GitLab to process. GitLab Shell queries the GitLab API to determine authorization and access. -Gitaly executes git operations from gitlab-shell and the GitLab web app, and provides an API to the GitLab web app to get attributes from git (e.g. title, branches, tags, other meta data), and to get blobs (e.g. diffs, commits, files). +Gitaly executes Git operations from GitLab Shell and the GitLab web app, and provides an API to the GitLab web app to get attributes from Git (e.g. title, branches, tags, other meta data), and to get blobs (e.g. diffs, commits, files). You may also be interested in the [production architecture of GitLab.com](https://about.gitlab.com/handbook/engineering/infrastructure/production-architecture/). @@ -130,7 +130,7 @@ Component statuses are linked to configuration documentation for each component. | [NGINX](#nginx) | Routes requests to appropriate components, terminates SSL | [✅][nginx-omnibus] | [✅][nginx-charts] | [⚙][nginx-charts] | [✅](https://about.gitlab.com/handbook/engineering/infrastructure/production-architecture/#service-architecture) | [⤓][nginx-source] | ❌ | CE & EE | | [Unicorn (GitLab Rails)](#unicorn) | Handles requests for the web interface and API | [✅][unicorn-omnibus] | [✅][unicorn-charts] | [✅][unicorn-charts] | [✅](../user/gitlab_com/index.md#unicorn) | [⚙][unicorn-source] | [✅][gitlab-yml] | CE & EE | | [Sidekiq](#sidekiq) | Background jobs processor | [✅][sidekiq-omnibus] | [✅][sidekiq-charts] | [✅](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/index.html) | [✅](../user/gitlab_com/index.md#sidekiq) | [✅][gitlab-yml] | [✅][gitlab-yml] | CE & EE | -| [Gitaly](#gitaly) | Git RPC service for handling all git calls made by GitLab | [✅][gitaly-omnibus] | [✅][gitaly-charts] | [✅][gitaly-charts] | [✅](https://about.gitlab.com/handbook/engineering/infrastructure/production-architecture/#service-architecture) | [⚙][gitaly-source] | ✅ | CE & EE | +| [Gitaly](#gitaly) | Git RPC service for handling all Git calls made by GitLab | [✅][gitaly-omnibus] | [✅][gitaly-charts] | [✅][gitaly-charts] | [✅](https://about.gitlab.com/handbook/engineering/infrastructure/production-architecture/#service-architecture) | [⚙][gitaly-source] | ✅ | CE & EE | | [GitLab Workhorse](#gitlab-workhorse) | Smart reverse proxy, handles large HTTP requests | [✅][workhorse-omnibus] | [✅][workhorse-charts] | [✅][workhorse-charts] | [✅](https://about.gitlab.com/handbook/engineering/infrastructure/production-architecture/#service-architecture) | [⚙][workhorse-source] | ✅ | CE & EE | | [GitLab Shell](#gitlab-shell) | Handles `git` over SSH sessions | [✅][shell-omnibus] | [✅][shell-charts] | [✅][shell-charts] | [✅](https://about.gitlab.com/handbook/engineering/infrastructure/production-architecture/#service-architecture) | [⚙][shell-source] | [✅][gitlab-yml] | CE & EE | | [GitLab Pages](#gitlab-pages) | Hosts static websites | [⚙][pages-omnibus] | [❌][pages-charts] | [❌][pages-charts] | [✅](../user/gitlab_com/index.md#gitlab-pages) | [⚙][pages-source] | [⚙][pages-gdk] | CE & EE | @@ -185,7 +185,7 @@ GitLab can be considered to have two layers from a process perspective: - Layer: Monitoring - Process: `alertmanager` -[Alert manager](https://prometheus.io/docs/alerting/alertmanager/) is a tool provided by Prometheus that _"handles alerts sent by client applications such as the Prometheus server. It takes care of deduplicating, grouping, and routing them to the correct receiver integration such as email, PagerDuty, or OpsGenie. It also takes care of silencing and inhibition of alerts."_ You can read more in [issue gitlab-ce#45740](https://gitlab.com/gitlab-org/gitlab-ce/issues/45740) about what we will be alerting on. +[Alert manager](https://prometheus.io/docs/alerting/alertmanager/) is a tool provided by Prometheus that _"handles alerts sent by client applications such as the Prometheus server. It takes care of deduplicating, grouping, and routing them to the correct receiver integration such as email, PagerDuty, or OpsGenie. It also takes care of silencing and inhibition of alerts."_ You can read more in [issue #45740](https://gitlab.com/gitlab-org/gitlab-ce/issues/45740) about what we will be alerting on. #### Certificate management @@ -223,12 +223,12 @@ Elasticsearch is a distributed RESTful search engine built for the cloud. Gitaly is a service designed by GitLab to remove our need for NFS for Git storage in distributed deployments of GitLab (think GitLab.com or High Availability Deployments). As of 11.3.0, this service handles all Git level access in GitLab. You can read more about the project [in the project's readme](https://gitlab.com/gitlab-org/gitaly). -#### Gitlab Geo +#### GitLab Geo - Configuration: [Omnibus][geo-omnibus], [Charts][geo-charts], [GDK][geo-gdk] - Layer: Core Service (Processor) -#### Gitlab Monitor +#### GitLab Monitor - [Project page](https://gitlab.com/gitlab-org/gitlab-monitor) - Configuration: [Omnibus][gitlab-monitor-omnibus], [Charts][gitlab-monitor-charts] @@ -237,7 +237,7 @@ Gitaly is a service designed by GitLab to remove our need for NFS for Git storag GitLab Monitor is a process designed in house that allows us to export metrics about GitLab application internals to Prometheus. You can read more [in the project's readme](https://gitlab.com/gitlab-org/gitlab-monitor). -#### Gitlab Pages +#### GitLab Pages - Configuration: [Omnibus][pages-omnibus], [Charts][pages-charts], [Source][pages-source], [GDK][pages-gdk] - Layer: Core Service (Processor) @@ -246,7 +246,7 @@ GitLab Pages is a feature that allows you to publish static websites directly fr You can use it either for personal or business websites, such as portfolios, documentation, manifestos, and business presentations. You can also attribute any license to your content. -#### Gitlab Runner +#### GitLab Runner - [Project page](https://gitlab.com/gitlab-org/gitlab-runner/blob/master/README.md) - Configuration: [Omnibus][runner-omnibus], [Charts][runner-charts], [Source][runner-source], [GDK][runner-gdk] @@ -256,7 +256,7 @@ GitLab Runner runs tests and sends the results to GitLab. GitLab CI is the open-source continuous integration service included with GitLab that coordinates the testing. The old name of this project was GitLab CI Multi Runner but please use "GitLab Runner" (without CI) from now on. -#### Gitlab Shell +#### GitLab Shell - [Project page](https://gitlab.com/gitlab-org/gitlab-shell/blob/master/README.md) - Configuration: [Omnibus][shell-omnibus], [Charts][shell-charts], [Source][shell-source], [GDK][gitlab-yml] @@ -264,7 +264,7 @@ GitLab CI is the open-source continuous integration service included with GitLab [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell) is a program designed at GitLab to handle ssh-based `git` sessions, and modifies the list of authorized keys. GitLab Shell is not a Unix shell nor a replacement for Bash or Zsh. -#### Gitlab Workhorse +#### GitLab Workhorse - [Project page](https://gitlab.com/gitlab-org/gitlab-workhorse/blob/master/README.md) - Configuration: [Omnibus][workhorse-omnibus], [Charts][workhorse-charts], [Source][workhorse-source] @@ -475,7 +475,7 @@ It's important to understand the distinction as some processes are used in both When making a request to an HTTP Endpoint (think `/users/sign_in`) the request will take the following path through the GitLab Service: - nginx - Acts as our first line reverse proxy. -- gitlab-workhorse - This determines if it needs to go to the Rails application or somewhere else to reduce load on Unicorn. +- GitLab Workhorse - This determines if it needs to go to the Rails application or somewhere else to reduce load on Unicorn. - unicorn - Since this is a web request, and it needs to access the application it will go to Unicorn. - Postgres/Gitaly/Redis - Depending on the type of request, it may hit these services to store or retrieve data. @@ -493,13 +493,13 @@ TODO ## System Layout -When referring to `~git` in the pictures it means the home directory of the git user which is typically `/home/git`. +When referring to `~git` in the pictures it means the home directory of the Git user which is typically `/home/git`. GitLab is primarily installed within the `/home/git` user home directory as `git` user. Within the home directory is where the gitlabhq server software resides as well as the repositories (though the repository location is configurable). The bare repositories are located in `/home/git/repositories`. GitLab is a ruby on rails application so the particulars of the inner workings can be learned by studying how a ruby on rails application works. -To serve repositories over SSH there's an add-on application called gitlab-shell which is installed in `/home/git/gitlab-shell`. +To serve repositories over SSH there's an add-on application called GitLab Shell which is installed in `/home/git/gitlab-shell`. ### Installation Folder Summary @@ -523,7 +523,7 @@ processes: `unicorn_rails master` (1 process), `unicorn_rails worker` ### Repository access -Repositories get accessed via HTTP or SSH. HTTP cloning/push/pull utilizes the GitLab API and SSH cloning is handled by gitlab-shell (previously explained). +Repositories get accessed via HTTP or SSH. HTTP cloning/push/pull utilizes the GitLab API and SSH cloning is handled by GitLab Shell (previously explained). ## Troubleshooting @@ -531,28 +531,28 @@ See the README for more information. ### Init scripts of the services -The GitLab init script starts and stops Unicorn and Sidekiq. +The GitLab init script starts and stops Unicorn and Sidekiq: ``` /etc/init.d/gitlab Usage: service gitlab {start|stop|restart|reload|status} ``` -Redis (key-value store/non-persistent database) +Redis (key-value store/non-persistent database): ``` /etc/init.d/redis Usage: /etc/init.d/redis {start|stop|status|restart|condrestart|try-restart} ``` -SSH daemon +SSH daemon: ``` /etc/init.d/sshd Usage: /etc/init.d/sshd {start|stop|restart|reload|force-reload|condrestart|try-restart|status} ``` -Web server (one of the following) +Web server (one of the following): ``` /etc/init.d/httpd @@ -562,7 +562,7 @@ $ /etc/init.d/nginx Usage: nginx {start|stop|restart|reload|force-reload|status|configtest} ``` -Persistent database +Persistent database: ``` $ /etc/init.d/postgresql @@ -571,34 +571,34 @@ Usage: /etc/init.d/postgresql {start|stop|restart|reload|force-reload|status} [v ### Log locations of the services -gitlabhq (includes Unicorn and Sidekiq logs) +gitlabhq (includes Unicorn and Sidekiq logs): - `/home/git/gitlab/log/` contains `application.log`, `production.log`, `sidekiq.log`, `unicorn.stdout.log`, `git_json.log` and `unicorn.stderr.log` normally. -gitlab-shell +GitLab Shell: - `/home/git/gitlab-shell/gitlab-shell.log` -ssh +SSH: - `/var/log/auth.log` auth log (on Ubuntu). - `/var/log/secure` auth log (on RHEL). -nginx +nginx: - `/var/log/nginx/` contains error and access logs. -Apache httpd +Apache httpd: - [Explanation of Apache logs](https://httpd.apache.org/docs/2.2/logs.html). - `/var/log/apache2/` contains error and output logs (on Ubuntu). - `/var/log/httpd/` contains error and output logs (on RHEL). -redis +Redis: - `/var/log/redis/redis.log` there are also log-rotated logs there. -PostgreSQL +PostgreSQL: - `/var/log/postgresql/*` @@ -610,7 +610,7 @@ GitLab has configuration files located in `/home/git/gitlab/config/*`. Commonly - `unicorn.rb` - Unicorn web server settings. - `database.yml` - Database connection settings. -gitlab-shell has a configuration file at `/home/git/gitlab-shell/config.yml`. +GitLab Shell has a configuration file at `/home/git/gitlab-shell/config.yml`. ### Maintenance Tasks diff --git a/doc/development/automatic_ce_ee_merge.md b/doc/development/automatic_ce_ee_merge.md index 158606aa6a2..c2700461467 100644 --- a/doc/development/automatic_ce_ee_merge.md +++ b/doc/development/automatic_ce_ee_merge.md @@ -174,9 +174,9 @@ Now, every time you create an MR for CE and EE: ## How we run the Automatic CE->EE merge at GitLab At GitLab, we use the [Merge Train](https://gitlab.com/gitlab-org/merge-train) -project to keep our [gitlab-ee](https://gitlab.com/gitlab-org/gitlab-ee) +project to keep our [GitLab EE](https://gitlab.com/gitlab-org/gitlab-ee) repository updated with commits from -[gitlab-ce](https://gitlab.com/gitlab-org/gitlab-ce). +[GitLab CE](https://gitlab.com/gitlab-org/gitlab-ce). We have a mirror of the [Merge Train](https://gitlab.com/gitlab-org/merge-train) project [configured](https://ops.gitlab.net/gitlab-org/merge-train) to run an diff --git a/doc/development/background_migrations.md b/doc/development/background_migrations.md index 642dac614c7..3fd95537eaa 100644 --- a/doc/development/background_migrations.md +++ b/doc/development/background_migrations.md @@ -294,7 +294,7 @@ to migrate you database down and up, which can result in other background migrations being called. That means that using `spy` test doubles with `have_received` is encouraged, instead of using regular test doubles, because your expectations defined in a `it` block can conflict with what is being -called in RSpec hooks. See [gitlab-org/gitlab-ce#35351][issue-rspec-hooks] +called in RSpec hooks. See [issue #35351][issue-rspec-hooks] for more details. ## Best practices diff --git a/doc/development/build_test_package.md b/doc/development/build_test_package.md index c5f6adfeaeb..21891f70d73 100644 --- a/doc/development/build_test_package.md +++ b/doc/development/build_test_package.md @@ -3,7 +3,7 @@ While developing a new feature or modifying an existing one, it is helpful if an installable package (or a docker image) containing those changes is available for testing. For this very purpose, a manual job is provided in the GitLab CI/CD -pipeline that can be used to trigger a pipeline in the omnibus-gitlab repository +pipeline that can be used to trigger a pipeline in the Omnibus GitLab repository that will create: - A deb package for Ubuntu 16.04, available as a build artifact, and @@ -12,7 +12,7 @@ that will create: (images titled `gitlab-ce` and `gitlab-ee` respectively and image tag is the commit which triggered the pipeline). -When you push a commit to either the gitlab-ce or gitlab-ee project, the +When you push a commit to either the GitLab CE or GitLab EE project, the pipeline for that commit will have a `build-package` manual action you can trigger. @@ -30,9 +30,9 @@ branch `0-1-stable`, modify the content of `GITALY_SERVER_VERSION` to `0-1-stable` and push the commit. This will create a manual job that can be used to trigger the build. -## Specifying the branch in omnibus-gitlab repository +## Specifying the branch in Omnibus GitLab repository -In scenarios where a configuration change is to be introduced and omnibus-gitlab +In scenarios where a configuration change is to be introduced and Omnibus GitLab repository already has the necessary changes in a specific branch, you can build a package against that branch through an environment variable named `OMNIBUS_BRANCH`. To do this, specify that environment variable with the name of diff --git a/doc/development/changelog.md b/doc/development/changelog.md index bd07a01e782..814624c7586 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -35,6 +35,7 @@ the `author` field. GitLab team members **should not**. - Any user-facing change **should** have a changelog entry. Example: "GitLab now uses system fonts for all text." +- Any docs-only changes **should not** have a changelog entry. - Any change behind a feature flag **should not** have a changelog entry. The entry should be added [in the merge request removing the feature flags](feature_flags/development.md). - A fix for a regression introduced and then fixed in the same release (i.e., fixing a bug introduced during a monthly release candidate) **should not** diff --git a/doc/development/database_debugging.md b/doc/development/database_debugging.md index eb3b227473b..6c9fa983c96 100644 --- a/doc/development/database_debugging.md +++ b/doc/development/database_debugging.md @@ -9,7 +9,7 @@ An easy first step is to search for your error in Slack or google "GitLab (my er Available `RAILS_ENV` -- `production` (generally not for your main GDK db, but you may need this for e.g. omnibus) +- `production` (generally not for your main GDK db, but you may need this for e.g. Omnibus) - `development` (this is your main GDK db) - `test` (used for tests like rspec) diff --git a/doc/development/database_review.md b/doc/development/database_review.md index 3f1b359cb0b..367a481ee11 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -91,7 +91,7 @@ and details for a database reviewer: concurrent index/foreign key helpers (with transactions disabled) - Check consistency with `db/schema.rb` and that migrations are [reversible](migration_style_guide.md#reversibility) - Check queries timing (If any): Queries executed in a migration - need to fit comfortable within `15s` - preferably much less than that - on GitLab.com. + need to fit comfortably within `15s` - preferably much less than that - on GitLab.com. - Check [background migrations](background_migrations.md): - For data migrations, establish a time estimate for execution - They should only be used when migrating data in larger tables. diff --git a/doc/development/ee_features.md b/doc/development/ee_features.md index a732a94b7c4..1358851f3cd 100644 --- a/doc/development/ee_features.md +++ b/doc/development/ee_features.md @@ -910,7 +910,7 @@ import bundle from 'ee_else_ce/protected_branches/protected_branches_bundle.js'; ``` See the frontend guide [performance section](fe_guide/performance.md) for -information on managing page-specific javascript within EE. +information on managing page-specific JavaScript within EE. ## Vue code in `assets/javascript` @@ -1057,7 +1057,7 @@ Here is a workflow to make sure those changes end up backported safely into CE t **Note:** regarding SCSS, make sure the files living outside `/ee/` don't diverge between CE and EE projects. -## gitlab-svgs +## GitLab-svgs Conflicts in `app/assets/images/icons.json` or `app/assets/images/icons.svg` can be resolved simply by regenerating those assets with diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md index 090e5235619..f2412c249c1 100644 --- a/doc/development/elasticsearch.md +++ b/doc/development/elasticsearch.md @@ -40,9 +40,11 @@ There is no need to install any plugins If you're interested on working with the new beta repo indexer, all you need to do is: -- git clone git@gitlab.com:gitlab-org/gitlab-elasticsearch-indexer.git -- make -- make install +```sh +git clone git@gitlab.com:gitlab-org/gitlab-elasticsearch-indexer.git +make +make install +``` this adds `gitlab-elasticsearch-indexer` to `$GOPATH/bin`, please make sure that is in your `$PATH`. After that GitLab will find it and you'll be able to enable it in the admin settings area. @@ -188,7 +190,7 @@ The global configurations per version are now in the `Elastic::(Version)::Config NOTE: **Note:** this is not applicable yet as multiple indices functionality is not fully implemented. -Folders like `ee/lib/elastic/v12p1` contain snapshots of search logic from different versions. To keep a continuous git history, the latest version lives under `ee/lib/elastic/latest`, but its classes are aliased under an actual version (e.g. `ee/lib/elastic/v12p3`). When referencing these classes, never use the `Latest` namespace directly, but use the actual version (e.g. `V12p3`). +Folders like `ee/lib/elastic/v12p1` contain snapshots of search logic from different versions. To keep a continuous Git history, the latest version lives under `ee/lib/elastic/latest`, but its classes are aliased under an actual version (e.g. `ee/lib/elastic/v12p3`). When referencing these classes, never use the `Latest` namespace directly, but use the actual version (e.g. `V12p3`). The version name basically follows GitLab's release version. If setting is changed in 12.3, we will create a new namespace called `V12p3` (p stands for "point"). Raise an issue if there is a need to name a version differently. diff --git a/doc/development/emails.md b/doc/development/emails.md index edec0f86989..5676c3b32f4 100644 --- a/doc/development/emails.md +++ b/doc/development/emails.md @@ -6,7 +6,7 @@ To view rendered emails "sent" in your development instance, visit [`/rails/letter_opener`](http://localhost:3000/rails/letter_opener). Please note that [S/MIME signed](../administration/smime_signing_email.md) emails -[cannot be currently previewed](https://github.com/fgrehm/letter_opener_web/issues/96) with +[cannot be currently previewed](https://github.com/fgrehm/letter_opener_web/issues/96) with `letter_opener`. ## Mailer previews diff --git a/doc/development/feature_flags/index.md b/doc/development/feature_flags/index.md index 56872f8c075..f1374b9e280 100644 --- a/doc/development/feature_flags/index.md +++ b/doc/development/feature_flags/index.md @@ -8,5 +8,5 @@ disable those changes, without having to revert an entire release. Before using feature flags for GitLab's development, read through the following: - [Process for using features flags](process.md). -- [Developing with feature flags documentation](development.md). -- [Controlling feature flags documentation](controls.md). +- [Developing with feature flags](development.md). +- [Controlling feature flags](controls.md). diff --git a/doc/development/filtering_by_label.md b/doc/development/filtering_by_label.md index 5e7376db725..dd8944ff1c8 100644 --- a/doc/development/filtering_by_label.md +++ b/doc/development/filtering_by_label.md @@ -40,16 +40,14 @@ In particular, note that: This is more complicated than is ideal. It makes the query construction more prone to errors (such as -[gitlab-org/gitlab-ce#15557](https://gitlab.com/gitlab-org/gitlab-ce/issues/15557)). +[issue #15557](https://gitlab.com/gitlab-org/gitlab-ce/issues/15557)). ## Attempt A: WHERE EXISTS ### Attempt A1: use multiple subqueries with WHERE EXISTS -In -[gitlab-org/gitlab-ce#37137](https://gitlab.com/gitlab-org/gitlab-ce/issues/37137) -and its associated merge request -[gitlab-org/gitlab-ce!14022](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14022), +In [issue #37137](https://gitlab.com/gitlab-org/gitlab-ce/issues/37137) +and its associated [merge request](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14022), we tried to replace the `GROUP BY` with multiple uses of `WHERE EXISTS`. For the example above, this would give: @@ -81,12 +79,11 @@ it did not improve query performance. ## Attempt B: Denormalize using an array column -Having [removed MySQL support in GitLab -12.1](https://about.gitlab.com/2019/06/27/removing-mysql-support/), using -[Postgres's arrays](https://www.postgresql.org/docs/9.6/arrays.html) became more +Having [removed MySQL support in GitLab 12.1](https://about.gitlab.com/2019/06/27/removing-mysql-support/), +using [Postgres's arrays](https://www.postgresql.org/docs/9.6/arrays.html) became more tractable as we didn't have to support two databases. We discussed denormalizing the `label_links` table for querying in -[gitlab-org/gitlab-ce#49651](https://gitlab.com/gitlab-org/gitlab-ce/issues/49651), +[issue #49651](https://gitlab.com/gitlab-org/gitlab-ce/issues/49651), with two options: label IDs and titles. We can think of both of those as array columns on `issues`, `merge_requests`, @@ -150,8 +147,7 @@ WHERE label_titles @> ARRAY['Plan', 'backend'] ``` -And our [tests in -gitlab-org/gitlab-ce#49651](https://gitlab.com/gitlab-org/gitlab-ce/issues/49651#note_188777346) +And our [tests in issue #49651](https://gitlab.com/gitlab-org/gitlab-ce/issues/49651#note_188777346) showed that this could be fast. However, at present, the disadvantages outweigh the advantages. diff --git a/doc/development/gemfile.md b/doc/development/gemfile.md index ec9718cea71..8d93c52e7bc 100644 --- a/doc/development/gemfile.md +++ b/doc/development/gemfile.md @@ -3,9 +3,9 @@ When adding a new entry to `Gemfile` or upgrading an existing dependency pay attention to the following rules. -## No gems fetched from git repositories +## No gems fetched from Git repositories -We do not allow gems that are fetched from git repositories. All gems have +We do not allow gems that are fetched from Git repositories. All gems have to be available in the RubyGems index. We want to minimize external build dependencies and build times. diff --git a/doc/development/geo.md b/doc/development/geo.md index 24f16eae9fa..cc3e2d1ccc5 100644 --- a/doc/development/geo.md +++ b/doc/development/geo.md @@ -170,7 +170,7 @@ while `pull` requests will continue to be served by the **secondary** node for m HTTPS and SSH requests are handled differently: - With HTTPS, we will give the user a `HTTP 302 Redirect` pointing to the project on the **primary** node. - The git client is wise enough to understand that status code and process the redirection. + The Git client is wise enough to understand that status code and process the redirection. - With SSH, because there is no equivalent way to perform a redirect, we have to proxy the request. This is done inside [`gitlab-shell`](https://gitlab.com/gitlab-org/gitlab-shell), by first translating the request to the HTTP protocol, and then proxying it to the **primary** node. diff --git a/doc/development/git_object_deduplication.md b/doc/development/git_object_deduplication.md index 5ce59891afa..4dd1edf9b5a 100644 --- a/doc/development/git_object_deduplication.md +++ b/doc/development/git_object_deduplication.md @@ -8,30 +8,6 @@ storage disk use. To counteract this problem, we are adding Git object deduplication for forks to GitLab. In this document, we will describe how GitLab implements Git object deduplication. -## Enabling Git object deduplication via feature flags - -As of GitLab 12.0, Git object deduplication in GitLab is still behind a -feature flag. In this document, you can read about the effects of -enabling the feature. Also, note that Git object deduplication is -limited to forks of public projects on hashed repository storage. - -You can enable deduplication globally by setting the `object_pools` -feature flag to `true`: - -``` {.ruby} -Feature.enable(:object_pools) -``` - -Or just for forks of a specific project: - -``` {.ruby} -fork_parent = Project.find(MY_PROJECT_ID) -Feature.enable(:object_pools, fork_parent) -``` - -To check if a project uses Git object deduplication, look in a Rails -console if `project.pool_repository` is present. - ## Pool repositories ### Understanding Git alternates @@ -193,7 +169,7 @@ There are three different things that can go wrong here. In this case, we miss out on disk space savings but all RPC's on A itself will function fine. The next time garbage collection runs on A, the alternates connection gets established in Gitaly. This is done by -`Projects::GitDeduplicationService` in gitlab-rails. +`Projects::GitDeduplicationService` in GitLab Rails. #### 2. SQL says repo A belongs to pool P1 but Gitaly says A has alternate objects in pool P2 diff --git a/doc/development/gitaly.md b/doc/development/gitaly.md index 2ade59b76ed..592fc13873b 100644 --- a/doc/development/gitaly.md +++ b/doc/development/gitaly.md @@ -45,13 +45,13 @@ The process for adding new Gitaly features is: - release a new version of gitaly-proto - write implementation and tests for the RPC [in Gitaly](https://gitlab.com/gitlab-org/gitaly), in Go or Ruby - release a new version of Gitaly -- write client code in gitlab-ce/ee, gitlab-workhorse or gitlab-shell that calls the new Gitaly RPC +- write client code in GitLab CE/EE, GitLab Workhorse or GitLab Shell that calls the new Gitaly RPC These steps often overlap. It is possible to use an unreleased version of Gitaly and gitaly-proto during testing and development. - See the [Gitaly repo](https://gitlab.com/gitlab-org/gitaly/blob/master/CONTRIBUTING.md#development-and-testing-with-a-custom-gitaly-proto) for instructions on writing server side code with an unreleased protocol. -- See [below](#running-tests-with-a-locally-modified-version-of-gitaly) for instructions on running gitlab-ce tests with a modified version of Gitaly. +- See [below](#running-tests-with-a-locally-modified-version-of-gitaly) for instructions on running GitLab CE tests with a modified version of Gitaly. - In GDK run `gdk install` and restart `gdk run` (or `gdk run app`) to use a locally modified Gitaly version for development ### Gitaly-ruby @@ -146,7 +146,7 @@ Once the code is wrapped in this block, this code-path will be excluded from n+1 ## Request counts -Commits and other git data, is now fetched through Gitaly. These fetches can, +Commits and other Git data, is now fetched through Gitaly. These fetches can, much like with a database, be batched. This improves performance for the client and for Gitaly itself and therefore for the users too. To keep performance stable and guard performance regressions, Gitaly calls can be counted and the call count @@ -164,10 +164,10 @@ end ## Running tests with a locally modified version of Gitaly -Normally, gitlab-ce/ee tests use a local clone of Gitaly in +Normally, GitLab CE/EE tests use a local clone of Gitaly in `tmp/tests/gitaly` pinned at the version specified in `GITALY_SERVER_VERSION`. The `GITALY_SERVER_VERSION` file supports -`=my-branch` syntax to use a custom branch in gitlab-org/gitaly. If +`=my-branch` syntax to use a custom branch in <https://gitlab.com/gitlab-org/gitaly>. If you want to run tests locally against a modified version of Gitaly you can replace `tmp/tests/gitaly` with a symlink. This is much faster because the `=my-branch` syntax forces a Gitaly re-install each time @@ -276,9 +276,9 @@ Here are the steps to gate a new feature in Gitaly behind a feature flag. require.NoError(t, err) ``` -### Gitlab-Rails +### GitLab Rails -1. Add feature flag to `lib/gitlab/gitaly_client.rb` (in gitlab-rails): +1. Add feature flag to `lib/gitlab/gitaly_client.rb` (in GitLab Rails): ```ruby SERVER_FEATURE_FLAGS = %w[go-find-all-tags].freeze diff --git a/doc/development/kubernetes.md b/doc/development/kubernetes.md index 4b2d48903ac..f4528667814 100644 --- a/doc/development/kubernetes.md +++ b/doc/development/kubernetes.md @@ -107,7 +107,7 @@ Mitigation strategies include: ## Debugging Logs related to the Kubernetes integration can be found in -[kubernetes.log](../administration/logs.md#kuberneteslog). On a local +[`kubernetes.log`](../administration/logs.md#kuberneteslog). On a local GDK install, this will be present in `log/kubernetes.log`. Some services such as diff --git a/doc/development/logging.md b/doc/development/logging.md index 4f63c84fc0e..b43f1029cc6 100644 --- a/doc/development/logging.md +++ b/doc/development/logging.md @@ -133,7 +133,7 @@ importer progresses. Here's what to do: logs in `/var/log/gitlab/gitlab-rails/*.log` every hour and [keep at most 30 compressed files](https://docs.gitlab.com/omnibus/settings/logs.html#logrotate). On GitLab.com, that setting is only 6 compressed files. These settings should suffice - for most users, but you may need to tweak them in [omnibus-gitlab](https://gitlab.com/gitlab-org/omnibus-gitlab). + for most users, but you may need to tweak them in [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab). 1. If you add a new file, submit an issue to the [production tracker](https://gitlab.com/gitlab-com/gl-infra/production/issues) or diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index 3181b3a88cc..077dd2f43d0 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -1,12 +1,12 @@ # Migration Style Guide When writing migrations for GitLab, you have to take into account that -these will be ran by hundreds of thousands of organizations of all sizes, some with +these will be run by hundreds of thousands of organizations of all sizes, some with many years of data in their database. In addition, having to take a server offline for an upgrade small or big is a -big burden for most organizations. For this reason it is important that your -migrations are written carefully, can be applied online and adhere to the style +big burden for most organizations. For this reason, it is important that your +migrations are written carefully, can be applied online, and adhere to the style guide below. Migrations are **not** allowed to require GitLab installations to be taken @@ -85,7 +85,38 @@ be possible to downgrade in case of a vulnerability or bugs. In your migration, add a comment describing how the reversibility of the migration was tested. -## Multi Threading +## Atomicity + +By default, migrations are single transaction. That is, a transaction is opened +at the beginning of the migration, and committed after all steps are processed. + +Running migrations in a single transaction makes sure that if one of the steps fails, +none of the steps will be executed, leaving the database in valid state. +Therefore, either: + +- Put all migrations in one single-transaction migration. +- If necessary, put most actions in one migration and create a separate migration + for the steps that cannot be done in a single transaction. + +For example, if you create an empty table and need to build an index for it, +it is recommended to use a regular single-transaction migration and the default +rails schema statement: [`add_index`](https://api.rubyonrails.org/v5.2/classes/ActiveRecord/ConnectionAdapters/SchemaStatements.html#method-i-add_index). +This is a blocking operation, but it won't cause problems because the table is not yet used, +and therefore it does not have any records yet. + +## Heavy operations in a single transaction + +When using a single-transaction migration, a transaction will hold on a database connection +for the duration of the migration, so you must make sure the actions in the migration +do not take too much time: In general, queries executed in a migration need to fit comfortably +within `15s` on GitLab.com. + +In case you need to insert, update, or delete a significant amount of data, you: + +- Must disable the single transaction with `disable_ddl_transaction!`. +- Should consider doing it in a [Background Migration](background_migrations.md). + +## Multi-Threading Sometimes a migration might need to use multiple Ruby threads to speed up a migration. For this to work your migration needs to include the module @@ -122,16 +153,16 @@ pool. This ensures each thread has its own connection object, and won't time out when trying to obtain one. **NOTE:** PostgreSQL has a maximum amount of connections that it allows. This -limit can vary from installation to installation. As a result it's recommended -you do not use more than 32 threads in a single migration. Usually 4-8 threads +limit can vary from installation to installation. As a result, it's recommended +you do not use more than 32 threads in a single migration. Usually, 4-8 threads should be more than enough. ## Removing indexes -When removing an index make sure to use the method `remove_concurrent_index` instead -of the regular `remove_index` method. The `remove_concurrent_index` method -automatically drops concurrent indexes when using PostgreSQL, removing the -need for downtime. To use this method you must disable single-transaction mode +If the table is not empty when removing an index, make sure to use the method +`remove_concurrent_index` instead of the regular `remove_index` method. +The `remove_concurrent_index` method drops indexes concurrently, so no locking is required, +and there is no need for downtime. To use this method, you must disable single-transaction mode by calling the method `disable_ddl_transaction!` in the body of your migration class like so: @@ -149,19 +180,25 @@ end Note that it is not necessary to check if the index exists prior to removing it. +For a small table (such as an empty one or one with less than `1,000` records), +it is recommended to use `remove_index` in a single-transaction migration, +combining it with other operations that don't require `disable_ddl_transaction!`. + ## Adding indexes -If you need to add a unique index please keep in mind there is the possibility +If you need to add a unique index, please keep in mind there is the possibility of existing duplicates being present in the database. This means that should always _first_ add a migration that removes any duplicates, before adding the unique index. -When adding an index make sure to use the method `add_concurrent_index` instead -of the regular `add_index` method. The `add_concurrent_index` method -automatically creates concurrent indexes when using PostgreSQL, removing the -need for downtime. To use this method you must disable transactions by calling -the method `disable_ddl_transaction!` in the body of your migration class like -so: +When adding an index to a non-empty table make sure to use the method +`add_concurrent_index` instead of the regular `add_index` method. +The `add_concurrent_index` method automatically creates concurrent indexes +when using PostgreSQL, removing the need for downtime. + +To use this method, you must disable single-transactions mode +by calling the method `disable_ddl_transaction!` in the body of your migration +class like so: ```ruby class MyMigration < ActiveRecord::Migration[4.2] @@ -179,16 +216,20 @@ class MyMigration < ActiveRecord::Migration[4.2] end ``` +For a small table (such as an empty one or one with less than `1,000` records), +it is recommended to use `add_index` in a single-transaction migration, combining it with other +operations that don't require `disable_ddl_transaction!`. + ## Adding foreign-key constraints -When adding a foreign-key constraint to either an existing or new -column remember to also add a index on the column. +When adding a foreign-key constraint to either an existing or a new column also +remember to add an index on the column. This is **required** for all foreign-keys, e.g., to support efficient cascading deleting: when a lot of rows in a table get deleted, the referenced records need to be deleted too. The database has to look for corresponding records in the referenced table. Without an index, this will result in a sequential scan on the -table which can take a long time. +table, which can take a long time. Here's an example where we add a new column with a foreign key constraint. Note it includes `index: true` to create an index for it. @@ -202,13 +243,17 @@ class Migration < ActiveRecord::Migration[4.2] end ``` -When adding a foreign-key constraint to an existing column, we -have to employ `add_concurrent_foreign_key` and `add_concurrent_index` +When adding a foreign-key constraint to an existing column in a non-empty table, +we have to employ `add_concurrent_foreign_key` and `add_concurrent_index` instead of `add_reference`. +For an empty table (such as a fresh one), it is recommended to use +`add_reference` in a single-transaction migration, combining it with other +operations that don't require `disable_ddl_transaction!`. + ## Adding Columns With Default Values -When adding columns with default values you must use the method +When adding columns with default values to non-empty tables, you must use `add_column_with_default`. This method ensures the table is updated without requiring downtime. This method is not reversible so you must manually define the `up` and `down` methods in your migration class. @@ -232,10 +277,14 @@ end ``` Keep in mind that this operation can easily take 10-15 minutes to complete on -larger installations (e.g. GitLab.com). As a result you should only add default -values if absolutely necessary. There is a RuboCop cop that will fail if this -method is used on some tables that are very large on GitLab.com, which would -cause other issues. +larger installations (e.g. GitLab.com). As a result, you should only add +default values if absolutely necessary. There is a RuboCop cop that will fail if +this method is used on some tables that are very large on GitLab.com, which +would cause other issues. + +For a small table (such as an empty one or one with less than `1,000` records), +use `add_column` and `change_column_default` in a single-transaction migration, +combining it with other operations that don't require `disable_ddl_transaction!`. ## Updating an existing column @@ -253,8 +302,10 @@ update_column_in_batches(:projects, :foo, 10) do |table, query| end ``` -To perform a computed update, the value can be wrapped in `Arel.sql`, so Arel -treats it as an SQL literal. The below example is the same as the one above, but +If a computed update is needed, the value can be wrapped in `Arel.sql`, so Arel +treats it as an SQL literal. It's also a required deprecation for [Rails 6](https://gitlab.com/gitlab-org/gitlab-ce/issues/61451). + +The below example is the same as the one above, but the value is set to the product of the `bar` and `baz` columns: ```ruby @@ -275,12 +326,12 @@ staging environment - or asking someone else to do so for you - beforehand. By default, an integer column can hold up to a 4-byte (32-bit) number. That is a max value of 2,147,483,647. Be aware of this when creating a column that will -hold file sizes in byte units. If you are tracking file size in bytes this +hold file sizes in byte units. If you are tracking file size in bytes, this restricts the maximum file size to just over 2GB. To allow an integer column to hold up to an 8-byte (64-bit) number, explicitly set the limit to 8-bytes. This will allow the column to hold a value up to -9,223,372,036,854,775,807. +`9,223,372,036,854,775,807`. Rails migration example: @@ -294,9 +345,11 @@ add_column(:projects, :foo, :integer, default: 10, limit: 8) ## Timestamp column type -By default, Rails uses the `timestamp` data type that stores timestamp data without timezone information. -The `timestamp` data type is used by calling either the `add_timestamps` or the `timestamps` method. -Also Rails converts the `:datetime` data type to the `timestamp` one. +By default, Rails uses the `timestamp` data type that stores timestamp data +without timezone information. The `timestamp` data type is used by calling +either the `add_timestamps` or the `timestamps` method. + +Also, Rails converts the `:datetime` data type to the `timestamp` one. Example: @@ -317,14 +370,16 @@ def up end ``` -Instead of using these methods one should use the following methods to store timestamps with timezones: +Instead of using these methods, one should use the following methods to store +timestamps with timezones: - `add_timestamps_with_timezone` - `timestamps_with_timezone` -This ensures all timestamps have a time zone specified. This in turn means existing timestamps won't -suddenly use a different timezone when the system's timezone changes. It also makes it very clear which -timezone was used in the first place. +This ensures all timestamps have a time zone specified. This, in turn, means +existing timestamps won't suddenly use a different timezone when the system's +timezone changes. It also makes it very clear which timezone was used in the +first place. ## Storing JSON in database @@ -359,7 +414,7 @@ Make sure your migration can be reversed. ## Data migration Please prefer Arel and plain SQL over usual ActiveRecord syntax. In case of -using plain SQL you need to quote all input manually with `quote_string` helper. +using plain SQL, you need to quote all input manually with `quote_string` helper. Example with Arel: @@ -384,7 +439,7 @@ select_all("SELECT name, COUNT(id) as cnt FROM tags GROUP BY name HAVING COUNT(i end ``` -If you need more complex logic you can define and use models local to a +If you need more complex logic, you can define and use models local to a migration. For example: ```ruby @@ -395,13 +450,13 @@ class MyMigration < ActiveRecord::Migration[4.2] end ``` -When doing so be sure to explicitly set the model's table name so it's not +When doing so be sure to explicitly set the model's table name, so it's not derived from the class name or namespace. ### Renaming reserved paths -When a new route for projects is introduced that could conflict with any -existing records. The path for this records should be renamed, and the +When a new route for projects is introduced, it could conflict with any +existing records. The path for these records should be renamed, and the related data should be moved on disk. Since we had to do this a few times already, there are now some helpers to help diff --git a/doc/development/namespaces_storage_statistics.md b/doc/development/namespaces_storage_statistics.md index b3285de4705..2c7e5935435 100644 --- a/doc/development/namespaces_storage_statistics.md +++ b/doc/development/namespaces_storage_statistics.md @@ -20,8 +20,8 @@ every time the project is saved. The summary of those statistics per namespace is then retrieved by [`Namespaces#with_statistics`](https://gitlab.com/gitlab-org/gitlab-ce/blob/v12.2.0.pre/app/models/namespace.rb#L70) scope. Analyzing this query we noticed that: -* It takes up to `1.2` seconds for namespaces with over `15k` projects. -* It can't be analyzed with [ChatOps](chatops_on_gitlabcom.md), as it times out. +- It takes up to `1.2` seconds for namespaces with over `15k` projects. +- It can't be analyzed with [ChatOps](chatops_on_gitlabcom.md), as it times out. Additionally, the pattern that is currently used to update the project statistics (the callback) doesn't scale adequately. It is currently one of the largest @@ -62,8 +62,8 @@ REFRESH MATERIALIZED VIEW root_namespace_storage_statistics; While this implied a single query update (and probably a fast one), it has some downsides: -* Materialized views syntax varies from PostgreSQL and MySQL. While this feature was worked on, MySQL was still supported by GitLab. -* Rails does not have native support for materialized views. We'd need to use a specialized gem to take care of the management of the database views, which implies additional work. +- Materialized views syntax varies from PostgreSQL and MySQL. While this feature was worked on, MySQL was still supported by GitLab. +- Rails does not have native support for materialized views. We'd need to use a specialized gem to take care of the management of the database views, which implies additional work. ### Attempt B: An update through a CTE @@ -131,8 +131,8 @@ WHERE namespace_id IN ( Even though this approach would make aggregating much easier, it has some major downsides: -* We'd have to migrate **all namespaces** by adding and filling a new column. Because of the size of the table, dealing with time/cost will not be great. The background migration will take approximately `153h`, see <https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/29772>. -* Background migration has to be shipped one release before, delaying the functionality by another milestone. +- We'd have to migrate **all namespaces** by adding and filling a new column. Because of the size of the table, dealing with time/cost will not be great. The background migration will take approximately `153h`, see <https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/29772>. +- Background migration has to be shipped one release before, delaying the functionality by another milestone. ### Attempt E (final): Update the namespace storage statistics in async way @@ -155,10 +155,10 @@ but we refresh them through Sidekiq jobs and in different transactions: This implementation has the following benefits: -* All the updates are done async, so we're not increasing the length of the transactions for `project_statistics`. -* We're doing the update in a single SQL query. -* It is compatible with PostgreSQL and MySQL. -* No background migration required. +- All the updates are done async, so we're not increasing the length of the transactions for `project_statistics`. +- We're doing the update in a single SQL query. +- It is compatible with PostgreSQL and MySQL. +- No background migration required. The only downside of this approach is that namespaces' statistics are updated up to `1.5` hours after the change is done, which means there's a time window in which the statistics are inaccurate. Because we're still not @@ -171,8 +171,8 @@ performant approach of aggregating the root namespaces. All the details regarding this use case can be found on: -* <https://gitlab.com/gitlab-org/gitlab-ce/issues/62214> -* Merge Request with the implementation: <https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/28996> +- <https://gitlab.com/gitlab-org/gitlab-ce/issues/62214> +- Merge Request with the implementation: <https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/28996> Performance of the namespace storage statistics were measured in staging and production (GitLab.com). All results were posted on <https://gitlab.com/gitlab-org/gitlab-ce/issues/64092>: No problem has been reported so far. diff --git a/doc/development/omnibus.md b/doc/development/omnibus.md index 0ba354d28a2..ea5c18f1a8c 100644 --- a/doc/development/omnibus.md +++ b/doc/development/omnibus.md @@ -1,32 +1,32 @@ -# What you should know about omnibus packages +# What you should know about Omnibus packages -Most users install GitLab using our omnibus packages. As a developer it can be -good to know how the omnibus packages differ from what you have on your laptop +Most users install GitLab using our Omnibus packages. As a developer it can be +good to know how the Omnibus packages differ from what you have on your laptop when you are coding. ## Files are owned by root by default -All the files in the Rails tree (`app/`, `config/` etc.) are owned by 'root' in -omnibus installations. This makes the installation simpler and it provides -extra security. The omnibus reconfigure script contains commands that give -write access to the 'git' user only where needed. +All the files in the Rails tree (`app/`, `config/` etc.) are owned by `root` in +Omnibus installations. This makes the installation simpler and it provides +extra security. The Omnibus reconfigure script contains commands that give +write access to the `git` user only where needed. -For example, the 'git' user is allowed to write in the `log/` directory, in +For example, the `git` user is allowed to write in the `log/` directory, in `public/uploads`, and they are allowed to rewrite the `db/schema.rb` file. In other cases, the reconfigure script tricks GitLab into not trying to write a file. For instance, GitLab will generate a `.secret` file if it cannot find one -and write it to the Rails root. In the omnibus packages, reconfigure writes the +and write it to the Rails root. In the Omnibus packages, reconfigure writes the `.secret` file first, so that GitLab never tries to write it. ## Code, data and logs are in separate directories -The omnibus design separates code (read-only, under `/opt/gitlab`) from data +The Omnibus design separates code (read-only, under `/opt/gitlab`) from data (read/write, under `/var/opt/gitlab`) and logs (read/write, under `/var/log/gitlab`). To make this happen the reconfigure script sets custom paths where it can in GitLab config files, and where there are no path settings, it uses symlinks. For example, `config/gitlab.yml` is treated as data so that file is a symlink. -The same goes for `public/uploads`. The `log/` directory is replaced by omnibus +The same goes for `public/uploads`. The `log/` directory is replaced by Omnibus with a symlink to `/var/log/gitlab/gitlab-rails`. diff --git a/doc/development/session.md b/doc/development/session.md index 9edce3dbda0..971795d8816 100644 --- a/doc/development/session.md +++ b/doc/development/session.md @@ -17,7 +17,7 @@ When storing values in a session it is best to: - Use simple primitives and avoid storing objects to avoid marshaling complications. - Clean up after unneeded variables to keep memory usage in Redis down. -## Gitlab::Session +## GitLab::Session Sometimes you might want to persist data in the session instead of another store like the database. `Gitlab::Session` lets you access this without passing the session around extensively. For example, you could access it from within a policy without having to pass the session through to each place permissions are checked from. diff --git a/doc/development/shell_commands.md b/doc/development/shell_commands.md index 7bdf676be58..1300c99622e 100644 --- a/doc/development/shell_commands.md +++ b/doc/development/shell_commands.md @@ -35,7 +35,7 @@ Gitlab::Popen.popen(%W(find /some/path -not -path /some/path -mmin +120 -delete) This coding style could have prevented CVE-2013-4490. -## Always use the configurable git binary path for git commands +## Always use the configurable Git binary path for Git commands ```ruby # Wrong @@ -114,7 +114,7 @@ user = `whoami` user, exit_status = Gitlab::Popen.popen(%W(whoami)) ``` -In other repositories, such as gitlab-shell you can also use `IO.popen`. +In other repositories, such as GitLab Shell you can also use `IO.popen`. ```ruby # Safe IO.popen example diff --git a/doc/development/uploads.md b/doc/development/uploads.md index 234539bb673..681ce9d9fe8 100644 --- a/doc/development/uploads.md +++ b/doc/development/uploads.md @@ -174,14 +174,14 @@ sequenceDiagram c ->>+w: POST /some/url/upload w->>+s: save the incoming file on a temporary location - s-->>-w: + s-->>-w: w->>+r: POST /some/url/upload Note over w,r: file was replaced with its location<br>and other metadata opt requires async processing r->>+redis: schedule a job - redis-->>-r: + redis-->>-r: end r-->>-c: request result @@ -230,17 +230,17 @@ sequenceDiagram w->>+os: PUT file Note over w,os: file is stored on a temporary location. Rails select the destination - os-->>-w: + os-->>-w: w->>+r: POST /some/url/upload Note over w,r: file was replaced with its location<br>and other metadata r->>+os: move object to final destination - os-->>-r: + os-->>-r: opt requires async processing r->>+redis: schedule a job - redis-->>-r: + redis-->>-r: end r-->>-c: request result @@ -268,4 +268,3 @@ sequenceDiagram This option affect the response to the `/authorize` call. When not enabled, the API response will not contain presigned URLs and workhorse will write the file the shared disk, on the path is provided by rails, acting like object storage was disabled. Once the request reachs rails, it will schedule an object storage upload as a sidekiq job. - diff --git a/doc/gitlab-basics/add-file.md b/doc/gitlab-basics/add-file.md index 41cc8bc4aeb..d547584bf80 100644 --- a/doc/gitlab-basics/add-file.md +++ b/doc/gitlab-basics/add-file.md @@ -70,7 +70,7 @@ git commit -m "DESCRIBE COMMIT IN A FEW WORDS" ``` Now you can push (send) your changes (in the branch `<branch-name>`) to GitLab -(the git remote named 'origin'): +(the Git remote named 'origin'): ```sh git push origin <branch-name> diff --git a/doc/gitlab-basics/command-line-commands.md b/doc/gitlab-basics/command-line-commands.md index ed70d3ce598..74539b33642 100644 --- a/doc/gitlab-basics/command-line-commands.md +++ b/doc/gitlab-basics/command-line-commands.md @@ -10,7 +10,7 @@ learn, in order to make full use of the command line. ## Start working on your project -To work on a git project locally (from your own computer), with the command line, +To work on a Git project locally (from your own computer), with the command line, first you will need to [clone (copy) it](start-using-git.md#clone-a-repository) to your computer. diff --git a/doc/gitlab-basics/start-using-git.md b/doc/gitlab-basics/start-using-git.md index 5de173abbff..a289b90b81b 100644 --- a/doc/gitlab-basics/start-using-git.md +++ b/doc/gitlab-basics/start-using-git.md @@ -102,7 +102,7 @@ files to your local computer, automatically preserving the Git connection with t remote repository. You can either clone it via HTTPS or [SSH](../ssh/README.md). If you chose to clone -it via HTTPS, you'll have to enter your credentials every time you pull and push. You can read more about credential storage in the [Git Credentials documentation](https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage). With SSH, you enter your credentials only once. +it via HTTPS, you'll have to enter your credentials every time you pull and push. You can read more about credential storage in the [Git Credentials documentation](https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage). With SSH, you enter your credentials only once. You can find both paths (HTTPS and SSH) by navigating to your project's landing page and clicking **Clone**. GitLab will prompt you with both paths, from which you can copy @@ -157,7 +157,7 @@ git pull <REMOTE> <name-of-branch> When you clone a repository, `REMOTE` is typically `origin`. This is where the repository was cloned from, and it indicates the SSH or HTTPS URL of the repository on the remote server. `<name-of-branch>` is usually `master`, but it may be any existing -branch. You can create additional named remotes and branches as necessary. +branch. You can create additional named remotes and branches as necessary. You can learn more on how Git manages remote repositories in the [Git Remote documentation](https://git-scm.com/book/en/v2/Git-Basics-Working-with-Remotes). @@ -169,7 +169,7 @@ To view your remote repositories, type: git remote -v ``` -The `-v` flag stands for verbose. +The `-v` flag stands for verbose. ### Add a remote repository diff --git a/doc/install/README.md b/doc/install/README.md index af98791c8e9..fd91527ed4c 100644 --- a/doc/install/README.md +++ b/doc/install/README.md @@ -13,7 +13,7 @@ and cost of hosting. There are many ways you can install GitLab depending on your platform: -1. **Omnibus Gitlab**: The official deb/rpm packages that contain a bundle of GitLab +1. **Omnibus GitLab**: The official deb/rpm packages that contain a bundle of GitLab and the various components it depends on like PostgreSQL, Redis, Sidekiq, etc. 1. **GitLab Helm chart**: The cloud native Helm chart for installing GitLab and all its components on Kubernetes. diff --git a/doc/install/installation.md b/doc/install/installation.md index 6d64b8a4936..2989a4edaf7 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -57,18 +57,18 @@ of this page: ``` - `/home/git/.ssh` - Contains OpenSSH settings. Specifically the `authorized_keys` - file managed by gitlab-shell. + file managed by GitLab Shell. - `/home/git/gitlab` - GitLab core software. - `/home/git/gitlab-shell` - Core add-on component of GitLab. Maintains SSH cloning and other functionality. - `/home/git/repositories` - Bare repositories for all projects organized by - namespace. This is where the git repositories which are pushed/pulled are + namespace. This is where the Git repositories which are pushed/pulled are maintained for all projects. **This area contains critical data for projects. [Keep a backup](../raketasks/backup_restore.md).** NOTE: **Note:** The default locations for repositories can be configured in `config/gitlab.yml` -of GitLab and `config.yml` of gitlab-shell. +of GitLab and `config.yml` of GitLab Shell. For a more in-depth overview, see the [GitLab architecture doc](../development/architecture.md). @@ -569,7 +569,7 @@ GitLab Shell application startup time can be greatly reduced by disabling RubyGe - Compile Ruby with `configure --disable-rubygems` to disable RubyGems by default. Not recommended for system-wide Ruby. - Omnibus GitLab [replaces the *shebang* line of the `gitlab-shell/bin/*` scripts](https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests/1707). -### Install gitlab-workhorse +### Install GitLab Workhorse GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). The following command-line will install GitLab-Workhorse in `/home/git/gitlab-workhorse` @@ -833,7 +833,7 @@ To use GitLab with HTTPS: 1. In `gitlab.yml`: 1. Set the `port` option in section 1 to `443`. 1. Set the `https` option in section 1 to `true`. -1. In the `config.yml` of gitlab-shell: +1. In the `config.yml` of GitLab Shell: 1. Set `gitlab_url` option to the HTTPS endpoint of GitLab (e.g. `https://git.example.com`). 1. Set the certificates using either the `ca_file` or `ca_path` option. 1. Use the `gitlab-ssl` Nginx example config instead of the `gitlab` config. @@ -852,7 +852,7 @@ Using a self-signed certificate is discouraged but if you must use it, follow th sudo chmod o-r gitlab.key ``` -1. In the `config.yml` of gitlab-shell set `self_signed_cert` to `true`. +1. In the `config.yml` of GitLab Shell set `self_signed_cert` to `true`. ### Enable Reply by email @@ -950,8 +950,8 @@ To use GitLab with Puma: If you see this message when attempting to clone a repository hosted by GitLab, this is likely due to an outdated Nginx or Apache configuration, or a missing or -misconfigured gitlab-workhorse instance. Double-check that you've -[installed Go](#3-go), [installed gitlab-workhorse](#install-gitlab-workhorse), +misconfigured GitLab Workhorse instance. Double-check that you've +[installed Go](#3-go), [installed GitLab Workhorse](#install-gitlab-workhorse), and correctly [configured Nginx](#site-configuration). ### google-protobuf "LoadError: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.14' not found" diff --git a/doc/install/relative_url.md b/doc/install/relative_url.md index bc6364f57f7..595304e27e2 100644 --- a/doc/install/relative_url.md +++ b/doc/install/relative_url.md @@ -110,7 +110,7 @@ Make sure to follow all steps below: **Note:** If you are using a custom init script, make sure to edit the above - gitlab-workhorse setting as needed. + GitLab Workhorse setting as needed. 1. [Restart GitLab][] for the changes to take effect. diff --git a/doc/integration/auth0.md b/doc/integration/auth0.md index 5061b863e79..2dd6fa3d5a2 100644 --- a/doc/integration/auth0.md +++ b/doc/integration/auth0.md @@ -30,7 +30,7 @@ application. 1. On your GitLab server, open the configuration file. - For omnibus package: + For Omnibus package: ```sh sudo editor /etc/gitlab/gitlab.rb @@ -48,7 +48,7 @@ application. 1. Add the provider configuration: - For omnibus package: + For Omnibus package: ```ruby gitlab_rails['omniauth_providers'] = [ diff --git a/doc/integration/azure.md b/doc/integration/azure.md index a9468f201ef..c30d79e3dab 100644 --- a/doc/integration/azure.md +++ b/doc/integration/azure.md @@ -32,7 +32,7 @@ To enable the Microsoft Azure OAuth2 OmniAuth provider you must register your ap 1. On your GitLab server, open the configuration file. - For omnibus package: + For Omnibus package: ```sh sudo editor /etc/gitlab/gitlab.rb @@ -50,7 +50,7 @@ To enable the Microsoft Azure OAuth2 OmniAuth provider you must register your ap 1. Add the provider configuration: - For omnibus package: + For Omnibus package: ```ruby gitlab_rails['omniauth_providers'] = [ diff --git a/doc/integration/cas.md b/doc/integration/cas.md index f99337376a8..83b64773c9f 100644 --- a/doc/integration/cas.md +++ b/doc/integration/cas.md @@ -4,7 +4,7 @@ To enable the CAS OmniAuth provider you must register your application with your 1. On your GitLab server, open the configuration file. - For omnibus package: + For Omnibus package: ```sh sudo editor /etc/gitlab/gitlab.rb @@ -22,7 +22,7 @@ To enable the CAS OmniAuth provider you must register your application with your 1. Add the provider configuration: - For omnibus package: + For Omnibus package: ```ruby gitlab_rails['omniauth_providers'] = [ diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index eee05eaef02..de49508b47a 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -123,8 +123,8 @@ production instances, they recommend considerably more resources. Storage requirements also vary based on the installation side, but as a rule of thumb, you should allocate the total size of your production database, **plus** -two-thirds of the total size of your git repositories. Efforts to reduce this -total are being tracked in this epic: [gitlab-org&153](https://gitlab.com/groups/gitlab-org/-/epics/153). +two-thirds of the total size of your Git repositories. Efforts to reduce this +total are being tracked in [epic &153](https://gitlab.com/groups/gitlab-org/-/epics/153). ## Enabling Elasticsearch @@ -341,27 +341,27 @@ Currently for repository and snippet files, GitLab would only index up to 1 MB o There are several rake tasks available to you via the command line: -- [sudo gitlab-rake gitlab:elastic:index](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) +- [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) - This is a wrapper task. It does the following: - `sudo gitlab-rake gitlab:elastic:create_empty_index` - `sudo gitlab-rake gitlab:elastic:clear_index_status` - `sudo gitlab-rake gitlab:elastic:index_projects` - `sudo gitlab-rake gitlab:elastic:index_snippets` -- [sudo gitlab-rake gitlab:elastic:index_projects](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) +- [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) - This iterates over all projects and queues sidekiq jobs to index them in the background. -- [sudo gitlab-rake gitlab:elastic:index_projects_status](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) +- [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) - This determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. -- [sudo gitlab-rake gitlab:elastic:create_empty_index](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) +- [`sudo gitlab-rake gitlab:elastic:create_empty_index`](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) - This generates an empty index on the Elasticsearch side. -- [sudo gitlab-rake gitlab:elastic:clear_index_status](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) +- [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) - This deletes all instances of IndexStatus for all projects. -- [sudo gitlab-rake gitlab:elastic:delete_index](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) +- [`sudo gitlab-rake gitlab:elastic:delete_index`](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) - This removes the GitLab index on the Elasticsearch instance. -- [sudo gitlab-rake gitlab:elastic:recreate_index](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) +- [`sudo gitlab-rake gitlab:elastic:recreate_index`](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) - Does the same thing as `sudo gitlab-rake gitlab:elastic:create_empty_index` -- [sudo gitlab-rake gitlab:elastic:index_snippets](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) +- [`sudo gitlab-rake gitlab:elastic:index_snippets`](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) - Performs an Elasticsearch import that indexes the snippets data. -- [sudo gitlab-rake gitlab:elastic:projects_not_indexed](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) +- [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) - Displays which projects are not indexed. ### Environment Variables diff --git a/doc/integration/facebook.md b/doc/integration/facebook.md index 49b3d194a01..d46486ad888 100644 --- a/doc/integration/facebook.md +++ b/doc/integration/facebook.md @@ -49,7 +49,7 @@ To enable the Facebook OmniAuth provider you must register your application with 1. On your GitLab server, open the configuration file. - For omnibus package: + For Omnibus package: ```sh sudo editor /etc/gitlab/gitlab.rb @@ -67,7 +67,7 @@ To enable the Facebook OmniAuth provider you must register your application with 1. Add the provider configuration: - For omnibus package: + For Omnibus package: ```ruby gitlab_rails['omniauth_providers'] = [ diff --git a/doc/integration/github.md b/doc/integration/github.md index c8dbae65465..f19b3109d15 100644 --- a/doc/integration/github.md +++ b/doc/integration/github.md @@ -36,7 +36,7 @@ To get the credentials (a pair of Client ID and Client Secret), you must registe 1. On your GitLab server, open the configuration file. - For omnibus package: + For Omnibus package: ```sh sudo editor /etc/gitlab/gitlab.rb @@ -54,7 +54,7 @@ To get the credentials (a pair of Client ID and Client Secret), you must registe 1. Add the provider configuration: - For omnibus package: + For Omnibus package: For GitHub.com: @@ -124,7 +124,7 @@ certificate and the imports are failing, you will need to disable SSL verificati It should be disabled by adding `verify_ssl` to `false` in the provider configuration and changing the global Git `sslVerify` option to `false` in the GitLab server. -For omnibus package: +For Omnibus package: ```ruby gitlab_rails['omniauth_providers'] = [ diff --git a/doc/integration/gitlab.md b/doc/integration/gitlab.md index 46da3d88d90..74f2d5cb403 100644 --- a/doc/integration/gitlab.md +++ b/doc/integration/gitlab.md @@ -30,7 +30,7 @@ GitLab.com will generate an application ID and secret key for you to use. 1. On your GitLab server, open the configuration file. - For omnibus package: + For Omnibus package: ```sh sudo editor /etc/gitlab/gitlab.rb @@ -48,7 +48,7 @@ GitLab.com will generate an application ID and secret key for you to use. 1. Add the provider configuration: - For omnibus package: + For Omnibus package: ```ruby gitlab_rails['omniauth_providers'] = [ diff --git a/doc/integration/jenkins_deprecated.md b/doc/integration/jenkins_deprecated.md index bac89ae2dc6..3e437eb688a 100644 --- a/doc/integration/jenkins_deprecated.md +++ b/doc/integration/jenkins_deprecated.md @@ -14,12 +14,12 @@ Integration includes: Requirements: - [Jenkins GitLab Hook plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Hook+Plugin) -- git clone access for Jenkins from GitLab repo (via ssh key) +- Git clone access for Jenkins from GitLab repo (via ssh key) ## Jenkins 1. Install [GitLab Hook plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Hook+Plugin) -1. Set up jenkins project +1. Set up Jenkins project  diff --git a/doc/integration/jira.md b/doc/integration/jira.md index c09fde08326..37eba25fb5a 100644 --- a/doc/integration/jira.md +++ b/doc/integration/jira.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/jira.md' --- -This document was moved to [integrations/jira](../user/project/integrations/jira.md). +This document was moved to [another location](../user/project/integrations/jira.md). diff --git a/doc/integration/jira_development_panel.md b/doc/integration/jira_development_panel.md index 3e894371df9..c413e07ec93 100644 --- a/doc/integration/jira_development_panel.md +++ b/doc/integration/jira_development_panel.md @@ -23,7 +23,7 @@ or instance admin (in the case of self-hosted GitLab) set up the integration, in order to simplify administration. TIP: **Tip:** -Create and use a single-purpose "jira" user in GitLab, so that removing +Create and use a single-purpose `jira` user in GitLab, so that removing regular users won't impact your integration. ## Requirements diff --git a/doc/integration/kerberos.md b/doc/integration/kerberos.md index b791cbd428d..81a1e9b0067 100644 --- a/doc/integration/kerberos.md +++ b/doc/integration/kerberos.md @@ -46,7 +46,7 @@ sudo chmod 0600 /etc/http.keytab For source installations, make sure the `kerberos` gem group [has been installed](../install/installation.md#install-gems). -1. Edit the kerberos section of [gitlab.yml] to enable Kerberos ticket-based +1. Edit the `kerberos` section of [`gitlab.yml`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example) to enable Kerberos ticket-based authentication. In most cases, you only need to enable Kerberos and specify the location of the keytab: @@ -153,7 +153,7 @@ keep offering only `basic` authentication. listen [::]:8443 ipv6only=on ssl; ``` -1. Update the Kerberos section of [gitlab.yml]: +1. Update the `kerberos` section of [`gitlab.yml`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example): ```yaml kerberos: @@ -203,7 +203,7 @@ remove the OmniAuth provider named `kerberos` from your `gitlab.yml` / **For installations from source** -1. Edit [gitlab.yml] and remove the `- { name: 'kerberos' }` line under omniauth +1. Edit [`gitlab.yml`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example) and remove the `- { name: 'kerberos' }` line under omniauth providers: ```yaml @@ -295,7 +295,6 @@ See also: [Git v2.11 release notes](https://github.com/git/git/blob/master/Docum - <http://blog.manula.org/2012/04/setting-up-kerberos-server-with-debian.html> - <http://www.roguelynn.com/words/explain-like-im-5-kerberos/> -[gitlab.yml]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example [restart gitlab]: ../administration/restart_gitlab.md#installations-from-source [reconfigure gitlab]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure [nginx]: http://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index 7a92ed994c7..ef319f7f0ce 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -69,7 +69,7 @@ that are in common for all providers that we need to consider. To change these settings: -- **For omnibus package** +- **For Omnibus package** Open the configuration file: @@ -277,3 +277,24 @@ omniauth: sync_profile_from_provider: ['twitter', 'google_oauth2'] sync_profile_attributes: ['email', 'location'] ``` + +## Bypassing two factor authentication + +Starting with GitLab 12.3, this allows users to login with the specified +providers without two factor authentication. + +Define the allowed providers using an array, e.g. `["twitter", 'google_oauth2']`, or as +`true`/`false` to allow all providers or none. This option should only be configured +for providers which already have two factor authentication (default: false). +This configration dose not apply to SAML. + +```ruby +gitlab_rails['omniauth_allow_bypass_two_factor'] = ['twitter', 'google_oauth2'] +``` + +**For installations from source** + +```yaml +omniauth: + allow_bypass_two_factor: ['twitter', 'google_oauth2'] +``` diff --git a/doc/integration/salesforce.md b/doc/integration/salesforce.md index 176622e8050..10ab9d3c126 100644 --- a/doc/integration/salesforce.md +++ b/doc/integration/salesforce.md @@ -29,7 +29,7 @@ To get the credentials (a pair of Client ID and Client Secret), you must [create 1. On your GitLab server, open the configuration file. - For omnibus package: + For Omnibus package: ```sh sudo editor /etc/gitlab/gitlab.rb @@ -46,7 +46,7 @@ To get the credentials (a pair of Client ID and Client Secret), you must [create 1. Add the provider configuration: - For omnibus package: + For Omnibus package: ```ruby gitlab_rails['omniauth_providers'] = [ diff --git a/doc/integration/saml.md b/doc/integration/saml.md index 22e07594d6f..de160e72dda 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -17,7 +17,7 @@ in your SAML IdP: 1. On your GitLab server, open the configuration file. - For omnibus package: + For Omnibus package: ```sh sudo editor /etc/gitlab/gitlab.rb @@ -34,7 +34,7 @@ in your SAML IdP: 1. To allow your users to use SAML to sign up without having to manually create an account first, don't forget to add the following values to your configuration: - For omnibus package: + For Omnibus package: ```ruby gitlab_rails['omniauth_enabled'] = true @@ -54,7 +54,7 @@ in your SAML IdP: 1. You can also automatically link SAML users with existing GitLab users if their email addresses match by adding the following setting: - For omnibus package: + For Omnibus package: ```ruby gitlab_rails['omniauth_auto_link_saml_user'] = true @@ -68,7 +68,7 @@ in your SAML IdP: 1. Add the provider configuration: - For omnibus package: + For Omnibus package: ```ruby gitlab_rails['omniauth_providers'] = [ @@ -342,7 +342,7 @@ You can add this setting to your GitLab configuration to automatically redirect to your SAML server for authentication, thus removing the need to click a button before actually signing in. -For omnibus package: +For Omnibus package: ```ruby gitlab_rails['omniauth_auto_sign_in_with_provider'] = 'saml' diff --git a/doc/integration/shibboleth.md b/doc/integration/shibboleth.md index 27355d25266..ca5a8077e73 100644 --- a/doc/integration/shibboleth.md +++ b/doc/integration/shibboleth.md @@ -1,18 +1,18 @@ # Shibboleth OmniAuth Provider -This documentation is for enabling shibboleth with omnibus-gitlab package. +This documentation is for enabling Shibboleth with the Omnibus GitLab package. -In order to enable Shibboleth support in gitlab we need to use Apache instead of Nginx (It may be possible to use Nginx, however this is difficult to configure using the bundled Nginx provided in the omnibus-gitlab package). Apache uses mod_shib2 module for shibboleth authentication and can pass attributes as headers to omniauth-shibboleth provider. +In order to enable Shibboleth support in GitLab we need to use Apache instead of Nginx (It may be possible to use Nginx, however this is difficult to configure using the bundled Nginx provided in the Omnibus GitLab package). Apache uses mod_shib2 module for Shibboleth authentication and can pass attributes as headers to Omniauth Shibboleth provider. -To enable the Shibboleth OmniAuth provider you must configure Apache shibboleth module. +To enable the Shibboleth OmniAuth provider you must configure Apache Shibboleth module. The installation and configuration of the module itself is out of the scope of this document. Check <https://wiki.shibboleth.net/confluence/display/SP3/Apache> for more info. -You can find Apache config in gitlab-recipes (<https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache>). +You can find Apache config in [GitLab Recipes](https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache). The following changes are needed to enable Shibboleth: -1. Protect omniauth-shibboleth callback URL: +1. Protect Omniauth Shibboleth callback URL: ``` <Location /users/auth/shibboleth/callback> @@ -32,7 +32,7 @@ The following changes are needed to enable Shibboleth: </Location> ``` -1. Exclude shibboleth URLs from rewriting. Add `RewriteCond %{REQUEST_URI} !/Shibboleth.sso` and `RewriteCond %{REQUEST_URI} !/shibboleth-sp`. Config should look like this: +1. Exclude Shibboleth URLs from rewriting. Add `RewriteCond %{REQUEST_URI} !/Shibboleth.sso` and `RewriteCond %{REQUEST_URI} !/shibboleth-sp`. Config should look like this: ``` # Apache equivalent of Nginx try files @@ -50,7 +50,7 @@ The following changes are needed to enable Shibboleth: attribute mapping. Therefore the values of the `args` hash should be in the form of `"HTTP_ATTRIBUTE"`. The keys in the hash are arguments to the [OmniAuth::Strategies::Shibboleth class](https://github.com/toyokazu/omniauth-shibboleth/blob/master/lib/omniauth/strategies/shibboleth.rb) - and are documented by the [omniauth-shibboleth gem](https://github.com/toyokazu/omniauth-shibboleth) + and are documented by the [`omniauth-shibboleth` gem](https://github.com/toyokazu/omniauth-shibboleth) (take care to note the version of the gem packaged with GitLab). If some of your users appear to be authenticated by Shibboleth and Apache, but GitLab rejects their account with a URI that contains "e-mail is invalid" then your @@ -94,7 +94,7 @@ On the sign in page, there should now be a "Sign in with: Shibboleth" icon below ## Apache 2.4 / GitLab 8.6 update The order of the first 2 Location directives is important. If they are reversed, -you will not get a shibboleth session! +you will not get a Shibboleth session! ``` <Location /> diff --git a/doc/integration/twitter.md b/doc/integration/twitter.md index d8096993885..b2bd1b57d0f 100644 --- a/doc/integration/twitter.md +++ b/doc/integration/twitter.md @@ -32,7 +32,7 @@ To enable the Twitter OmniAuth provider you must register your application with 1. On your GitLab server, open the configuration file. - For omnibus package: + For Omnibus package: ```sh sudo editor /etc/gitlab/gitlab.rb @@ -50,7 +50,7 @@ To enable the Twitter OmniAuth provider you must register your application with 1. Add the provider configuration: - For omnibus package: + For Omnibus package: ```ruby gitlab_rails['omniauth_providers'] = [ diff --git a/doc/integration/ultra_auth.md b/doc/integration/ultra_auth.md index 9ed1bdb4882..fb950ba989a 100644 --- a/doc/integration/ultra_auth.md +++ b/doc/integration/ultra_auth.md @@ -25,7 +25,7 @@ To get the credentials (a pair of Client ID and Client Secret), you must registe 1. Select **Register application**. 1. On your GitLab server, open the configuration file. - For omnibus package: + For Omnibus package: ```sh sudo editor /etc/gitlab/gitlab.rb @@ -41,7 +41,7 @@ To get the credentials (a pair of Client ID and Client Secret), you must registe 1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. 1. Add the provider configuration: - For omnibus package: + For Omnibus package: ```ruby gitlab_rails['omniauth_providers'] = [ diff --git a/doc/project_services/bamboo.md b/doc/project_services/bamboo.md index a1ff8909f87..b1d37898516 100644 --- a/doc/project_services/bamboo.md +++ b/doc/project_services/bamboo.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/bamboo.md' --- -This document was moved to [user/project/integrations/bamboo.md](../user/project/integrations/bamboo.md). +This document was moved to [another location](../user/project/integrations/bamboo.md). diff --git a/doc/project_services/bugzilla.md b/doc/project_services/bugzilla.md index 1abf1b28939..17dff538c0e 100644 --- a/doc/project_services/bugzilla.md +++ b/doc/project_services/bugzilla.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/bugzilla.md' --- -This document was moved to [user/project/integrations/bugzilla.md](../user/project/integrations/bugzilla.md). +This document was moved to [another location](../user/project/integrations/bugzilla.md). diff --git a/doc/project_services/emails_on_push.md b/doc/project_services/emails_on_push.md index c5ab8aa5c70..a7d91934ce9 100644 --- a/doc/project_services/emails_on_push.md +++ b/doc/project_services/emails_on_push.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/emails_on_push.md' --- -This document was moved to [user/project/integrations/emails_on_push.md](../user/project/integrations/emails_on_push.md). +This document was moved to [another location](../user/project/integrations/emails_on_push.md). diff --git a/doc/project_services/hipchat.md b/doc/project_services/hipchat.md index 4ae9f6c6b2e..a2fbbd5cce5 100644 --- a/doc/project_services/hipchat.md +++ b/doc/project_services/hipchat.md @@ -1 +1,5 @@ -This document was moved to [user/project/integrations/hipchat.md](../user/project/integrations/hipchat.md). +--- +redirect_to: '../user/project/integrations/hipchat.md' +--- + +This document was moved to [another location](../user/project/integrations/hipchat.md). diff --git a/doc/project_services/irker.md b/doc/project_services/irker.md index af47abab117..70e46b1b364 100644 --- a/doc/project_services/irker.md +++ b/doc/project_services/irker.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/irker.md' --- -This document was moved to [user/project/integrations/irker.md](../user/project/integrations/irker.md). +This document was moved to [another location](../user/project/integrations/irker.md). diff --git a/doc/project_services/jira.md b/doc/project_services/jira.md index 6a0108f400f..37eba25fb5a 100644 --- a/doc/project_services/jira.md +++ b/doc/project_services/jira.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/jira.md' --- -This document was moved to [user/project/integrations/jira.md](../user/project/integrations/jira.md). +This document was moved to [another location](../user/project/integrations/jira.md). diff --git a/doc/project_services/kubernetes.md b/doc/project_services/kubernetes.md index cfe36fcd1b2..585c5ddb002 100644 --- a/doc/project_services/kubernetes.md +++ b/doc/project_services/kubernetes.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/kubernetes.md' --- -This document was moved to [user/project/integrations/kubernetes.md](../user/project/integrations/kubernetes.md). +This document was moved to [another location](../user/project/integrations/kubernetes.md). diff --git a/doc/project_services/mattermost.md b/doc/project_services/mattermost.md index de9f4d14cf7..78888395031 100644 --- a/doc/project_services/mattermost.md +++ b/doc/project_services/mattermost.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/mattermost.md' --- -This document was moved to [user/project/integrations/mattermost.md](../user/project/integrations/mattermost.md). +This document was moved to [another location](../user/project/integrations/mattermost.md). diff --git a/doc/project_services/mattermost_slash_commands.md b/doc/project_services/mattermost_slash_commands.md index 82ec34739c1..0c2774d95e0 100644 --- a/doc/project_services/mattermost_slash_commands.md +++ b/doc/project_services/mattermost_slash_commands.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/mattermost_slash_commands.md' --- -This document was moved to [user/project/integrations/mattermost_slash_commands.md](../user/project/integrations/mattermost_slash_commands.md). +This document was moved to [another location](../user/project/integrations/mattermost_slash_commands.md). diff --git a/doc/project_services/project_services.md b/doc/project_services/project_services.md index a355851a273..2ae7a0ce3ce 100644 --- a/doc/project_services/project_services.md +++ b/doc/project_services/project_services.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/project_services.md' --- -This document was moved to [user/project/integrations/project_services.md](../user/project/integrations/project_services.md). +This document was moved to [another location](../user/project/integrations/project_services.md). diff --git a/doc/project_services/redmine.md b/doc/project_services/redmine.md index 05f28f00adc..141c72d6b6b 100644 --- a/doc/project_services/redmine.md +++ b/doc/project_services/redmine.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/redmine.md' --- -This document was moved to [user/project/integrations/redmine.md](../user/project/integrations/redmine.md). +This document was moved to [another location](../user/project/integrations/redmine.md). diff --git a/doc/project_services/services_templates.md b/doc/project_services/services_templates.md index ac6b85cc801..8b2c85802de 100644 --- a/doc/project_services/services_templates.md +++ b/doc/project_services/services_templates.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/services_templates.md' --- -This document was moved to [user/project/integrations/services_templates.md](../user/project/integrations/services_templates.md). +This document was moved to [another location](../user/project/integrations/services_templates.md). diff --git a/doc/project_services/slack.md b/doc/project_services/slack.md index 4c89ce92002..815032a08d5 100644 --- a/doc/project_services/slack.md +++ b/doc/project_services/slack.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/slack.md' --- -This document was moved to [user/project/integrations/slack.md](../user/project/integrations/slack.md). +This document was moved to [another location](../user/project/integrations/slack.md). diff --git a/doc/project_services/slack_slash_commands.md b/doc/project_services/slack_slash_commands.md index ca0034256f1..caae4d2ba4b 100644 --- a/doc/project_services/slack_slash_commands.md +++ b/doc/project_services/slack_slash_commands.md @@ -2,4 +2,4 @@ redirect_to: '../user/project/integrations/slack_slash_commands.md' --- -This document was moved to [user/project/integrations/slack_slash_commands.md](../user/project/integrations/slack_slash_commands.md). +This document was moved to [another location](../user/project/integrations/slack_slash_commands.md). diff --git a/doc/raketasks/README.md b/doc/raketasks/README.md index dcc96507676..ad86555fc17 100644 --- a/doc/raketasks/README.md +++ b/doc/raketasks/README.md @@ -12,7 +12,7 @@ comments: false - [General Maintenance](../administration/raketasks/maintenance.md) and self-checks - [User management](user_management.md) - [Webhooks](web_hooks.md) -- [Import](import.md) of git repositories in bulk +- [Import](import.md) of Git repositories in bulk - [Rebuild authorized_keys file](../administration/raketasks/maintenance.md#rebuild-authorized_keys-file) task for administrators - [Migrate Uploads](../administration/raketasks/uploads/migrate.md) - [Sanitize Uploads](../administration/raketasks/uploads/sanitize.md) diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index b4b7d711d5a..336000f6cb9 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -94,7 +94,7 @@ docker exec -t <container name> gitlab-backup create If you are using the [GitLab helm chart](https://gitlab.com/charts/gitlab) on a Kubernetes cluster, you can run the backup task using `backup-utility` script on -the gitlab task runner pod via `kubectl`. Refer to [backing up a GitLab installation](https://gitlab.com/charts/gitlab/blob/master/doc/backup-restore/backup.md#backing-up-a-gitlab-installation) for more details: +the GitLab task runner pod via `kubectl`. Refer to [backing up a GitLab installation](https://gitlab.com/charts/gitlab/blob/master/doc/backup-restore/backup.md#backing-up-a-gitlab-installation) for more details: ```sh kubectl exec -it <gitlab task-runner pod> backup-utility @@ -458,7 +458,7 @@ You may also send backups to a mounted share (`NFS` / `CIFS` / `SMB` / etc.) by using the Fog [`Local`](https://github.com/fog/fog-local#usage) storage provider. The directory pointed to by the `local_root` key **must** be owned by the `git` user **when mounted** (mounting with the `uid=` of the `git` user for `CIFS` and -`SMB`) or the user that you are executing the backup tasks under (for omnibus +`SMB`) or the user that you are executing the backup tasks under (for Omnibus packages, this is the `git` user). The `backup_upload_remote_directory` **must** be set in addition to the @@ -507,7 +507,7 @@ For installations from source: ### Backup archive permissions The backup archives created by GitLab (`1393513186_2014_02_27_gitlab_backup.tar`) -will have owner/group git:git and 0600 permissions by default. +will have owner/group `git`/`git` and 0600 permissions by default. This is meant to avoid other system users reading GitLab's data. If you need the backup archives to have different permissions you can use the 'archive_permissions' setting. @@ -614,7 +614,7 @@ GitLab that you created it on, for example CE 9.1.0. You need to have a working GitLab installation before you can perform a restore. This is mainly because the system user performing the -restore actions ('git') is usually not allowed to create or delete +restore actions (`git`) is usually not allowed to create or delete the SQL database it needs to import data into ('gitlabhq_production'). All existing data will be either erased (SQL) or moved to a separate directory (repositories, uploads). @@ -773,14 +773,14 @@ In this case you can consider using filesystem snapshots as part of your backup Example: Amazon EBS -> A GitLab server using omnibus-gitlab hosted on Amazon AWS. +> A GitLab server using Omnibus GitLab hosted on Amazon AWS. > An EBS drive containing an ext4 filesystem is mounted at `/var/opt/gitlab`. > In this case you could make an application backup by taking an EBS snapshot. > The backup includes all repositories, uploads and Postgres data. Example: LVM snapshots + rsync -> A GitLab server using omnibus-gitlab, with an LVM logical volume mounted at `/var/opt/gitlab`. +> A GitLab server using Omnibus GitLab, with an LVM logical volume mounted at `/var/opt/gitlab`. > Replicating the `/var/opt/gitlab` directory using rsync would not be reliable because too many files would change while rsync is running. > Instead of rsync-ing `/var/opt/gitlab`, we create a temporary LVM snapshot, which we mount as a read-only filesystem at `/mnt/gitlab_backup`. > Now we can have a longer running rsync job which will create a consistent replica on the remote server. @@ -804,7 +804,7 @@ will have all your repositories, but not any other data. ## Troubleshooting -### Restoring database backup using omnibus packages outputs warnings +### Restoring database backup using Omnibus packages outputs warnings If you are using backup restore procedures you might encounter the following warnings: @@ -923,6 +923,29 @@ backup beforehand. UPDATE ci_runners SET token = null, token_encrypted = null; ``` +#### Reset pending pipeline jobs + +1. Enter the DB console: + + For Omnibus GitLab packages: + + ```sh + sudo gitlab-rails dbconsole + ``` + + For installations from source: + + ```sh + sudo -u git -H bundle exec rails dbconsole RAILS_ENV=production + ``` + +1. Clear all the tokens for pending jobs: + + ```sql + -- Clear build tokens + UPDATE ci_builds SET token = null, token_encrypted = null; + ``` + A similar strategy can be employed for the remaining features - by removing the data that cannot be decrypted, GitLab can be brought back into working order, and the lost data can be manually replaced. diff --git a/doc/raketasks/features.md b/doc/raketasks/features.md index 57c16f110e4..c06800a2aa3 100644 --- a/doc/raketasks/features.md +++ b/doc/raketasks/features.md @@ -6,7 +6,7 @@ This command will enable the namespaces feature introduced in v4.0. It will move Note: -- Because the **repository location will change**, you will need to **update all your git URLs** to point to the new location. +- Because the **repository location will change**, you will need to **update all your Git URLs** to point to the new location. - Username can be changed at **Profile ➔ Account**. **Example:** diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md index c76180a242e..d93e7241fda 100644 --- a/doc/raketasks/import.md +++ b/doc/raketasks/import.md @@ -13,7 +13,7 @@ ### Create a new folder to import your Git repositories from -The new folder needs to have git user ownership and read/write/execute access for git user and its group: +The new folder needs to have Git user ownership and read/write/execute access for Git user and its group: ``` sudo -u git mkdir -p /var/opt/gitlab/git-data/repository-import-<date>/new_group @@ -21,7 +21,7 @@ sudo -u git mkdir -p /var/opt/gitlab/git-data/repository-import-<date>/new_group ### Copy your bare repositories inside this newly created folder -- Any .git repositories found on any of the subfolders will be imported as projects +- Any `.git` repositories found on any of the subfolders will be imported as projects - Groups will be created as needed, these could be nested folders. Example: If we copy the repos to `/var/opt/gitlab/git-data/repository-import-<date>`, and repo A needs to be under the groups G1 and G2, it will @@ -34,7 +34,7 @@ sudo cp -r /old/git/foo.git /var/opt/gitlab/git-data/repository-import-<date>/ne sudo chown -R git:git /var/opt/gitlab/git-data/repository-import-<date> ``` -`foo.git` needs to be owned by the git user and git users group. +`foo.git` needs to be owned by the `git` user and `git` users group. If you are using an installation from source, replace `/var/opt/gitlab/` with `/home/git`. @@ -96,7 +96,7 @@ Importing bare repositories from hashed storage is unsupported. To support importing bare repositories from hashed storage, GitLab 10.4 and later stores the full project path with each repository, in a special section of -the git repository's config file. This section is formatted as follows: +the Git repository's config file. This section is formatted as follows: ``` [gitlab] diff --git a/doc/security/README.md b/doc/security/README.md index 5d498ac7602..e3fb07c69c2 100644 --- a/doc/security/README.md +++ b/doc/security/README.md @@ -5,6 +5,7 @@ type: index # Security +- [Password storage](password_storage.md) - [Password length limits](password_length_limits.md) - [Restrict SSH key technologies and minimum length](ssh_keys_restrictions.md) - [Rate limits](rate_limits.md) diff --git a/doc/security/password_storage.md b/doc/security/password_storage.md new file mode 100644 index 00000000000..f4e32f96f7b --- /dev/null +++ b/doc/security/password_storage.md @@ -0,0 +1,13 @@ +--- +type: reference +--- + +# Password Storage + +GitLab stores user passwords in a hashed format, to prevent passwords from being visible. + +GitLab uses the [Devise](https://github.com/plataformatec/devise) authentication library, which handles the hashing of user passwords. Password hashes are created with the following attributes: + +- **Hashing**: the [bcrypt](https://en.wikipedia.org/wiki/Bcrypt) hashing function is used to generate the hash of the provided password. This is a strong, industry-standard cryptographic hashing function. +- **Stretching**: Password hashes are [stretched](https://en.wikipedia.org/wiki/Key_stretching) to harden against brute-force attacks. GitLab uses a stretching factor of 10 by default. +- **Salting**: A [cryptographic salt](https://en.wikipedia.org/wiki/Salt_(cryptography)) is added to each password to harden against pre-computed hash and dictionary attacks. Each salt is randomly generated for each password, so that no two passwords share a salt, to further increase security. diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 26d5221acc5..6021e8cff1d 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -111,9 +111,8 @@ To make full use of Auto DevOps, you will need: a domain configured with wildcard DNS which is going to be used by all of your Auto DevOps applications. [Read the specifics](#auto-devops-base-domain). - **Kubernetes** (needed for Auto Review Apps, Auto Deploy, and Auto Monitoring) - - To enable deployments, you will need Kubernetes 1.5+. You need a [Kubernetes cluster][kubernetes-clusters] - for the project, or a Kubernetes [default service template](../../user/project/integrations/services_templates.md) - for the entire GitLab installation. + To enable deployments, you will need Kubernetes 1.5+. You will need a [Kubernetes cluster][kubernetes-clusters] + for the project. - **A load balancer** - You can use NGINX ingress by deploying it to your Kubernetes cluster using the [`nginx-ingress`](https://github.com/kubernetes/charts/tree/master/stable/nginx-ingress) diff --git a/doc/user/admin_area/monitoring/health_check.md b/doc/user/admin_area/monitoring/health_check.md index a3f4bc774ce..52f24c602df 100644 --- a/doc/user/admin_area/monitoring/health_check.md +++ b/doc/user/admin_area/monitoring/health_check.md @@ -28,14 +28,6 @@ With default whitelist settings, the probes can be accessed from localhost using ```text GET http://localhost/-/health ``` -```text -GET http://localhost/-/readiness -``` -```text -GET http://localhost/-/liveness -```text -GET http://localhost/-/health -``` ```text GET http://localhost/-/readiness diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 86491c7d74e..7b631a5a1cd 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -94,6 +94,36 @@ If you want to whitelist some specific vulnerabilities, you can do so by definin them in a YAML file named `clair-whitelist.yml`. Read more in the [Clair documentation](https://github.com/arminc/clair-scanner/blob/master/README.md#example-whitelist-yaml-file). +## Example + +The following is a sample `.gitlab-ci.yml` that will build your Docker Image, push it to the container registry and run Container Scanning. + +```yaml +variables: + DOCKER_DRIVER: overlay2 + +services: + - docker:stable-dind + +stages: + - build + - test + +include: + - template: Container-Scanning.gitlab-ci.yml + +build: + image: docker:stable + stage: build + variables: + IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA + script: + - docker info + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - docker build -t $IMAGE . + - docker push $IMAGE +``` + ## Security Dashboard The Security Dashboard is a good place to get an overview of all the security @@ -125,4 +155,4 @@ docker: Error response from daemon: failed to copy xattrs: failed to set xattr " This is a result of a bug in Docker which is now [fixed](https://github.com/containerd/continuity/pull/138 "fs: add WithAllowXAttrErrors CopyOpt"). To prevent the error, ensure the Docker version that the Runner is using is `18.09.03` or higher. For more information, see -[issue #10241](https://gitlab.com/gitlab-org/gitlab-ee/issues/10241 "Investigate why Container Scanning is not working with NFS mounts"). +[issue #10241](https://gitlab.com/gitlab-org/gitlab-ee/issues/10241 "Investigate why Container Scanning is not working with NFS mounts").
\ No newline at end of file diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 3148ec63c79..b40392e12d5 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -141,22 +141,22 @@ dependency_scanning: Dependency Scanning can be [configured](#customizing-the-dependency-scanning-settings) using environment variables. -| Environment variable | Function | -|-------------------------------- |----------| -| `DS_ANALYZER_IMAGES` | Comma separated list of custom images. The official default images are still enabled. Read more about [customizing analyzers](analyzers.md). | -| `DS_ANALYZER_IMAGE_PREFIX` | Override the name of the Docker registry providing the official default images (proxy). Read more about [customizing analyzers](analyzers.md). | -| `DS_ANALYZER_IMAGE_TAG` | Override the Docker tag of the official default images. Read more about [customizing analyzers](analyzers.md). | -| `DS_PYTHON_VERSION` | Version of Python. If set to 2, dependencies are installed using Python 2.7 instead of Python 3.6. ([Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/12296) in GitLab 12.1)| -| `DS_PIP_DEPENDENCY_PATH` | Path to load Python pip dependencies from. ([Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/12412) in GitLab 12.2) | -| `DS_DEFAULT_ANALYZERS` | Override the names of the official default images. Read more about [customizing analyzers](analyzers.md). | -| `DS_DISABLE_REMOTE_CHECKS` | Do not send any data to GitLab. Used in the [Gemnasium analyzer](#remote-checks). | -| `DS_PULL_ANALYZER_IMAGES` | Pull the images from the Docker registry (set to `0` to disable). | -| `DS_EXCLUDED_PATHS` | Exclude vulnerabilities from output based on the paths. A comma-separated list of patterns. Patterns can be globs, file or folder paths. Parent directories will also match patterns. | -| `DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT` | Time limit for Docker client negotiation. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h`, or `2h45m`. | -| `DS_PULL_ANALYZER_IMAGE_TIMEOUT` | Time limit when pulling the image of an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h`, or `2h45m`. | -| `DS_RUN_ANALYZER_TIMEOUT` | Time limit when running an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h`, or `2h45m`. | -| `PIP_INDEX_URL` | Base URL of Python Package Index (default `https://pypi.org/simple`). | -| `PIP_EXTRA_INDEX_URL` | Array of [extra URLs](https://pip.pypa.io/en/stable/reference/pip_install/#cmdoption-extra-index-url) of package indexes to use in addition to `PIP_INDEX_URL`. Comma separated. | +| Environment variable | Description | Example usage | +|-------------------------------- |-------------| | +| `DS_ANALYZER_IMAGES` | Comma separated list of custom images. The official default images are still enabled. Read more about [customizing analyzers](analyzers.md). | | +| `DS_ANALYZER_IMAGE_PREFIX` | Override the name of the Docker registry providing the official default images (proxy). Read more about [customizing analyzers](analyzers.md). | | +| `DS_ANALYZER_IMAGE_TAG` | Override the Docker tag of the official default images. Read more about [customizing analyzers](analyzers.md). | | +| `DS_PYTHON_VERSION` | Version of Python. If set to 2, dependencies are installed using Python 2.7 instead of Python 3.6. ([Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/12296) in GitLab 12.1)| | +| `DS_PIP_DEPENDENCY_PATH` | Path to load Python pip dependencies from. ([Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/12412) in GitLab 12.2) | | +| `DS_DEFAULT_ANALYZERS` | Override the names of the official default images. Read more about [customizing analyzers](analyzers.md). | | +| `DS_DISABLE_REMOTE_CHECKS` | Do not send any data to GitLab. Used in the [Gemnasium analyzer](#remote-checks). | | +| `DS_PULL_ANALYZER_IMAGES` | Pull the images from the Docker registry (set to `0` to disable). | | +| `DS_EXCLUDED_PATHS` | Exclude vulnerabilities from output based on the paths. A comma-separated list of patterns. Patterns can be globs, file or folder paths. Parent directories will also match patterns. | `DS_EXCLUDED_PATHS=doc,spec` | +| `DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT` | Time limit for Docker client negotiation. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h`, or `2h45m`. | | +| `DS_PULL_ANALYZER_IMAGE_TIMEOUT` | Time limit when pulling the image of an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h`, or `2h45m`. | | +| `DS_RUN_ANALYZER_TIMEOUT` | Time limit when running an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. For example, `300ms`, `1.5h`, or `2h45m`. | | +| `PIP_INDEX_URL` | Base URL of Python Package Index (default `https://pypi.org/simple`). | | +| `PIP_EXTRA_INDEX_URL` | Array of [extra URLs](https://pip.pypa.io/en/stable/reference/pip_install/#cmdoption-extra-index-url) of package indexes to use in addition to `PIP_INDEX_URL`. Comma separated. | | ## Reports JSON format @@ -276,8 +276,8 @@ it highlighted: Here is the description of the report file structure nodes and their meaning. All fields are mandatory to be present in the report JSON unless stated otherwise. Presence of optional fields depends on the underlying analyzers being used. -| Report JSON node | Function | -|------------------------------------------------------|----------| +| Report JSON node | Description | +|------------------------------------------------------|-------------| | `version` | Report syntax version used to generate this JSON. | | `vulnerabilities` | Array of vulnerability objects. | | `vulnerabilities[].category` | Where this vulnerability belongs (SAST, Dependency Scanning etc.). For Dependency Scanning, it will always be `dependency_scanning`. | diff --git a/doc/user/application_security/license_management/index.md b/doc/user/application_security/license_management/index.md index 912f2f0e209..44b2671930e 100644 --- a/doc/user/application_security/license_management/index.md +++ b/doc/user/application_security/license_management/index.md @@ -169,9 +169,9 @@ If you still need to run tests during `mvn install`, add `-DskipTests=false` to > [Introduced](https://gitlab.com/gitlab-org/security-products/license-management/merge_requests/36) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 12.0. -License Compliance uses Python 2.7 and pip 10.0 by default. -If your project requires Python 3, you can switch to Python 3.5 and pip 19.1 -by setting the `LM_PYTHON_VERSION` environment variable to `3`. +License Compliance uses Python 3.5 and pip 19.1 by default. +If your project requires Python 2, you can switch to Python 2.7 and pip 10.0 +by setting the `LM_PYTHON_VERSION` environment variable to `2`. ```yaml include: @@ -179,7 +179,7 @@ include: license_management: variables: - LM_PYTHON_VERSION: 3 + LM_PYTHON_VERSION: 2 ``` ## Project policies for License Compliance diff --git a/doc/user/application_security/sast/analyzers.md b/doc/user/application_security/sast/analyzers.md index f730a25a9fc..a1bd00f34e3 100644 --- a/doc/user/application_security/sast/analyzers.md +++ b/doc/user/application_security/sast/analyzers.md @@ -128,7 +128,7 @@ custom analyzer can scan the source code. | Start column | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | ✓ | 𐄂 | ✓ | ✓ | ✓ | 𐄂 | | End column | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | 𐄂 | | External id (e.g. CVE) | 𐄂 | 𐄂 | ⚠ | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | -| URLs | ✓ | 𐄂 | ✓ | 𐄂 | ⚠ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| URLs | ✓ | 𐄂 | ✓ | 𐄂 | ⚠ | 𐄂 | ⚠ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | | Internal doc/explanation | ✓ | ⚠ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | | Solution | ✓ | 𐄂 | 𐄂 | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | | Confidence | 𐄂 | ✓ | ✓ | 𐄂 | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index 2f15d997b5b..3eead6ccd3f 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -160,17 +160,21 @@ The following are Docker image-related variables. Some analyzers make it possible to filter out vulnerabilities under a given threshold. -| `SAST_BANDIT_EXCLUDED_PATHS` | - | comma-separated list of paths to exclude from scan. Uses Python's [`fnmatch` syntax](https://docs.python.org/2/library/fnmatch.html) | -| `SAST_BRAKEMAN_LEVEL` | 1 | Ignore Brakeman vulnerabilities under given confidence level. Integer, 1=Low 3=High. | -| `SAST_FLAWFINDER_LEVEL` | 1 | Ignore Flawfinder vulnerabilities under given risk level. Integer, 0=No risk, 5=High risk. | -| `SAST_GITLEAKS_ENTROPY_LEVEL` | 8.0 | Minimum entropy for secret detection. Float, 0.0 = low, 8.0 = high. | -| `SAST_GOSEC_LEVEL` | 0 | Ignore gosec vulnerabilities under given confidence level. Integer, 0=Undefined, 1=Low, 2=Medium, 3=High. | -| `SAST_EXCLUDED_PATHS` | - | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, file or folder paths. Parent directories will also match patterns. | +| Environment variable | Default value | Description | Example usage | +|----------------------|---------------|-------------|---| +| `SAST_BANDIT_EXCLUDED_PATHS` | - | comma-separated list of paths to exclude from scan. Uses Python's [`fnmatch` syntax](https://docs.python.org/2/library/fnmatch.html) | | +| `SAST_BRAKEMAN_LEVEL` | 1 | Ignore Brakeman vulnerabilities under given confidence level. Integer, 1=Low 3=High. | | +| `SAST_FLAWFINDER_LEVEL` | 1 | Ignore Flawfinder vulnerabilities under given risk level. Integer, 0=No risk, 5=High risk. | | +| `SAST_GITLEAKS_ENTROPY_LEVEL` | 8.0 | Minimum entropy for secret detection. Float, 0.0 = low, 8.0 = high. | | +| `SAST_GOSEC_LEVEL` | 0 | Ignore gosec vulnerabilities under given confidence level. Integer, 0=Undefined, 1=Low, 2=Medium, 3=High. | | +| `SAST_EXCLUDED_PATHS` | - | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, file or folder paths. Parent directories will also match patterns. | `SAST_EXCLUDED_PATHS=doc,spec` | ### Timeouts The following variables configure timeouts. +| Environment variable | Default value | Description | +|----------------------|---------------|-------------| | `SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT` | 2m | Time limit for Docker client negotiation. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". For example, "300ms", "1.5h" or "2h45m". | | `SAST_PULL_ANALYZER_IMAGE_TIMEOUT` | 5m | Time limit when pulling the image of an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". For example, "300ms", "1.5h" or "2h45m". | | `SAST_RUN_ANALYZER_TIMEOUT` | 20m | Time limit when running an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". For example, "300ms", "1.5h" or "2h45m".| diff --git a/doc/user/index.md b/doc/user/index.md index c93f64cd528..27e75189fc3 100644 --- a/doc/user/index.md +++ b/doc/user/index.md @@ -53,7 +53,7 @@ With GitLab Enterprise Edition, you can also: - Improve collaboration with [Merge Request Approvals](project/merge_requests/index.md#merge-request-approvals-starter), [Multiple Assignees for Issues](project/issues/multiple_assignees_for_issues.md), - and [Multiple Issue Boards](project/issue_board.md#multiple-issue-boards-starter). + and [Multiple Issue Boards](project/issue_board.md#multiple-issue-boards). - Create formal relationships between issues with [Related Issues](project/issues/related_issues.md). - Use [Burndown Charts](project/milestones/burndown_charts.md) to track progress during a sprint or while working on a new version of their software. - Leverage [Elasticsearch](../integration/elasticsearch.md) with [Advanced Global Search](search/advanced_global_search.md) and [Advanced Syntax Search](search/advanced_search_syntax.md) for faster, more advanced code search across your entire GitLab instance. diff --git a/doc/user/project/index.md b/doc/user/project/index.md index 932e7fd10b2..64c4066683b 100644 --- a/doc/user/project/index.md +++ b/doc/user/project/index.md @@ -17,7 +17,7 @@ When you create a project in GitLab, you'll have access to a large number of - [Issue tracker](issues/index.md): Discuss implementations with your team within issues - [Issue Boards](issue_board.md): Organize and prioritize your workflow - - [Multiple Issue Boards](issue_board.md#multiple-issue-boards-starter): Allow your teams to create their own workflows (Issue Boards) for the same project **(STARTER)** + - [Multiple Issue Boards](issue_board.md#multiple-issue-boards): Allow your teams to create their own workflows (Issue Boards) for the same project - [Repositories](repository/index.md): Host your code in a fully integrated platform - [Branches](repository/branches/index.md): use Git branching strategies to @@ -34,7 +34,7 @@ When you create a project in GitLab, you'll have access to a large number of - [Issue tracker](issues/index.md): Discuss implementations with your team within issues - [Issue Boards](issue_board.md): Organize and prioritize your workflow - - [Multiple Issue Boards](issue_board.md#multiple-issue-boards-starter): Allow your teams to create their own workflows (Issue Boards) for the same project **(STARTER)** + - [Multiple Issue Boards](issue_board.md#multiple-issue-boards): Allow your teams to create their own workflows (Issue Boards) for the same project - [Merge Requests](merge_requests/index.md): Apply your branching strategy and get reviewed by your team - [Merge Request Approvals](merge_requests/merge_request_approvals.md): Ask for approval before diff --git a/doc/user/project/integrations/prometheus.md b/doc/user/project/integrations/prometheus.md index 9fc0ade809e..a80332adc46 100644 --- a/doc/user/project/integrations/prometheus.md +++ b/doc/user/project/integrations/prometheus.md @@ -194,7 +194,7 @@ The following tables outline the details of expected properties. | Property | Type | Required | Description | | ------ | ------ | ------ | ------- | -| `type` | enum | no, defaults to `area-chart` | Specifies the chart type to use. | +| `type` | enum | no, defaults to `area-chart` | Specifies the chart type to use, can be `area-chart` or `line-chart` | | `title` | string | yes | Heading for the panel. | | `y_label` | string | no, but highly encouraged | Y-Axis label for the panel. | | `weight` | number | no, defaults to order in file | Order to appear within the grouping. Lower number means higher priority, which will be higher on the page. Numbers do not need to be consecutive. | diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index eaca5f8cfb8..519c02cf0ad 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -13,7 +13,7 @@ keeping everything in the same place, so that you don't need to jump between different platforms to organize your workflow. With GitLab Issue Boards, you organize your issues in lists that correspond to -their assigned labels, visualizing issues designed as cards throughout that lists. +their assigned labels, visualizing issues designed as cards throughout those lists. You define your process and GitLab organizes it for you. You add your labels then create the corresponding list to pull in your existing issues. When @@ -27,12 +27,9 @@ Issue Boards** (version introduced in GitLab 8.11 - August 2016). ### Advanced features of Issue Boards -With [GitLab Starter](https://about.gitlab.com/pricing/), you can create -[multiple issue boards](#multiple-issue-boards-starter) for a given project. **(STARTER)** - -With [GitLab Premium](https://about.gitlab.com/pricing/), you can also create multiple -issue boards for your groups, and add lists for [assignees](#assignee-lists-premium) and -[milestones](#milestone-lists-premium). **(PREMIUM)** +- Create multiple issue boards per project. +- Create multiple issue boards per group. **(PREMIUM)** +- Add lists for [assignees](#assignee-lists-premium) and [milestones](#milestone-lists-premium). **(PREMIUM)** Check all the [advanced features of Issue Boards](#gitlab-enterprise-features-for-issue-boards) below. @@ -58,8 +55,7 @@ You create issues, host code, perform reviews, build, test, and deploy from one single platform. Issue Boards help you to visualize and manage the entire process _in_ GitLab. -With [Multiple Issue Boards](#use-cases-for-multiple-issue-boards), available -only in [different tiers of GitLab Enterprise Edition](#gitlab-enterprise-features-for-issue-boards), +With [Multiple Issue Boards](#use-cases-for-multiple-issue-boards), you go even further, as you can not only keep yourself and your project organized from a broader perspective with one Issue Board per project, but also allow your team members to organize their own workflow by creating @@ -88,7 +84,7 @@ If we have the labels "**backend**", "**frontend**", "**staging**", and "**production**", and an Issue Board with a list for each, we can: - Visualize the entire flow of implementations since the - beginning of the development lifecycle until deployed to production + beginning of the development life cycle until deployed to production - Prioritize the issues in a list by moving them vertically - Move issues between lists to organize them according to the labels you've set - Add multiple issues to lists in the board by selecting one or more existing issues @@ -97,8 +93,7 @@ If we have the labels "**backend**", "**frontend**", "**staging**", and ### Use cases for Multiple Issue Boards -With [Multiple Issue Boards](#multiple-issue-boards-starter), available only in -[GitLab Enterprise Edition](https://about.gitlab.com/pricing/), +With [Multiple Issue Boards](#multiple-issue-boards), each team can have their own board to organize their workflow individually. #### Scrum team @@ -159,13 +154,14 @@ Issue Board, that is, create or delete lists and drag issues from one list to an GitLab Issue Boards are available on GitLab Core and GitLab.com Free, but some advanced functionalities are only present in higher tiers: GitLab.com Bronze, Silver, or Gold, or GitLab self-managed Starter, Premium, and Ultimate, as described -on the following sections. +in the following sections. For a collection of [features per tier](#summary-of-features-per-tier), check the summary below. -### Multiple Issue Boards **(STARTER)** +### Multiple Issue Boards -> Introduced in [GitLab Enterprise Edition 8.13](https://about.gitlab.com/2016/10/22/gitlab-8-13-released/#multiple-issue-boards-ee). +> - Multiple Issue Boards per project [moved](https://gitlab.com/gitlab-org/gitlab-ce/issues/53811) to [GitLab Core](https://about.gitlab.com/pricing/) in GitLab 12.1. +> - Multiple Issue Boards per group is available in [GitLab Premium Edition](https://about.gitlab.com/pricing/). Multiple Issue Boards, as the name suggests, allow for more than one Issue Board for a given project or group. This is great for large projects with more than one team @@ -184,10 +180,6 @@ These are shortcuts to your last 4 visited boards. When you're revisiting an issue board in a project or group with multiple boards, GitLab will automatically load the last board you visited. -NOTE: **Note:** -The Multiple Issue Boards feature is available for -**projects in GitLab Starter Edition** and for **groups in GitLab Premium Edition**. - ### Configurable Issue Boards **(STARTER)** > Introduced in [GitLab Starter Edition 10.2](https://about.gitlab.com/2017/11/22/gitlab-10-2-released/#issue-boards-configuration). diff --git a/doc/workflow/forking/branch_select.png b/doc/workflow/forking/branch_select.png Binary files differindex 77236137190..0ea5410f832 100644 --- a/doc/workflow/forking/branch_select.png +++ b/doc/workflow/forking/branch_select.png diff --git a/doc/workflow/forking/merge_request.png b/doc/workflow/forking/merge_request.png Binary files differindex 407ddfb4799..43851203f3f 100644 --- a/doc/workflow/forking/merge_request.png +++ b/doc/workflow/forking/merge_request.png diff --git a/doc/workflow/forking_workflow.md b/doc/workflow/forking_workflow.md index 869a0a621c3..82c2709143d 100644 --- a/doc/workflow/forking_workflow.md +++ b/doc/workflow/forking_workflow.md @@ -10,8 +10,7 @@ document more information about using branches to work together. Forking a project is in most cases a two-step process. -1. Click on the fork button located in the middle of the page or a project's - home page right next to the stars button. +1. Click on the fork button located located in between the star and clone buttons on the project's home page.  @@ -25,7 +24,7 @@ Forking a project is in most cases a two-step process. **Note:** If the namespace you chose to fork the project to has another project with the same path name, you will be presented with a warning that the forking - could not be completed. Try to resolve the error and repeat the forking + could not be completed. Try to resolve the error before repeating the forking process.  @@ -44,7 +43,7 @@ create the [merge request](merge_requests.md).  You can then assign the merge request to someone to have them review -your changes. Upon pressing the 'Accept Merge Request' button, your +your changes. Upon pressing the 'Submit Merge Request' button, your changes will be added to the repository and branch you're merging into.  diff --git a/doc/workflow/img/forking_workflow_choose_namespace.png b/doc/workflow/img/forking_workflow_choose_namespace.png Binary files differindex b34b12090a1..eb023ca85f2 100644 --- a/doc/workflow/img/forking_workflow_choose_namespace.png +++ b/doc/workflow/img/forking_workflow_choose_namespace.png diff --git a/doc/workflow/img/forking_workflow_fork_button.png b/doc/workflow/img/forking_workflow_fork_button.png Binary files differindex 941d5363c35..7fb07529b6d 100644 --- a/doc/workflow/img/forking_workflow_fork_button.png +++ b/doc/workflow/img/forking_workflow_fork_button.png diff --git a/doc/workflow/img/forking_workflow_path_taken_error.png b/doc/workflow/img/forking_workflow_path_taken_error.png Binary files differindex df938da5677..ef62d0ab6a9 100644 --- a/doc/workflow/img/forking_workflow_path_taken_error.png +++ b/doc/workflow/img/forking_workflow_path_taken_error.png diff --git a/lib/api/issues.rb b/lib/api/issues.rb index e16eeef202c..215178478d0 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -163,7 +163,8 @@ module API with_labels_details: declared_params[:with_labels_details], current_user: current_user, project: user_project, - issuable_metadata: issuable_meta_data(issues, 'Issue', current_user) + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user), + include_subscribed: false } present issues, options diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 83d645ca07a..de89e94b0c0 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -62,6 +62,31 @@ module API delete ':id/labels' do delete_label(user_project) end + + desc 'Promote a label to a group label' do + detail 'This feature was added in GitLab 12.3' + success Entities::GroupLabel + end + params do + requires :name, type: String, desc: 'The name of the label to be promoted' + end + put ':id/labels/promote' do + authorize! :admin_label, user_project + + label = find_label(user_project, params[:name], include_ancestor_groups: false) + + begin + group_label = ::Labels::PromoteService.new(user_project, current_user).execute(label) + + if group_label + present group_label, with: Entities::GroupLabel, current_user: current_user, parent: user_project.group + else + render_api_error!('Failed to promote project label to group label', 400) + end + rescue => error + render_api_error!(error.to_s, 400) + end + end end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index e17a096ef19..6769bd95c2b 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -198,12 +198,10 @@ module Gitlab end.uniq end - # rubocop: disable CodeReuse/ActiveRecord def deploy_token_check(login, password) return unless password.present? - token = - DeployToken.active.find_by(token: password) + token = DeployToken.active.find_by_token(password) return unless token && login return if login != token.username @@ -214,7 +212,6 @@ module Gitlab Gitlab::Auth::Result.new(token, token.project, :deploy_token, scopes) end end - # rubocop: enable CodeReuse/ActiveRecord def lfs_token_check(login, encoded_token, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 09d1d79fefc..f121dce4cbb 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -77,7 +77,12 @@ module Gitlab end def bypass_two_factor? - false + providers = Gitlab.config.omniauth.allow_bypass_two_factor + if providers.is_a?(Array) + providers.include?(auth_hash.provider) + else + providers + end end protected diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 332ca8bf9b8..5424298723e 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -126,6 +126,7 @@ module Gitlab %r{\A(ee/)?vendor/(languages\.yml|licenses\.csv)\z} => :backend, %r{\A(Dangerfile|Gemfile|Gemfile.lock|Procfile|Rakefile|\.gitlab-ci\.yml)\z} => :backend, %r{\A[A-Z_]+_VERSION\z} => :backend, + %r{\A\.rubocop(_todo)?\.yml\z} => :backend, %r{\A(ee/)?qa/} => :qa, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index cbdff0ab060..a12bbededc4 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -195,13 +195,14 @@ module Gitlab # pool_size - The size of the DB pool. # host - An optional host name to use instead of the default one. - def self.create_connection_pool(pool_size, host = nil) + def self.create_connection_pool(pool_size, host = nil, port = nil) # See activerecord-4.2.7.1/lib/active_record/connection_adapters/connection_specification.rb env = Rails.env original_config = ActiveRecord::Base.configurations env_config = original_config[env].merge('pool' => pool_size) env_config['host'] = host if host + env_config['port'] = port if port config = original_config.merge(env => env_config) diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 8bc45f6e78c..57a413f8e04 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -470,7 +470,7 @@ module Gitlab # We set the default value _after_ adding the column so we don't end up # updating any existing data with the default value. This isn't # necessary since we copy over old values further down. - change_column_default(table, new, old_col.default) if old_col.default + change_column_default(table, new, old_col.default) unless old_col.default.nil? install_rename_triggers(table, old, new) @@ -482,6 +482,16 @@ module Gitlab copy_foreign_keys(table, old, new) end + def undo_rename_column_concurrently(table, old, new) + trigger_name = rename_trigger_name(table, old, new) + + check_trigger_permissions!(table) + + remove_rename_triggers_for_postgresql(table, trigger_name) + + remove_column(table, new) + end + # Installs triggers in a table that keep a new column in sync with an old # one. # @@ -547,6 +557,35 @@ module Gitlab remove_column(table, old) end + def undo_cleanup_concurrent_column_rename(table, old, new, type: nil) + if transaction_open? + raise 'undo_cleanup_concurrent_column_rename can not be run inside a transaction' + end + + check_trigger_permissions!(table) + + new_column = column_for(table, new) + + add_column(table, old, type || new_column.type, + limit: new_column.limit, + precision: new_column.precision, + scale: new_column.scale) + + # We set the default value _after_ adding the column so we don't end up + # updating any existing data with the default value. This isn't + # necessary since we copy over old values further down. + change_column_default(table, old, new_column.default) unless new_column.default.nil? + + install_rename_triggers(table, old, new) + + update_column_in_batches(table, old, Arel::Table.new(table)[new]) + + change_column_null(table, old, false) unless new_column.null + + copy_indexes(table, new, old) + copy_foreign_keys(table, new, old) + end + # Changes the column type of a table using a background migration. # # Because this method uses a background migration it's more suitable for diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb index 6badacc2fc9..3a170e8b5f8 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -4,8 +4,6 @@ module Gitlab module DatabaseImporters module SelfMonitoring module Project - include Stepable - class CreateService < ::BaseService include Stepable diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index 3c71031a8d9..841f9de8d4a 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -20,7 +20,7 @@ module Gitlab path: repo.path, namespace: namespace, creator: current_user, - visibility_level: Gitlab::VisibilityLevel::INTERNAL, + visibility_level: Gitlab::VisibilityLevel::PRIVATE, import_type: 'fogbugz', import_source: repo.name, import_url: Project::UNKNOWN_IMPORT_URL, diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb index ab03c40c22d..941a4f434a1 100644 --- a/lib/gitlab/graphql/present/instrumentation.rb +++ b/lib/gitlab/graphql/present/instrumentation.rb @@ -23,7 +23,9 @@ module Gitlab end presenter = presented_in.presenter_class.new(object, **context.to_h) - wrapped = presented_type.class.new(presenter, context) + + # we have to use the new `authorized_new` method, as `new` is protected + wrapped = presented_type.class.authorized_new(presenter, context) old_resolver.call(wrapped, args, context) end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 353298e67b3..a93301cb4ce 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -142,7 +142,8 @@ module Gitlab Gitlab::UsageDataCounters::SnippetCounter, Gitlab::UsageDataCounters::SearchCounter, Gitlab::UsageDataCounters::CycleAnalyticsCounter, - Gitlab::UsageDataCounters::SourceCodeCounter + Gitlab::UsageDataCounters::SourceCodeCounter, + Gitlab::UsageDataCounters::MergeRequestCounter ] end diff --git a/lib/gitlab/usage_data_counters/merge_request_counter.rb b/lib/gitlab/usage_data_counters/merge_request_counter.rb new file mode 100644 index 00000000000..e786e595f77 --- /dev/null +++ b/lib/gitlab/usage_data_counters/merge_request_counter.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class MergeRequestCounter < BaseCounter + KNOWN_EVENTS = %w[create].freeze + PREFIX = 'merge_request' + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 61642fbbd59..e44c18979cd 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -2977,6 +2977,9 @@ msgstr "" msgid "Commands applied" msgstr "" +msgid "Commands did not apply" +msgstr "" + msgid "Comment" msgstr "" @@ -7110,9 +7113,6 @@ msgstr "" msgid "Minimum length is %{minimum_password_length} characters." msgstr "" -msgid "Mirror a repository" -msgstr "" - msgid "Mirror direction" msgstr "" @@ -8299,7 +8299,7 @@ msgstr "" msgid "Preferences|Behavior" msgstr "" -msgid "Preferences|Choose between fixed (max. 1280px) and fluid (100%%) application layout." +msgid "Preferences|Choose between fixed (max. 1280px) and fluid (%{percentage}) application layout." msgstr "" msgid "Preferences|Choose what content you want to see on a project’s overview page." diff --git a/package.json b/package.json index 384584a944a..8fa34cdea28 100644 --- a/package.json +++ b/package.json @@ -37,8 +37,8 @@ "@babel/plugin-syntax-import-meta": "^7.2.0", "@babel/preset-env": "^7.5.5", "@gitlab/csslab": "^1.9.0", - "@gitlab/svgs": "^1.68.0", - "@gitlab/ui": "5.18.0", + "@gitlab/svgs": "^1.70.0", + "@gitlab/ui": "5.19.0", "@gitlab/visual-review-tools": "^1.0.0", "apollo-cache-inmemory": "^1.5.1", "apollo-client": "^2.5.1", diff --git a/qa/Dockerfile b/qa/Dockerfile index 3309f5b6ce3..84dbfae1008 100644 --- a/qa/Dockerfile +++ b/qa/Dockerfile @@ -50,6 +50,8 @@ RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ WORKDIR /home/gitlab/qa COPY ./qa/Gemfile* /home/gitlab/qa/ COPY ./config/initializers/0_inject_enterprise_edition_module.rb /home/gitlab/config/initializers/ +# Copy VERSION to ensure the COPY succeeds to copy at least one file since ee/app/models/license.rb isn't present in CE +COPY VERSION ./ee/app/models/license.r[b] /home/gitlab/ee/app/models/ COPY ./lib/gitlab.rb /home/gitlab/lib/ COPY ./INSTALLATION_TYPE /home/gitlab/ COPY ./VERSION /home/gitlab/ @@ -359,6 +359,13 @@ module QA autoload :KubernetesCluster, 'qa/service/kubernetes_cluster' autoload :Omnibus, 'qa/service/omnibus' autoload :Runner, 'qa/service/runner' + + module ClusterProvider + autoload :Base, 'qa/service/cluster_provider/base' + autoload :Gcloud, 'qa/service/cluster_provider/gcloud' + autoload :Minikube, 'qa/service/cluster_provider/minikube' + autoload :K3d, 'qa/service/cluster_provider/k3d' + end end ## diff --git a/qa/qa/service/cluster_provider/base.rb b/qa/qa/service/cluster_provider/base.rb new file mode 100644 index 00000000000..a9678557aca --- /dev/null +++ b/qa/qa/service/cluster_provider/base.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module QA + module Service + module ClusterProvider + class Base + include Service::Shellout + + attr_reader :rbac + + def initialize(rbac:) + @rbac = rbac + end + + def cluster_name + @cluster_name ||= "qa-cluster-#{Time.now.utc.strftime("%Y%m%d%H%M%S")}-#{SecureRandom.hex(4)}" + end + + def set_credentials(admin_user) + raise NotImplementedError + end + + def validate_dependencies + raise NotImplementedError + end + + def setup + raise NotImplementedError + end + + def teardown + raise NotImplementedError + end + + def filter_credentials(credentials) + credentials + end + end + end + end +end diff --git a/qa/qa/service/cluster_provider/gcloud.rb b/qa/qa/service/cluster_provider/gcloud.rb new file mode 100644 index 00000000000..9c82151666c --- /dev/null +++ b/qa/qa/service/cluster_provider/gcloud.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module QA + module Service + module ClusterProvider + class Gcloud < Base + def validate_dependencies + find_executable('gcloud') || raise("You must first install `gcloud` executable to run these tests.") + end + + def set_credentials(admin_user) + master_auth = JSON.parse(`gcloud container clusters describe #{cluster_name} --region #{Runtime::Env.gcloud_region} --format 'json(masterAuth.username, masterAuth.password)'`) + + shell <<~CMD.tr("\n", ' ') + kubectl config set-credentials #{admin_user} + --username #{master_auth['masterAuth']['username']} + --password #{master_auth['masterAuth']['password']} + CMD + end + + def setup + login_if_not_already_logged_in + create_cluster + end + + def teardown + delete_cluster + end + + private + + def login_if_not_already_logged_in + if Runtime::Env.has_gcloud_credentials? + attempt_login_with_env_vars + else + account = `gcloud auth list --filter=status:ACTIVE --format="value(account)"` + if account.empty? + raise "Failed to login to gcloud. No credentials provided in environment and no credentials found locally." + else + puts "gcloud account found. Using: #{account} for creating K8s cluster." + end + end + end + + def attempt_login_with_env_vars + puts "No gcloud account. Attempting to login from env vars GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY." + gcloud_account_key = Tempfile.new('gcloud-account-key') + gcloud_account_key.write(Runtime::Env.gcloud_account_key) + gcloud_account_key.close + gcloud_account_email = Runtime::Env.gcloud_account_email + shell("gcloud auth activate-service-account #{gcloud_account_email} --key-file #{gcloud_account_key.path}") + ensure + gcloud_account_key && gcloud_account_key.unlink + end + + def auth_options + "--enable-legacy-authorization" unless rbac + end + + def create_cluster + shell <<~CMD.tr("\n", ' ') + gcloud container clusters + create #{cluster_name} + #{auth_options} + --enable-basic-auth + --region #{Runtime::Env.gcloud_region} + --disk-size 10GB + --num-nodes #{Runtime::Env.gcloud_num_nodes} + && gcloud container clusters + get-credentials + --region #{Runtime::Env.gcloud_region} + #{cluster_name} + CMD + end + + def delete_cluster + shell <<~CMD.tr("\n", ' ') + gcloud container clusters delete + --region #{Runtime::Env.gcloud_region} + #{cluster_name} + --quiet --async + CMD + end + end + end + end +end diff --git a/qa/qa/service/cluster_provider/k3d.rb b/qa/qa/service/cluster_provider/k3d.rb new file mode 100644 index 00000000000..8e117c2dbd5 --- /dev/null +++ b/qa/qa/service/cluster_provider/k3d.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +module QA + module Service + module ClusterProvider + class K3d < Base + def validate_dependencies + find_executable('k3d') || raise("You must first install `k3d` executable to run these tests.") + end + + def set_credentials(admin_user) + end + + def setup + shell "k3d create --workers 1 --name #{cluster_name} --wait 0" + + @old_kubeconfig = ENV['KUBECONFIG'] + ENV['KUBECONFIG'] = fetch_kubeconfig + raise "Could not fetch kubeconfig" unless ENV['KUBECONFIG'] + + install_local_storage + end + + def teardown + ENV['KUBECONFIG'] = @old_kubeconfig + shell "k3d delete --name #{cluster_name}" + end + + # Fetch "real" certificate + # See https://github.com/rancher/k3s/issues/27 + def filter_credentials(credentials) + kubeconfig = YAML.load_file(ENV['KUBECONFIG']) + ca_certificate = kubeconfig.dig('clusters', 0, 'cluster', 'certificate-authority-data') + + credentials.merge('data' => credentials['data'].merge('ca.crt' => ca_certificate)) + end + + private + + def retry_until(max_attempts: 10, wait: 1) + max_attempts.times do + result = yield + return result if result + + sleep wait + end + + raise "Retried #{max_attempts} times. Aborting" + end + + def fetch_kubeconfig + retry_until do + config = `k3d get-kubeconfig --name #{cluster_name}`.chomp + config if config =~ /kubeconfig.yaml/ + end + end + + def install_local_storage + shell('kubectl apply -f -', stdin_data: local_storage_config) + end + + # See https://github.com/rancher/k3d/issues/67 + def local_storage_config + <<~YAML + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: storage-provisioner + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: storage-provisioner + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:persistent-volume-provisioner + subjects: + - kind: ServiceAccount + name: storage-provisioner + namespace: kube-system + --- + apiVersion: v1 + kind: Pod + metadata: + name: storage-provisioner + namespace: kube-system + spec: + serviceAccountName: storage-provisioner + tolerations: + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 300 + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 300 + hostNetwork: true + containers: + - name: storage-provisioner + image: gcr.io/k8s-minikube/storage-provisioner:v1.8.1 + command: ["/storage-provisioner"] + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /tmp + name: tmp + volumes: + - name: tmp + hostPath: + path: /tmp + type: Directory + --- + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: standard + namespace: kube-system + annotations: + storageclass.kubernetes.io/is-default-class: "true" + labels: + addonmanager.kubernetes.io/mode: EnsureExists + provisioner: k8s.io/minikube-hostpath + YAML + end + end + end + end +end diff --git a/qa/qa/service/cluster_provider/minikube.rb b/qa/qa/service/cluster_provider/minikube.rb new file mode 100644 index 00000000000..fc916245357 --- /dev/null +++ b/qa/qa/service/cluster_provider/minikube.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module QA + module Service + module ClusterProvider + class Minikube < Base + def validate_dependencies + find_executable('minikube') || raise("You must first install `minikube` executable to run these tests.") + end + + def set_credentials(admin_user) + end + + def setup + shell 'minikube stop' + shell "minikube profile #{cluster_name}" + shell 'minikube start' + end + + def teardown + shell 'minikube delete' + end + end + end + end +end diff --git a/qa/qa/service/kubernetes_cluster.rb b/qa/qa/service/kubernetes_cluster.rb index ac0b6313167..26b5f58d2d3 100644 --- a/qa/qa/service/kubernetes_cluster.rb +++ b/qa/qa/service/kubernetes_cluster.rb @@ -9,90 +9,63 @@ module QA class KubernetesCluster include Service::Shellout - attr_reader :api_url, :ca_certificate, :token, :rbac + attr_reader :api_url, :ca_certificate, :token, :rbac, :provider - def initialize(rbac: true) + def initialize(rbac: true, provider_class: QA::Service::ClusterProvider::Gcloud) @rbac = rbac - end - - def cluster_name - @cluster_name ||= "qa-cluster-#{SecureRandom.hex(4)}-#{Time.now.utc.strftime("%Y%m%d%H%M%S")}" + @provider = provider_class.new(rbac: rbac) end def create! validate_dependencies - login_if_not_already_logged_in - - shell <<~CMD.tr("\n", ' ') - gcloud container clusters - create #{cluster_name} - #{auth_options} - --enable-basic-auth - --region #{Runtime::Env.gcloud_region} - --disk-size 10GB - --num-nodes #{Runtime::Env.gcloud_num_nodes} - && gcloud container clusters - get-credentials - --region #{Runtime::Env.gcloud_region} - #{cluster_name} - CMD - - @api_url = `kubectl config view --minify -o jsonpath='{.clusters[].cluster.server}'` - - @admin_user = "#{cluster_name}-admin" - master_auth = JSON.parse(`gcloud container clusters describe #{cluster_name} --region #{Runtime::Env.gcloud_region} --format 'json(masterAuth.username, masterAuth.password)'`) - shell <<~CMD.tr("\n", ' ') - kubectl config set-credentials #{@admin_user} - --username #{master_auth['masterAuth']['username']} - --password #{master_auth['masterAuth']['password']} - CMD - - if rbac - create_service_account - - secrets = JSON.parse(`kubectl get secrets -o json`) - gitlab_account = secrets['items'].find do |item| - item['metadata']['annotations']['kubernetes.io/service-account.name'] == 'gitlab-account' - end - - @ca_certificate = Base64.decode64(gitlab_account['data']['ca.crt']) - @token = Base64.decode64(gitlab_account['data']['token']) - else - @ca_certificate = Base64.decode64(`kubectl get secrets -o jsonpath="{.items[0].data['ca\\.crt']}"`) - @token = Base64.decode64(`kubectl get secrets -o jsonpath='{.items[0].data.token}'`) - end + + @provider.validate_dependencies + @provider.setup + + @api_url = fetch_api_url + + credentials = @provider.filter_credentials(fetch_credentials) + @ca_certificate = Base64.decode64(credentials.dig('data', 'ca.crt')) + @token = Base64.decode64(credentials.dig('data', 'token')) self end def remove! - shell <<~CMD.tr("\n", ' ') - gcloud container clusters delete - --region #{Runtime::Env.gcloud_region} - #{cluster_name} - --quiet --async - CMD + @provider.teardown + end + + def cluster_name + @provider.cluster_name end private - def create_service_account - shell('kubectl create -f -', stdin_data: service_account) - shell("kubectl --user #{@admin_user} create -f -", stdin_data: service_account_role_binding) + def fetch_api_url + `kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}'` + end + + def fetch_credentials + return global_credentials unless rbac + + @provider.set_credentials(admin_user) + create_service_account(admin_user) + account_credentials end - def service_account - <<~YAML + def admin_user + @admin_user ||= "#{@provider.cluster_name}-admin" + end + + def create_service_account(user) + service_account = <<~YAML + --- apiVersion: v1 kind: ServiceAccount metadata: name: gitlab-account namespace: default - YAML - end - - def service_account_role_binding - <<~YAML + --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -106,39 +79,24 @@ module QA name: cluster-admin apiGroup: rbac.authorization.k8s.io YAML - end - def auth_options - "--enable-legacy-authorization" unless rbac + shell('kubectl apply -f -', stdin_data: service_account) end - def validate_dependencies - find_executable('gcloud') || raise("You must first install `gcloud` executable to run these tests.") - find_executable('kubectl') || raise("You must first install `kubectl` executable to run these tests.") - end + def account_credentials + secrets = JSON.parse(`kubectl get secrets -o json`) - def login_if_not_already_logged_in - if Runtime::Env.has_gcloud_credentials? - attempt_login_with_env_vars - else - account = `gcloud auth list --filter=status:ACTIVE --format="value(account)"` - if account.empty? - raise "Failed to login to gcloud. No credentials provided in environment and no credentials found locally." - else - puts "gcloud account found. Using: #{account} for creating K8s cluster." - end + secrets['items'].find do |item| + item['metadata']['annotations']['kubernetes.io/service-account.name'] == 'gitlab-account' end end - def attempt_login_with_env_vars - puts "No gcloud account. Attempting to login from env vars GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY." - gcloud_account_key = Tempfile.new('gcloud-account-key') - gcloud_account_key.write(Runtime::Env.gcloud_account_key) - gcloud_account_key.close - gcloud_account_email = Runtime::Env.gcloud_account_email - shell("gcloud auth activate-service-account #{gcloud_account_email} --key-file #{gcloud_account_key.path}") - ensure - gcloud_account_key && gcloud_account_key.unlink + def global_credentials + JSON.parse(`kubectl get secrets -o jsonpath='{.items[0]}'`) + end + + def validate_dependencies + find_executable('kubectl') || raise("You must first install `kubectl` executable to run these tests.") end end end diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/add_file_template_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/add_file_template_spec.rb index 458072b1507..567c6a83ddf 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/add_file_template_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/add_file_template_spec.rb @@ -1,8 +1,7 @@ # frozen_string_literal: true module QA - # Failure issue: https://gitlab.com/gitlab-org/quality/nightly/issues/127 - context 'Create', :quarantine do + context 'Create' do describe 'File templates' do include Runtime::Fixtures diff --git a/qa/qa/specs/features/browser_ui/3_create/web_ide/add_file_template_spec.rb b/qa/qa/specs/features/browser_ui/3_create/web_ide/add_file_template_spec.rb index 9ff7919f199..c09c65a57a5 100644 --- a/qa/qa/specs/features/browser_ui/3_create/web_ide/add_file_template_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/web_ide/add_file_template_spec.rb @@ -1,8 +1,7 @@ # frozen_string_literal: true module QA - context 'Create', :quarantine do - # Failure issue: https://gitlab.com/gitlab-org/quality/nightly/issues/127 + context 'Create' do describe 'Web IDE file templates' do include Runtime::Fixtures diff --git a/rubocop/cop/gitlab/httparty.rb b/rubocop/cop/gitlab/httparty.rb index 215f18b6993..8acebff624d 100644 --- a/rubocop/cop/gitlab/httparty.rb +++ b/rubocop/cop/gitlab/httparty.rb @@ -1,11 +1,9 @@ -require_relative '../../spec_helpers' +# frozen_string_literal: true module RuboCop module Cop module Gitlab class HTTParty < RuboCop::Cop::Cop - include SpecHelpers - MSG_SEND = <<~EOL.freeze Avoid calling `HTTParty` directly. Instead, use the Gitlab::HTTP wrapper. To allow request to localhost or the private network set @@ -27,8 +25,6 @@ module RuboCop PATTERN def on_send(node) - return if in_spec?(node) - add_offense(node, location: :expression, message: MSG_SEND) if httparty_node?(node) add_offense(node, location: :expression, message: MSG_INCLUDE) if includes_httparty?(node) end diff --git a/rubocop/cop/gitlab/union.rb b/rubocop/cop/gitlab/union.rb index 09541d8af3b..c44c847657b 100644 --- a/rubocop/cop/gitlab/union.rb +++ b/rubocop/cop/gitlab/union.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -require_relative '../../spec_helpers' module RuboCop module Cop @@ -7,8 +6,6 @@ module RuboCop # Cop that disallows the use of `Gitlab::SQL::Union`, in favour of using # the `FromUnion` module. class Union < RuboCop::Cop::Cop - include SpecHelpers - MSG = 'Use the `FromUnion` concern, instead of using `Gitlab::SQL::Union` directly' def_node_matcher :raw_union?, <<~PATTERN @@ -17,7 +14,6 @@ module RuboCop def on_send(node) return unless raw_union?(node) - return if in_spec?(node) add_offense(node, location: :expression) end diff --git a/rubocop/cop/graphql/authorize_types.rb b/rubocop/cop/graphql/authorize_types.rb index 93fe80c3edf..cd8bdbaee59 100644 --- a/rubocop/cop/graphql/authorize_types.rb +++ b/rubocop/cop/graphql/authorize_types.rb @@ -1,13 +1,9 @@ # frozen_string_literal: true -require_relative '../../spec_helpers' - module RuboCop module Cop module Graphql class AuthorizeTypes < RuboCop::Cop::Cop - include SpecHelpers - MSG = 'Add an `authorize :ability` call to the type: '\ 'https://docs.gitlab.com/ee/development/api_graphql_styleguide.html#type-authorization' @@ -32,8 +28,6 @@ module RuboCop private def in_type?(node) - return if in_spec?(node) - path = node.location.expression.source_buffer.name path.include?(TYPES_DIR) diff --git a/rubocop/cop/include_action_view_context.rb b/rubocop/cop/include_action_view_context.rb index 14662a33e95..52c84a711c9 100644 --- a/rubocop/cop/include_action_view_context.rb +++ b/rubocop/cop/include_action_view_context.rb @@ -1,13 +1,9 @@ # frozen_string_literal: true -require_relative '../spec_helpers' - module RuboCop module Cop # Cop that makes sure workers include `::Gitlab::ActionViewOutput::Context`, not `ActionView::Context`. class IncludeActionViewContext < RuboCop::Cop::Cop - include SpecHelpers - MSG = 'Include `::Gitlab::ActionViewOutput::Context`, not `ActionView::Context`, for Rails 5.'.freeze def_node_matcher :includes_action_view_context?, <<~PATTERN @@ -15,7 +11,6 @@ module RuboCop PATTERN def on_send(node) - return if in_spec?(node) return unless includes_action_view_context?(node) add_offense(node.arguments.first, location: :expression) diff --git a/rubocop/cop/include_sidekiq_worker.rb b/rubocop/cop/include_sidekiq_worker.rb index 8da4a147219..e69bc018add 100644 --- a/rubocop/cop/include_sidekiq_worker.rb +++ b/rubocop/cop/include_sidekiq_worker.rb @@ -1,11 +1,9 @@ -require_relative '../spec_helpers' +# frozen_string_literal: true module RuboCop module Cop # Cop that makes sure workers include `ApplicationWorker`, not `Sidekiq::Worker`. class IncludeSidekiqWorker < RuboCop::Cop::Cop - include SpecHelpers - MSG = 'Include `ApplicationWorker`, not `Sidekiq::Worker`.'.freeze def_node_matcher :includes_sidekiq_worker?, <<~PATTERN @@ -13,7 +11,6 @@ module RuboCop PATTERN def on_send(node) - return if in_spec?(node) return unless includes_sidekiq_worker?(node) add_offense(node.arguments.first, location: :expression) diff --git a/rubocop/cop/rspec/env_assignment.rb b/rubocop/cop/rspec/env_assignment.rb index 8b61fa8e264..73e108c2232 100644 --- a/rubocop/cop/rspec/env_assignment.rb +++ b/rubocop/cop/rspec/env_assignment.rb @@ -1,4 +1,4 @@ -require_relative '../../spec_helpers' +# frozen_string_literal: true module RuboCop module Cop @@ -17,8 +17,6 @@ module RuboCop # stub_env('FOO', 'bar') # end class EnvAssignment < RuboCop::Cop::Cop - include SpecHelpers - MESSAGE = "Don't assign to ENV, use `stub_env` instead.".freeze def_node_search :env_assignment?, <<~PATTERN @@ -28,7 +26,6 @@ module RuboCop # Following is what node.children looks like on a match # [s(:const, nil, :ENV), :[]=, s(:str, "key"), s(:str, "value")] def on_send(node) - return unless in_spec?(node) return unless env_assignment?(node) add_offense(node, location: :expression, message: MESSAGE) diff --git a/rubocop/cop/rspec/factories_in_migration_specs.rb b/rubocop/cop/rspec/factories_in_migration_specs.rb index 0c5aa838a2c..65c7638a0f4 100644 --- a/rubocop/cop/rspec/factories_in_migration_specs.rb +++ b/rubocop/cop/rspec/factories_in_migration_specs.rb @@ -1,4 +1,4 @@ -require_relative '../../spec_helpers' +# frozen_string_literal: true module RuboCop module Cop @@ -14,8 +14,6 @@ module RuboCop # let(:users) { table(:users) } # let(:user) { users.create!(name: 'User 1', username: 'user1') } class FactoriesInMigrationSpecs < RuboCop::Cop::Cop - include SpecHelpers - MESSAGE = "Don't use FactoryBot.%s in migration specs, use `table` instead.".freeze FORBIDDEN_METHODS = %i[build build_list create create_list].freeze @@ -27,7 +25,6 @@ module RuboCop # - Without FactoryBot namespace: [nil, :build, s(:sym, :user)] # - With FactoryBot namespace: [s(:const, nil, :FactoryBot), :build, s(:sym, :user)] def on_send(node) - return unless in_migration_spec?(node) return unless forbidden_factory_usage?(node) method = node.children[1] diff --git a/rubocop/cop/sidekiq_options_queue.rb b/rubocop/cop/sidekiq_options_queue.rb index 253d2958866..499c712175e 100644 --- a/rubocop/cop/sidekiq_options_queue.rb +++ b/rubocop/cop/sidekiq_options_queue.rb @@ -1,11 +1,9 @@ -require_relative '../spec_helpers' +# frozen_string_literal: true module RuboCop module Cop # Cop that prevents manually setting a queue in Sidekiq workers. class SidekiqOptionsQueue < RuboCop::Cop::Cop - include SpecHelpers - MSG = 'Do not manually set a queue; `ApplicationWorker` sets one automatically.'.freeze def_node_matcher :sidekiq_options?, <<~PATTERN @@ -13,7 +11,6 @@ module RuboCop PATTERN def on_send(node) - return if in_spec?(node) return unless sidekiq_options?(node) node.arguments.first.each_node(:pair) do |pair| diff --git a/rubocop/spec_helpers.rb b/rubocop/spec_helpers.rb deleted file mode 100644 index ecd77c4351d..00000000000 --- a/rubocop/spec_helpers.rb +++ /dev/null @@ -1,30 +0,0 @@ -module RuboCop - module SpecHelpers - SPEC_HELPERS = %w[fast_spec_helper.rb rails_helper.rb spec_helper.rb].freeze - MIGRATION_SPEC_DIRECTORIES = ['spec/migrations', 'spec/lib/gitlab/background_migration'].freeze - - # Returns true if the given node originated from the spec directory. - def in_spec?(node) - path = node.location.expression.source_buffer.name - pwd = RuboCop::PathUtil.pwd - - !SPEC_HELPERS.include?(File.basename(path)) && - path.start_with?(File.join(pwd, 'spec'), File.join(pwd, 'ee', 'spec')) - end - - def migration_directories - @migration_directories ||= MIGRATION_SPEC_DIRECTORIES.map do |dir| - pwd = RuboCop::PathUtil.pwd - [File.join(pwd, dir), File.join(pwd, 'ee', dir)] - end.flatten - end - - # Returns true if the given node originated from a migration spec. - def in_migration_spec?(node) - path = node.location.expression.source_buffer.name - - in_spec?(node) && - path.start_with?(*migration_directories) - end - end -end diff --git a/spec/factories/ci/job_artifacts.rb b/spec/factories/ci/job_artifacts.rb index 2d68a8e9fe3..6f553cadfa3 100644 --- a/spec/factories/ci/job_artifacts.rb +++ b/spec/factories/ci/job_artifacts.rb @@ -8,6 +8,10 @@ FactoryBot.define do file_type :archive file_format :zip + trait :expired do + expire_at { Date.yesterday } + end + trait :remote_store do file_store JobArtifactUploader::Store::REMOTE end diff --git a/spec/factories/deploy_tokens.rb b/spec/factories/deploy_tokens.rb index a96258f5cbe..99486acc2ab 100644 --- a/spec/factories/deploy_tokens.rb +++ b/spec/factories/deploy_tokens.rb @@ -2,7 +2,8 @@ FactoryBot.define do factory :deploy_token do - token { SecureRandom.hex(50) } + token nil + token_encrypted { Gitlab::CryptoHelper.aes256_gcm_encrypt( SecureRandom.hex(50) ) } sequence(:name) { |n| "PDT #{n}" } read_repository true read_registry true diff --git a/spec/features/dashboard/projects_spec.rb b/spec/features/dashboard/projects_spec.rb index e4728f37217..973d5a2dcfc 100644 --- a/spec/features/dashboard/projects_spec.rb +++ b/spec/features/dashboard/projects_spec.rb @@ -220,4 +220,26 @@ describe 'Dashboard Projects' do expect(find('input#merge_request_target_branch', visible: false).value).to eq 'master' end end + + it 'avoids an N+1 query in dashboard index' do + create(:ci_pipeline, :with_job, status: :success, project: project, ref: project.default_branch, sha: project.commit.sha) + visit dashboard_projects_path + + control_count = ActiveRecord::QueryRecorder.new { visit dashboard_projects_path }.count + + new_project = create(:project, :repository, name: 'new project') + create(:ci_pipeline, :with_job, status: :success, project: new_project, ref: new_project.commit.sha) + new_project.add_developer(user) + + ActiveRecord::QueryRecorder.new { visit dashboard_projects_path }.count + + # There are three known N+1 queries: + # 1. Project#open_issues_count + # 2. Project#open_merge_requests_count + # 3. Project#forks_count + # + # In addition, ProjectsHelper#load_pipeline_status also adds an + # additional query. + expect { visit dashboard_projects_path }.not_to exceed_query_limit(control_count + 4) + end end diff --git a/spec/features/oauth_login_spec.rb b/spec/features/oauth_login_spec.rb index a47eaa9bda7..c6e69fa3fb0 100644 --- a/spec/features/oauth_login_spec.rb +++ b/spec/features/oauth_login_spec.rb @@ -55,6 +55,18 @@ describe 'OAuth Login', :js, :allow_forgery_protection do expect(current_path).to eq root_path end + + it 'when bypass-two-factor is enabled' do + allow(Gitlab.config.omniauth).to receive_messages(allow_bypass_two_factor: true) + login_via(provider.to_s, user, uid, remember_me: false) + expect(current_path).to eq root_path + end + + it 'when bypass-two-factor is disabled' do + allow(Gitlab.config.omniauth).to receive_messages(allow_bypass_two_factor: false) + login_with_provider(provider, enter_two_factor: true) + expect(current_path).to eq root_path + end end context 'when "remember me" is checked' do diff --git a/spec/graphql/resolvers/echo_resolver_spec.rb b/spec/graphql/resolvers/echo_resolver_spec.rb new file mode 100644 index 00000000000..466501a4227 --- /dev/null +++ b/spec/graphql/resolvers/echo_resolver_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Resolvers::EchoResolver do + include GraphqlHelpers + + let(:current_user) { create(:user) } + let(:text) { 'Message test' } + + describe '#resolve' do + it 'echoes text and username' do + expect(resolve_echo(text)).to eq %Q("#{current_user.username}" says: #{text}) + end + + it 'echoes text and nil as username' do + expect(resolve_echo(text, { current_user: nil })).to eq "nil says: #{text}" + end + end + + def resolve_echo(text, context = { current_user: current_user }) + resolve(described_class, obj: nil, args: { text: text }, ctx: context) + end +end diff --git a/spec/javascripts/notes/stores/actions_spec.js b/spec/javascripts/notes/stores/actions_spec.js index e55aa0e965a..1fd4a9a7612 100644 --- a/spec/javascripts/notes/stores/actions_spec.js +++ b/spec/javascripts/notes/stores/actions_spec.js @@ -336,7 +336,7 @@ describe('Actions Notes Store', () => { }); }); - describe('deleteNote', () => { + describe('removeNote', () => { const endpoint = `${TEST_HOST}/note`; let axiosMock; @@ -357,7 +357,7 @@ describe('Actions Notes Store', () => { const note = { path: endpoint, id: 1 }; testAction( - actions.deleteNote, + actions.removeNote, note, store.state, [ @@ -384,7 +384,7 @@ describe('Actions Notes Store', () => { $('body').attr('data-page', 'projects:merge_requests:show'); testAction( - actions.deleteNote, + actions.removeNote, note, store.state, [ @@ -409,6 +409,45 @@ describe('Actions Notes Store', () => { }); }); + describe('deleteNote', () => { + const endpoint = `${TEST_HOST}/note`; + let axiosMock; + + beforeEach(() => { + axiosMock = new AxiosMockAdapter(axios); + axiosMock.onDelete(endpoint).replyOnce(200, {}); + + $('body').attr('data-page', ''); + }); + + afterEach(() => { + axiosMock.restore(); + + $('body').attr('data-page', ''); + }); + + it('dispatches removeNote', done => { + const note = { path: endpoint, id: 1 }; + + testAction( + actions.deleteNote, + note, + {}, + [], + [ + { + type: 'removeNote', + payload: { + id: 1, + path: 'http://test.host/note', + }, + }, + ], + done, + ); + }); + }); + describe('createNewNote', () => { describe('success', () => { const res = { diff --git a/spec/lib/gitlab/auth/o_auth/user_spec.rb b/spec/lib/gitlab/auth/o_auth/user_spec.rb index a9b15c411dc..1e3da4f7c2d 100644 --- a/spec/lib/gitlab/auth/o_auth/user_spec.rb +++ b/spec/lib/gitlab/auth/o_auth/user_spec.rb @@ -787,11 +787,25 @@ describe Gitlab::Auth::OAuth::User do end end - describe '#bypass_two_factor?' do - subject { oauth_user.bypass_two_factor? } + describe "#bypass_two_factor?" do + it "when with allow_bypass_two_factor disabled (Default)" do + stub_omniauth_config(allow_bypass_two_factor: false) + expect(oauth_user.bypass_two_factor?).to be_falsey + end + + it "when with allow_bypass_two_factor enabled" do + stub_omniauth_config(allow_bypass_two_factor: true) + expect(oauth_user.bypass_two_factor?).to be_truthy + end + + it "when provider in allow_bypass_two_factor array" do + stub_omniauth_config(allow_bypass_two_factor: [provider]) + expect(oauth_user.bypass_two_factor?).to be_truthy + end - it 'returns always false' do - is_expected.to be_falsey + it "when provider not in allow_bypass_two_factor array" do + stub_omniauth_config(allow_bypass_two_factor: ["foo"]) + expect(oauth_user.bypass_two_factor?).to be_falsey end end end diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index fe44b09aa24..cff4eb398bf 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -576,6 +576,38 @@ describe Gitlab::Database::MigrationHelpers do model.rename_column_concurrently(:users, :old, :new) end + + context 'when default is false' do + let(:old_column) do + double(:column, + type: :boolean, + limit: nil, + default: false, + null: false, + precision: nil, + scale: nil) + end + + it 'copies the default to the new column' do + expect(model).to receive(:change_column_default) + .with(:users, :new, old_column.default) + + model.rename_column_concurrently(:users, :old, :new) + end + end + end + end + + describe '#undo_rename_column_concurrently' do + it 'reverses the operations of rename_column_concurrently' do + expect(model).to receive(:check_trigger_permissions!).with(:users) + + expect(model).to receive(:remove_rename_triggers_for_postgresql) + .with(:users, /trigger_.{12}/) + + expect(model).to receive(:remove_column).with(:users, :new) + + model.undo_rename_column_concurrently(:users, :old, :new) end end @@ -592,6 +624,80 @@ describe Gitlab::Database::MigrationHelpers do end end + describe '#undo_cleanup_concurrent_column_rename' do + context 'in a transaction' do + it 'raises RuntimeError' do + allow(model).to receive(:transaction_open?).and_return(true) + + expect { model.undo_cleanup_concurrent_column_rename(:users, :old, :new) } + .to raise_error(RuntimeError) + end + end + + context 'outside a transaction' do + let(:new_column) do + double(:column, + type: :integer, + limit: 8, + default: 0, + null: false, + precision: 5, + scale: 1) + end + + let(:trigger_name) { model.rename_trigger_name(:users, :old, :new) } + + before do + allow(model).to receive(:transaction_open?).and_return(false) + allow(model).to receive(:column_for).and_return(new_column) + end + + it 'reverses the operations of cleanup_concurrent_column_rename' do + expect(model).to receive(:check_trigger_permissions!).with(:users) + + expect(model).to receive(:install_rename_triggers_for_postgresql) + .with(trigger_name, '"users"', '"old"', '"new"') + + expect(model).to receive(:add_column) + .with(:users, :old, :integer, + limit: new_column.limit, + precision: new_column.precision, + scale: new_column.scale) + + expect(model).to receive(:change_column_default) + .with(:users, :old, new_column.default) + + expect(model).to receive(:update_column_in_batches) + + expect(model).to receive(:change_column_null).with(:users, :old, false) + + expect(model).to receive(:copy_indexes).with(:users, :new, :old) + expect(model).to receive(:copy_foreign_keys).with(:users, :new, :old) + + model.undo_cleanup_concurrent_column_rename(:users, :old, :new) + end + + context 'when default is false' do + let(:new_column) do + double(:column, + type: :boolean, + limit: nil, + default: false, + null: false, + precision: nil, + scale: nil) + end + + it 'copies the default to the old column' do + expect(model).to receive(:change_column_default) + .with(:users, :old, new_column.default) + + model.undo_cleanup_concurrent_column_rename(:users, :old, :new) + end + end + end + end + describe '#change_column_type_concurrently' do it 'changes the column type' do expect(model).to receive(:rename_column_concurrently) diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index 77e58b6d5c7..8d37de32179 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -347,6 +347,17 @@ describe Gitlab::Database do pool.disconnect! end end + + it 'allows setting of a custom hostname and port' do + pool = described_class.create_connection_pool(5, '127.0.0.1', 5432) + + begin + expect(pool.spec.config[:host]).to eq('127.0.0.1') + expect(pool.spec.config[:port]).to eq(5432) + ensure + pool.disconnect! + end + end end describe '.cached_column_exists?' do diff --git a/spec/lib/gitlab/fogbugz_import/project_creator_spec.rb b/spec/lib/gitlab/fogbugz_import/project_creator_spec.rb new file mode 100644 index 00000000000..503fe897e29 --- /dev/null +++ b/spec/lib/gitlab/fogbugz_import/project_creator_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::FogbugzImport::ProjectCreator do + let(:user) { create(:user) } + + let(:repo) do + instance_double(Gitlab::FogbugzImport::Repository, + name: 'Vim', + safe_name: 'vim', + path: 'vim', + raw_data: '') + end + + let(:uri) { 'https://testing.fogbugz.com' } + let(:token) { 'token' } + let(:fb_session) { { uri: uri, token: token } } + let(:project_creator) { described_class.new(repo, fb_session, user.namespace, user) } + + subject do + project_creator.execute + end + + it 'creates project with private visibility level' do + expect(subject.persisted?).to eq(true) + expect(subject.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end +end diff --git a/spec/lib/gitlab/graphql/authorize/authorize_field_service_spec.rb b/spec/lib/gitlab/graphql/authorize/authorize_field_service_spec.rb index d60d1b7559a..7a7ae373058 100644 --- a/spec/lib/gitlab/graphql/authorize/authorize_field_service_spec.rb +++ b/spec/lib/gitlab/graphql/authorize/authorize_field_service_spec.rb @@ -30,7 +30,10 @@ describe Gitlab::Graphql::Authorize::AuthorizeFieldService do describe '#authorized_resolve' do let(:presented_object) { double('presented object') } let(:presented_type) { double('parent type', object: presented_object) } - subject(:resolved) { service.authorized_resolve.call(presented_type, {}, { current_user: current_user }) } + let(:query_type) { GraphQL::ObjectType.new } + let(:schema) { GraphQL::Schema.define(query: query_type, mutation: nil)} + let(:context) { GraphQL::Query::Context.new(query: OpenStruct.new(schema: schema), values: { current_user: current_user }, object: nil) } + subject(:resolved) { service.authorized_resolve.call(presented_type, {}, context) } context 'scalar types' do shared_examples 'checking permissions on the presented object' do diff --git a/spec/lib/gitlab/graphql/markdown_field_spec.rb b/spec/lib/gitlab/graphql/markdown_field_spec.rb index a8566aa8e1c..866a20801d3 100644 --- a/spec/lib/gitlab/graphql/markdown_field_spec.rb +++ b/spec/lib/gitlab/graphql/markdown_field_spec.rb @@ -30,17 +30,20 @@ describe Gitlab::Graphql::MarkdownField do let(:note) { build(:note, note: '# Markdown!') } let(:thing_with_markdown) { double('markdown thing', object: note) } let(:expected_markdown) { '<h1 data-sourcepos="1:1-1:11" dir="auto">Markdown!</h1>' } + let(:query_type) { GraphQL::ObjectType.new } + let(:schema) { GraphQL::Schema.define(query: query_type, mutation: nil)} + let(:context) { GraphQL::Query::Context.new(query: OpenStruct.new(schema: schema), values: nil, object: nil) } it 'renders markdown from the same property as the field name without the `_html` suffix' do field = class_with_markdown_field(:note_html, null: false).fields['noteHtml'] - expect(field.to_graphql.resolve(thing_with_markdown, {}, {})).to eq(expected_markdown) + expect(field.to_graphql.resolve(thing_with_markdown, {}, context)).to eq(expected_markdown) end it 'renders markdown from a specific property when a `method` argument is passed' do field = class_with_markdown_field(:test_html, null: false, method: :note).fields['testHtml'] - expect(field.to_graphql.resolve(thing_with_markdown, {}, {})).to eq(expected_markdown) + expect(field.to_graphql.resolve(thing_with_markdown, {}, context)).to eq(expected_markdown) end end end diff --git a/spec/lib/gitlab/usage_data_counters/merge_request_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/merge_request_counter_spec.rb new file mode 100644 index 00000000000..4be4a661260 --- /dev/null +++ b/spec/lib/gitlab/usage_data_counters/merge_request_counter_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::UsageDataCounters::MergeRequestCounter do + it_behaves_like 'a redis usage counter', 'Merge Request', :create + + it_behaves_like 'a redis usage counter with totals', :merge_request, create: 5 +end diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index dda119e09b1..b3a179e276b 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -69,6 +69,7 @@ describe Gitlab::UsageData do snippet_update: a_kind_of(Integer), snippet_comment: a_kind_of(Integer), merge_request_comment: a_kind_of(Integer), + merge_request_create: a_kind_of(Integer), commit_comment: a_kind_of(Integer), wiki_pages_create: a_kind_of(Integer), wiki_pages_update: a_kind_of(Integer), diff --git a/spec/migrations/encrypt_deploy_tokens_tokens_spec.rb b/spec/migrations/encrypt_deploy_tokens_tokens_spec.rb new file mode 100644 index 00000000000..a398e079731 --- /dev/null +++ b/spec/migrations/encrypt_deploy_tokens_tokens_spec.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require Rails.root.join('db', 'post_migrate', '20190711201818_encrypt_deploy_tokens_tokens.rb') + +describe EncryptDeployTokensTokens, :migration do + let(:migration) { described_class.new } + let(:deployment_tokens) { table(:deploy_tokens) } + let(:plaintext) { "secret-token" } + let(:expires_at) { DateTime.now + 1.year } + let(:ciphertext) { Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext) } + + describe '#up' do + it 'keeps plaintext token the same and populates token_encrypted if not present' do + deploy_token = deployment_tokens.create!( + name: 'test_token', + read_repository: true, + expires_at: expires_at, + username: 'gitlab-token-1', + token: plaintext + ) + + migration.up + + expect(deploy_token.reload.token).to eq(plaintext) + expect(deploy_token.reload.token_encrypted).to eq(ciphertext) + end + end + + describe '#down' do + it 'decrypts encrypted token and saves it' do + deploy_token = deployment_tokens.create!( + name: 'test_token', + read_repository: true, + expires_at: expires_at, + username: 'gitlab-token-1', + token_encrypted: ciphertext + ) + + migration.down + + expect(deploy_token.reload.token).to eq(plaintext) + expect(deploy_token.reload.token_encrypted).to eq(ciphertext) + end + end +end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index ff9e94afc12..bd352db2236 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -4877,35 +4877,22 @@ describe Project do describe '#git_objects_poolable?' do subject { project } - - context 'when the feature flag is turned off' do - before do - stub_feature_flags(object_pools: false) - end - - let(:project) { create(:project, :repository, :public) } + context 'when not using hashed storage' do + let(:project) { create(:project, :legacy_storage, :public, :repository) } it { is_expected.not_to be_git_objects_poolable } end - context 'when the feature flag is enabled' do - context 'when not using hashed storage' do - let(:project) { create(:project, :legacy_storage, :public, :repository) } - - it { is_expected.not_to be_git_objects_poolable } - end + context 'when the project is not public' do + let(:project) { create(:project, :private) } - context 'when the project is not public' do - let(:project) { create(:project, :private) } - - it { is_expected.not_to be_git_objects_poolable } - end + it { is_expected.not_to be_git_objects_poolable } + end - context 'when objects are poolable' do - let(:project) { create(:project, :repository, :public) } + context 'when objects are poolable' do + let(:project) { create(:project, :repository, :public) } - it { is_expected.to be_git_objects_poolable } - end + it { is_expected.to be_git_objects_poolable } end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 46b86e8393d..8338d2b5b39 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -103,6 +103,14 @@ describe User do it { is_expected.to validate_length_of(:name).is_at_most(128) } end + describe 'first name' do + it { is_expected.to validate_length_of(:first_name).is_at_most(255) } + end + + describe 'last name' do + it { is_expected.to validate_length_of(:last_name).is_at_most(255) } + end + describe 'username' do it 'validates presence' do expect(subject).to validate_presence_of(:username) @@ -678,6 +686,18 @@ describe User do end end + describe 'name getters' do + let(:user) { create(:user, name: 'Kane Martin William') } + + it 'derives first name from full name, if not present' do + expect(user.first_name).to eq('Kane') + end + + it 'derives last name from full name, if not present' do + expect(user.last_name).to eq('Martin William') + end + end + describe '#highest_role' do let(:user) { create(:user) } diff --git a/spec/policies/project_policy_spec.rb b/spec/policies/project_policy_spec.rb index 8fd54e0bf1d..71ba73d5661 100644 --- a/spec/policies/project_policy_spec.rb +++ b/spec/policies/project_policy_spec.rb @@ -94,6 +94,19 @@ describe ProjectPolicy do permissions.each { |p| is_expected.not_to be_allowed(p) } end + context 'with no project feature' do + subject { described_class.new(owner, project) } + + before do + project.project_feature.destroy + project.reload + end + + it 'returns false' do + is_expected.to be_disallowed(:read_build) + end + end + it 'does not include the read_issue permission when the issue author is not a member of the private project' do project = create(:project, :private) issue = create(:issue, project: project, author: create(:user)) diff --git a/spec/requests/api/graphql/multiplexed_queries_spec.rb b/spec/requests/api/graphql/multiplexed_queries_spec.rb index 844fd979285..9ebb57f6b9c 100644 --- a/spec/requests/api/graphql/multiplexed_queries_spec.rb +++ b/spec/requests/api/graphql/multiplexed_queries_spec.rb @@ -6,9 +6,9 @@ describe 'Multiplexed queries' do it 'returns responses for multiple queries' do queries = [ - { query: 'query($text: String) { echo(text: $text) }', + { query: 'query($text: String!) { echo(text: $text) }', variables: { 'text' => 'Hello' } }, - { query: 'query($text: String) { echo(text: $text) }', + { query: 'query($text: String!) { echo(text: $text) }', variables: { 'text' => 'World' } } ] @@ -23,8 +23,8 @@ describe 'Multiplexed queries' do it 'returns error and data combinations' do queries = [ - { query: 'query($text: String) { broken query }' }, - { query: 'query working($text: String) { echo(text: $text) }', + { query: 'query($text: String!) { broken query }' }, + { query: 'query working($text: String!) { echo(text: $text) }', variables: { 'text' => 'World' } } ] diff --git a/spec/requests/api/issues/get_project_issues_spec.rb b/spec/requests/api/issues/get_project_issues_spec.rb index 521d6b88734..b7aa3f93451 100644 --- a/spec/requests/api/issues/get_project_issues_spec.rb +++ b/spec/requests/api/issues/get_project_issues_spec.rb @@ -446,6 +446,14 @@ describe API::Issues do expect_paginated_array_response([closed_issue.id, confidential_issue.id, issue.id]) end + it 'exposes known attributes' do + get api("#{base_url}/issues", user) + + expect(response).to have_gitlab_http_status(200) + expect(json_response.last.keys).to include(*%w(id iid project_id title description)) + expect(json_response.last).not_to have_key('subscribed') + end + context 'issues_statistics' do context 'no state is treated as all state' do let(:params) { {} } diff --git a/spec/requests/api/labels_spec.rb b/spec/requests/api/labels_spec.rb index f6640fe41d0..9aef67e28a7 100644 --- a/spec/requests/api/labels_spec.rb +++ b/spec/requests/api/labels_spec.rb @@ -473,6 +473,55 @@ describe API::Labels do end end + describe 'PUT /projects/:id/labels/promote' do + let(:group) { create(:group) } + + before do + group.add_owner(user) + project.update!(group: group) + end + + it 'returns 200 if label is promoted' do + put api("/projects/#{project.id}/labels/promote", user), params: { name: label1.name } + + expect(response).to have_gitlab_http_status(200) + expect(json_response['name']).to eq(label1.name) + expect(json_response['color']).to eq(label1.color) + end + + it 'returns 200 if group label already exists' do + create(:group_label, title: label1.name, group: group) + + expect { put api("/projects/#{project.id}/labels/promote", user), params: { name: label1.name } } + .to change(project.labels, :count).by(-1) + .and change(group.labels, :count).by(0) + + expect(response).to have_gitlab_http_status(200) + end + + it 'returns 403 if guest promotes label' do + guest = create(:user) + project.add_guest(guest) + + put api("/projects/#{project.id}/labels/promote", guest), params: { name: label1.name } + + expect(response).to have_gitlab_http_status(403) + end + + it 'returns 404 if label does not exist' do + put api("/projects/#{project.id}/labels/promote", user), params: { name: 'unknown' } + + expect(response).to have_gitlab_http_status(404) + end + + it 'returns 400 if no label name given' do + put api("/projects/#{project.id}/labels/promote", user) + + expect(response).to have_gitlab_http_status(400) + expect(json_response['error']).to eq('name is missing') + end + end + describe "POST /projects/:id/labels/:label_id/subscribe" do context "when label_id is a label title" do it "subscribes to the label" do diff --git a/spec/rubocop/cop/gitlab/union_spec.rb b/spec/rubocop/cop/gitlab/union_spec.rb index 5b06f30b25f..f0544fdb66e 100644 --- a/spec/rubocop/cop/gitlab/union_spec.rb +++ b/spec/rubocop/cop/gitlab/union_spec.rb @@ -16,10 +16,4 @@ describe RuboCop::Cop::Gitlab::Union do ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Use the `FromUnion` concern, instead of using `Gitlab::SQL::Union` directly SOURCE end - - it 'does not flag the use of Gitlab::SQL::Union in a spec' do - allow(cop).to receive(:in_spec?).and_return(true) - - expect_no_offenses('Gitlab::SQL::Union.new([foo])') - end end diff --git a/spec/rubocop/cop/rspec/env_assignment_spec.rb b/spec/rubocop/cop/rspec/env_assignment_spec.rb index 659633f6467..621afbad3ba 100644 --- a/spec/rubocop/cop/rspec/env_assignment_spec.rb +++ b/spec/rubocop/cop/rspec/env_assignment_spec.rb @@ -33,27 +33,13 @@ describe RuboCop::Cop::RSpec::EnvAssignment do end end - context 'in a spec file' do - before do - allow(cop).to receive(:in_spec?).and_return(true) - end - - context 'with a key using single quotes' do - it_behaves_like 'an offensive ENV#[]= call', OFFENSE_CALL_SINGLE_QUOTES_KEY - it_behaves_like 'an autocorrected ENV#[]= call', OFFENSE_CALL_SINGLE_QUOTES_KEY, %(stub_env('FOO', 'bar')) - end - - context 'with a key using double quotes' do - it_behaves_like 'an offensive ENV#[]= call', OFFENSE_CALL_DOUBLE_QUOTES_KEY - it_behaves_like 'an autocorrected ENV#[]= call', OFFENSE_CALL_DOUBLE_QUOTES_KEY, %(stub_env("FOO", 'bar')) - end + context 'with a key using single quotes' do + it_behaves_like 'an offensive ENV#[]= call', OFFENSE_CALL_SINGLE_QUOTES_KEY + it_behaves_like 'an autocorrected ENV#[]= call', OFFENSE_CALL_SINGLE_QUOTES_KEY, %(stub_env('FOO', 'bar')) end - context 'outside of a spec file' do - it "does not register an offense for `#{OFFENSE_CALL_SINGLE_QUOTES_KEY}` in a non-spec file" do - inspect_source(OFFENSE_CALL_SINGLE_QUOTES_KEY) - - expect(cop.offenses.size).to eq(0) - end + context 'with a key using double quotes' do + it_behaves_like 'an offensive ENV#[]= call', OFFENSE_CALL_DOUBLE_QUOTES_KEY + it_behaves_like 'an autocorrected ENV#[]= call', OFFENSE_CALL_DOUBLE_QUOTES_KEY, %(stub_env("FOO", 'bar')) end end diff --git a/spec/rubocop/cop/rspec/factories_in_migration_specs_spec.rb b/spec/rubocop/cop/rspec/factories_in_migration_specs_spec.rb index 2763f2bda21..94324bc615d 100644 --- a/spec/rubocop/cop/rspec/factories_in_migration_specs_spec.rb +++ b/spec/rubocop/cop/rspec/factories_in_migration_specs_spec.rb @@ -8,8 +8,6 @@ require_relative '../../../../rubocop/cop/rspec/factories_in_migration_specs' describe RuboCop::Cop::RSpec::FactoriesInMigrationSpecs do include CopHelper - let(:source_file) { 'spec/migrations/foo_spec.rb' } - subject(:cop) { described_class.new } shared_examples 'an offensive factory call' do |namespace| @@ -27,22 +25,6 @@ describe RuboCop::Cop::RSpec::FactoriesInMigrationSpecs do end end - context 'in a migration spec file' do - before do - allow(cop).to receive(:in_migration_spec?).and_return(true) - end - - it_behaves_like 'an offensive factory call', '' - it_behaves_like 'an offensive factory call', 'FactoryBot.' - end - - context 'outside of a migration spec file' do - it "does not register an offense" do - expect_no_offenses(<<-RUBY) - describe 'foo' do - let(:user) { create(:user) } - end - RUBY - end - end + it_behaves_like 'an offensive factory call', '' + it_behaves_like 'an offensive factory call', 'FactoryBot.' end diff --git a/spec/services/application_settings/update_service_spec.rb b/spec/services/application_settings/update_service_spec.rb index adb5219d691..ab06c1a1209 100644 --- a/spec/services/application_settings/update_service_spec.rb +++ b/spec/services/application_settings/update_service_spec.rb @@ -201,6 +201,24 @@ describe ApplicationSettings::UpdateService do enable_external_authorization_service_check end + it 'does not validate labels if external authorization gets disabled' do + expect_any_instance_of(described_class).not_to receive(:validate_classification_label) + + described_class.new(application_settings, admin, { external_authorization_service_enabled: false }).execute + end + + it 'does validate labels if external authorization gets enabled ' do + expect_any_instance_of(described_class).to receive(:validate_classification_label) + + described_class.new(application_settings, admin, { external_authorization_service_enabled: true }).execute + end + + it 'does validate labels if external authorization is left unchanged' do + expect_any_instance_of(described_class).to receive(:validate_classification_label) + + described_class.new(application_settings, admin, { external_authorization_service_default_label: 'new-label' }).execute + end + it 'does not save the settings with an error if the service denies access' do expect(::Gitlab::ExternalAuthorization) .to receive(:access_allowed?).with(admin, 'new-label') { false } diff --git a/spec/services/merge_requests/create_service_spec.rb b/spec/services/merge_requests/create_service_spec.rb index ed48f4b1e44..699f2a98088 100644 --- a/spec/services/merge_requests/create_service_spec.rb +++ b/spec/services/merge_requests/create_service_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -describe MergeRequests::CreateService do +describe MergeRequests::CreateService, :clean_gitlab_redis_shared_state do include ProjectForksHelper let(:project) { create(:project, :repository) } @@ -285,6 +285,12 @@ describe MergeRequests::CreateService do end end end + + it 'increments the usage data counter of create event' do + counter = Gitlab::UsageDataCounters::MergeRequestCounter + + expect { service.execute }.to change { counter.read(:create) }.by(1) + end end it_behaves_like 'new issuable record that supports quick actions' do diff --git a/spec/support/shared_examples/quick_actions/issue/move_quick_action_shared_examples.rb b/spec/support/shared_examples/quick_actions/issue/move_quick_action_shared_examples.rb index a37b2392d52..bebc8509d53 100644 --- a/spec/support/shared_examples/quick_actions/issue/move_quick_action_shared_examples.rb +++ b/spec/support/shared_examples/quick_actions/issue/move_quick_action_shared_examples.rb @@ -89,5 +89,54 @@ shared_examples 'move quick action' do it_behaves_like 'applies the commands to issues in both projects, target and source' end end + + context 'when editing comments' do + let(:target_project) { create(:project, :public) } + + before do + target_project.add_maintainer(user) + + sign_in(user) + visit project_issue_path(project, issue) + wait_for_all_requests + end + + it 'moves the issue after quickcommand note was updated' do + # misspelled quick action + add_note("test note.\n/mvoe #{target_project.full_path}") + + expect(issue.reload).not_to be_closed + + edit_note("/mvoe #{target_project.full_path}", "test note.\n/move #{target_project.full_path}") + wait_for_all_requests + + expect(page).to have_content 'test note.' + expect(issue.reload).to be_closed + + visit project_issue_path(target_project, issue) + wait_for_all_requests + + expect(page).to have_content 'Issues 1' + end + + it 'deletes the note if it was updated to just contain a command' do + # missspelled quick action + add_note("test note.\n/mvoe #{target_project.full_path}") + + expect(page).not_to have_content 'Commands applied' + expect(issue.reload).not_to be_closed + + edit_note("/mvoe #{target_project.full_path}", "/move #{target_project.full_path}") + wait_for_all_requests + + expect(page).not_to have_content "/move #{target_project.full_path}" + expect(issue.reload).to be_closed + + visit project_issue_path(target_project, issue) + wait_for_all_requests + + expect(page).to have_content 'Issues 1' + end + end end end diff --git a/spec/views/layouts/_head.html.haml_spec.rb b/spec/views/layouts/_head.html.haml_spec.rb index 70cdc08b4b6..d7f24950e6f 100644 --- a/spec/views/layouts/_head.html.haml_spec.rb +++ b/spec/views/layouts/_head.html.haml_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' describe 'layouts/_head' do + include StubConfiguration + before do allow(view).to receive(:current_application_settings).and_return(Gitlab::CurrentSettings.current_application_settings) end @@ -87,6 +89,24 @@ describe 'layouts/_head' do end end + context 'when a Piwik config is set' do + let(:piwik_host) { 'piwik.example.com' } + + before do + stub_config(extra: { + piwik_url: piwik_host, + piwik_site_id: 12345 + }) + end + + it 'add a Piwik Javascript' do + render + + expect(rendered).to match(/<script.*>.*var u="\/\/#{piwik_host}\/".*<\/script>/m) + expect(rendered).to match(%r(<noscript>.*<img src="//#{piwik_host}/piwik.php.*</noscript>)) + end + end + def stub_helper_with_safe_string(method) allow_any_instance_of(PageLayoutHelper).to receive(method) .and_return(%q{foo" http-equiv="refresh}.html_safe) diff --git a/yarn.lock b/yarn.lock index dd85729333a..712cdace0d7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -998,15 +998,15 @@ dependencies: vue-eslint-parser "^6.0.4" -"@gitlab/svgs@^1.68.0": - version "1.68.0" - resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-1.68.0.tgz#d631bd84ea7907592240d8417e82ba66d6a54c0c" - integrity sha512-3JmIq0bHg4InjLooM+kQFPfg3d7B1Pye67pN9+12kZXIa0nRGuwKEq3WSbcS+ACdg5jcVPNPYqStItEO4teHdw== - -"@gitlab/ui@5.18.0": - version "5.18.0" - resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-5.18.0.tgz#f7f93ad41673f5ae081cd2246dda5456d3c956a2" - integrity sha512-umB7JCKVDZOajed0N563JBYSpiyK+VDL2eCdkinW+twWhMct8onoW4CLTpkgBi6Z9Y1Bq47aqnFtGf+1IKNIGw== +"@gitlab/svgs@^1.70.0": + version "1.70.0" + resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-1.70.0.tgz#bdae478148c15d955fc06e69fd5d5ecae8298943" + integrity sha512-0uV9fgTwe17Fyy0hTcrsGX2jJuCrz3uRIe8yffuqc6pbQrSfYJyN66mfCCB45wq8lKTgOB5q0qcUyJx3RQEcKg== + +"@gitlab/ui@5.19.0": + version "5.19.0" + resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-5.19.0.tgz#ef5431e0e91eb4009a30edcf49a40716cc2570d9" + integrity sha512-gCeTymtVzzO2uOWZGIweLM5JcHq3TN1QwP61CXw7b9Lp5A2MysRb8upehtR5d4JLOf/GQg8rHQB26uRvUc+AXQ== dependencies: "@babel/standalone" "^7.0.0" "@gitlab/vue-toasted" "^1.2.1" |
