diff options
51 files changed, 957 insertions, 689 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f926cbc2939..4c7a8c05b37 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,15 @@ image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-9.6-graphicsmagick-1.3.33" +stages: + - build + - prepare + - quick-test + - test + - review + - qa + - post-test + - pages + variables: RAILS_ENV: "test" NODE_ENV: "test" @@ -11,25 +21,9 @@ variables: FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json BUILD_ASSETS_IMAGE: "false" -before_script: - - date - - source scripts/utils.sh - - source scripts/prepare_build.sh - - date - after_script: - date -stages: - - build - - prepare - - merge - - test - - review - - qa - - post-test - - pages - include: - local: .gitlab/ci/global.gitlab-ci.yml - local: .gitlab/ci/cng.gitlab-ci.yml diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index dae3c349ff4..0f2dd081e9e 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -9,12 +9,13 @@ app/assets/ @ClemMakesApps @fatihacet @filipa @mikegreiling @timzallmann @kushalpandya @pslaughter *.scss @annabeldunstone @ClemMakesApps @fatihacet @filipa @mikegreiling @timzallmann @kushalpandya @pslaughter -# Maintainers from the Database team should review changes in `db/` -db/ @gl-database -lib/gitlab/background_migration/ @gl-database -lib/gitlab/database/ @gl-database -lib/gitlab/sql/ @gl-database -/ee/db/ @gl-database +# Database maintainers should review changes in `db/` +db/ @gitlab-org/maintainers/database +lib/gitlab/background_migration/ @gitlab-org/maintainers/database +lib/gitlab/database/ @gitlab-org/maintainers/database +lib/gitlab/sql/ @gitlab-org/maintainers/database +lib/gitlab/github_import/ @gitlab-org/maintainers/database +/ee/db/ @gitlab-org/maintainers/database # Feature specific owners /ee/lib/gitlab/code_owners/ @reprazent diff --git a/.gitlab/ci/cng.gitlab-ci.yml b/.gitlab/ci/cng.gitlab-ci.yml index d624e8d09f6..a43d3694103 100644 --- a/.gitlab/ci/cng.gitlab-ci.yml +++ b/.gitlab/ci/cng.gitlab-ci.yml @@ -1,16 +1,15 @@ cloud-native-image: image: ruby:2.6-alpine - before_script: [] dependencies: [] stage: post-test allow_failure: true variables: GIT_DEPTH: "1" - cache: {} when: manual script: - install_gitlab_gem - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng only: - - tags@gitlab-org/gitlab-ce - - tags@gitlab-org/gitlab-ee + refs: + - tags@gitlab-org/gitlab-ce + - tags@gitlab-org/gitlab-ee diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index d724ab07663..1fad731543c 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -1,24 +1,34 @@ -.review-docs: &review-docs - extends: .single-script-job-dedicated-runner +.review-docs: + extends: + - .default-tags + - .default-retry + image: ruby:2.6-alpine + stage: review + dependencies: [] variables: - SCRIPT_NAME: trigger-build-docs + GIT_STRATEGY: none environment: name: review-docs/$CI_COMMIT_REF_SLUG # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are CI variables # Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693 url: http://$CI_ENVIRONMENT_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX on_stop: review-docs-cleanup + before_script: + # We don't clone the repo by using GIT_STRATEGY: none and only download the + # single script we need here so it's much faster than cloning. + - apk add --update openssl + - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/trigger-build-docs + - chmod 755 trigger-build-docs # Trigger a manual docs build in gitlab-docs only on non docs-only branches. # Useful to preview the docs changes live. review-docs-deploy-manual: extends: - .review-docs - - .no-docs-and-no-qa - stage: review + - .except-docs-qa script: - gem install gitlab --no-document - - ./$SCRIPT_NAME deploy + - ./trigger-build-docs deploy when: manual only: - branches@gitlab-org/gitlab-ce @@ -27,39 +37,40 @@ review-docs-deploy-manual: # Always trigger a docs build in gitlab-docs only on docs-only branches. # Useful to preview the docs changes live. review-docs-deploy: - <<: *review-docs - stage: review + extends: + - .review-docs + - .except-qa script: - gem install gitlab --no-document - - ./$SCRIPT_NAME deploy + - ./trigger-build-docs deploy only: - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-ce - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-ee - except: - - /(^qa[\/-].*|.*-qa$)/ # Cleanup remote environment of gitlab-docs review-docs-cleanup: - <<: *review-docs - stage: review + extends: + - .review-docs + - .except-qa environment: name: review-docs/$CI_COMMIT_REF_SLUG action: stop script: - gem install gitlab --no-document - - ./$SCRIPT_NAME cleanup + - ./trigger-build-docs cleanup when: manual only: - branches@gitlab-org/gitlab-ce - branches@gitlab-org/gitlab-ee docs lint: - extends: .dedicated-runner + extends: + - .default-tags + - .default-retry + - .except-qa image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint" stage: test - cache: {} dependencies: [] - before_script: [] script: - scripts/lint-doc.sh - mv doc/ /tmp/gitlab-docs/content/$DOCS_GITLAB_REPO_SUFFIX @@ -72,5 +83,3 @@ docs lint: - bundle exec nanoc check internal_links # Check the internal anchor links - bundle exec nanoc check internal_anchors - except: - - /(^qa[\/-].*|.*-qa$)/ diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 5c3278fcf53..df38cb4ff8e 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -1,23 +1,19 @@ -.assets-compile-cache: &assets-compile-cache +.assets-compile-cache: cache: - key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6" paths: - vendor/ruby/ - .yarn-cache/ - tmp/cache/assets/sprockets -.use-pg: &use-pg - services: - - name: postgres:9.6.14 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - .gitlab:assets:compile-metadata: - <<: *assets-compile-cache - extends: .dedicated-no-docs-pull-cache-job + extends: + - .default-tags + - .default-retry + - .assets-compile-cache + - .default-before_script + - .except-docs image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1 - dependencies: - - setup-test-env + dependencies: ["setup-test-env"] services: - docker:19.03.0-dind variables: @@ -30,6 +26,14 @@ NODE_OPTIONS: --max_old_space_size=3584 DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 + cache: + key: "assets-compile:production:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6" + artifacts: + name: webpack-report + expire_in: 31d + paths: + - webpack-report/ + - public/assets/ script: - node --version - retry yarn install --frozen-lockfile --production --cache-folder .yarn-cache --prefer-offline @@ -42,43 +46,41 @@ - install_api_client_dependencies_with_apt - play_job "review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played - play_job "schedule:review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played - artifacts: - name: webpack-report - expire_in: 31d - paths: - - webpack-report/ - - public/assets/ only: - /.+/@gitlab-org/gitlab-ce - /.+/@gitlab-org/gitlab-ee - /.+/@gitlab/gitlabhq - /.+/@gitlab/gitlab-ee tags: - - docker - gitlab-org + - docker gitlab:assets:compile: extends: .gitlab:assets:compile-metadata + only: + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee cache: policy: pull-push - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee gitlab:assets:compile pull-cache: extends: .gitlab:assets:compile-metadata - cache: - policy: pull except: refs: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee - /(^docs[\/-].+|.+-docs$)/ + cache: + policy: pull .compile-assets-metadata: - extends: .dedicated-runner - <<: *use-pg - <<: *assets-compile-cache + extends: + - .default-tags + - .default-retry + - .assets-compile-cache + - .default-before_script + - .use-pg stage: prepare script: - node --version @@ -89,6 +91,8 @@ gitlab:assets:compile pull-cache: variables: # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 + cache: + key: "assets-compile:test:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6" artifacts: expire_in: 7d paths: @@ -96,30 +100,34 @@ gitlab:assets:compile pull-cache: - public/assets compile-assets: - extends: .compile-assets-metadata + extends: + - .compile-assets-metadata + only: + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee cache: policy: pull-push - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee compile-assets pull-cache: extends: .compile-assets-metadata - cache: - policy: pull except: refs: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee - /(^docs[\/-].+|.+-docs$)/ + cache: + policy: pull karma: - extends: .dedicated-no-docs-pull-cache-job - <<: *use-pg - dependencies: - - compile-assets - - compile-assets pull-cache - - setup-test-env + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs + dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"] variables: # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 @@ -142,12 +150,14 @@ karma: junit: junit_karma.xml jest: - extends: .dedicated-no-docs-and-no-qa-pull-cache-job - <<: *use-pg - dependencies: - - compile-assets - - compile-assets pull-cache - - setup-test-env + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"] script: - scripts/gitaly-test-spawn - date @@ -170,36 +180,41 @@ jest: - tmp/jest/jest/ policy: pull-push -qa:internal: - extends: .dedicated-no-docs-no-db-pull-cache-job - services: [] - script: +.qa: + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + dependencies: ["setup-test-env"] + variables: + SETUP_DB: "false" + before_script: - cd qa/ - bundle install + +qa:internal: + extends: .qa + script: - bundle exec rspec - dependencies: - - setup-test-env qa:selectors: - extends: .dedicated-no-docs-no-db-pull-cache-job - services: [] + extends: .qa script: - - cd qa/ - - bundle install - bundle exec bin/qa Test::Sanity::Selectors - dependencies: - - setup-test-env -.qa-frontend-node: &qa-frontend-node - extends: .dedicated-no-docs-no-db-pull-cache-job - stage: test +.qa-frontend-node: + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + dependencies: [] cache: key: "$CI_JOB_NAME" paths: - .yarn-cache/ policy: pull-push - dependencies: [] - before_script: [] script: - date - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline @@ -207,23 +222,28 @@ qa:selectors: - yarn run webpack-prod qa-frontend-node:8: - <<: *qa-frontend-node + extends: .qa-frontend-node image: node:carbon qa-frontend-node:10: - <<: *qa-frontend-node + extends: .qa-frontend-node image: node:dubnium qa-frontend-node:latest: - <<: *qa-frontend-node + extends: .qa-frontend-node image: node:latest allow_failure: true lint:javascript:report: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + variables: + SETUP_DB: "false" stage: post-test dependencies: [] - before_script: [] script: - date - yarn run eslint-report || true # ignore exit code @@ -234,12 +254,15 @@ lint:javascript:report: - eslint-report.html jsdoc: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + variables: + SETUP_DB: "false" stage: post-test - dependencies: - - compile-assets - - compile-assets pull-cache - before_script: [] + dependencies: ["compile-assets", "compile-assets pull-cache"] script: - date - yarn run jsdoc || true # ignore exit code diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 56fe9739d5f..04135447ca4 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -1,83 +1,56 @@ -.dedicated-runner: +.default-tags: + tags: + - gitlab-org + +.default-retry: retry: max: 2 # This is confusing but this means "3 runs at max". when: - unknown_failure - api_failure - runner_system_failure - tags: - - gitlab-org - -.default-cache: &default-cache - key: "debian-stretch-ruby-2.6.3-node-12.x" - paths: - - vendor/ruby - - .yarn-cache/ - - vendor/gitaly-ruby -.dedicated-runner-default-cache: - extends: .dedicated-runner - cache: - <<: *default-cache +.default-before_script: + before_script: + - date + - source scripts/utils.sh + - source scripts/prepare_build.sh + - date # Jobs that only need to pull cache -.dedicated-pull-cache-job: - extends: .dedicated-runner +.default-cache: cache: - <<: *default-cache + key: "debian-stretch-ruby-2.6.3-node-12.x" + paths: + - vendor/ruby + - .yarn-cache/ + - vendor/gitaly-ruby policy: pull - stage: test -.no-docs: +.except-docs: except: refs: - /(^docs[\/-].+|.+-docs$)/ -.no-docs-and-no-qa: +.except-qa: except: refs: - - /(^docs[\/-].+|.+-docs$)/ - /(^qa[\/-].*|.*-qa$)/ -.dedicated-no-docs-pull-cache-job: - extends: - - .dedicated-pull-cache-job - - .no-docs - -.dedicated-no-docs-and-no-qa-pull-cache-job: - extends: - - .dedicated-pull-cache-job - - .no-docs-and-no-qa - -# Jobs that do not need a DB -.dedicated-no-docs-no-db-pull-cache-job: - extends: .dedicated-no-docs-pull-cache-job - variables: - SETUP_DB: "false" - -# Jobs that need a dedicated runner, with no cache -.dedicated-no-docs: - extends: - - .dedicated-runner - - .no-docs +.except-docs-qa: + except: + refs: + - /(^docs[\/-].+|.+-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ -.single-script-job-dedicated-runner: - extends: .dedicated-runner - image: ruby:2.6-alpine - stage: test - cache: {} - dependencies: [] - variables: - GIT_STRATEGY: none - before_script: - # We don't clone the repo by using GIT_STRATEGY: none and only download the - # single script we need here so it's much faster than cloning. - - export SCRIPT_NAME="${SCRIPT_NAME:-$CI_JOB_NAME}" - - apk add --update openssl - - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/$SCRIPT_NAME - - chmod 755 $(basename $SCRIPT_NAME) +.except-docs-qa-geo: + except: + refs: + - /(^docs[\/-].+|.+-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ + - /(^geo[\/-].*|.*-geo$)/ -.review-only: &review-only +.review-only: only: refs: - branches@gitlab-org/gitlab-ce @@ -88,3 +61,16 @@ - master - /^\d+-\d+-auto-deploy-\d+$/ - /(^docs[\/-].+|.+-docs$)/ + +.use-pg: + services: + - name: postgres:9.6.14 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:alpine + +.use-pg-10: + image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33" + services: + - name: postgres:10.9 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:alpine diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml index 9923732e587..1936933cca4 100644 --- a/.gitlab/ci/memory.gitlab-ci.yml +++ b/.gitlab/ci/memory.gitlab-ci.yml @@ -1,5 +1,12 @@ memory-static: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs + variables: + SETUP_DB: "false" script: # Uses two different reports from the 'derailed_benchmars' gem. @@ -23,7 +30,13 @@ memory-static: # The application is booted in `production` environment. # All tests are run without a webserver (directly using Rack::Mock by default). memory-on-boot: - extends: .rspec-metadata-pg-10 + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg-10 + - .except-docs-qa variables: NODE_ENV: "production" RAILS_ENV: "production" diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index f7b18b809b4..3247d7c4bce 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -1,13 +1,15 @@ pages: - extends: .dedicated-no-docs-no-db-pull-cache-job - before_script: [] + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + only: + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee stage: pages - dependencies: - - coverage - - karma - - gitlab:assets:compile - - lint:javascript:report - - jsdoc + dependencies: ["coverage", "karma", "gitlab:assets:compile", "lint:javascript:report", "jsdoc"] script: - mv public/ .public/ - mkdir public/ @@ -21,6 +23,3 @@ pages: artifacts: paths: - public - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 69b932801ad..ac2a70dda0b 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -1,10 +1,8 @@ .package-and-qa-base: image: ruby:2.6-alpine stage: qa - before_script: [] needs: ["build-qa-image", "gitlab:assets:compile pull-cache"] dependencies: [] - cache: {} variables: GIT_DEPTH: "1" retry: 0 @@ -19,7 +17,9 @@ - master package-and-qa-manual: - extends: .package-and-qa-base + extends: + - .package-and-qa-base + - .except-docs-qa when: manual except: - master diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 50476b43dd6..2e8b197829b 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -1,52 +1,31 @@ -.use-pg: &use-pg - services: - - name: postgres:9.6.14 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - -.use-pg-10: &use-pg-10 - services: - - name: postgres:10.9 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - -.only-schedules-master: &only-schedules-master +.only-schedules-master: only: - - schedules@gitlab-org/gitlab-ce - - schedules@gitlab-org/gitlab-ee - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - master@gitlab/gitlabhq - - master@gitlab/gitlab-ee + refs: + - schedules@gitlab-org/gitlab-ce + - schedules@gitlab-org/gitlab-ee + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - master@gitlab/gitlabhq + - master@gitlab/gitlab-ee -.gitlab-setup: &gitlab-setup +.rake-exec: extends: - - .dedicated-no-docs-and-no-qa-pull-cache-job - - .use-pg + - .default-tags + - .default-retry + - .default-cache + - .default-before_script variables: SETUP_DB: "false" script: - # Manually clone gitlab-test and only seed this project in - # db/fixtures/development/04_project.rb thanks to SIZE=1 below - - git clone https://gitlab.com/gitlab-org/gitlab-test.git - /home/git/repositories/gitlab-org/gitlab-test.git - - scripts/gitaly-test-spawn - - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup - artifacts: - when: on_failure - expire_in: 1d - paths: - - log/development.log - -.rake-exec: &rake-exec - extends: .dedicated-no-docs-no-db-pull-cache-job - script: - bundle exec rake $CI_JOB_NAME -.rspec-metadata: &rspec-metadata +.rspec-base: extends: - - .dedicated-pull-cache-job - - .no-docs-and-no-qa + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs-qa stage: test script: - JOB_NAME=( $CI_JOB_NAME ) @@ -83,52 +62,24 @@ reports: junit: junit_rspec.xml -.rspec-metadata-pg: &rspec-metadata-pg - <<: *rspec-metadata - <<: *use-pg - -.rspec-metadata-pg-10: &rspec-metadata-pg-10 - <<: *rspec-metadata - <<: *use-pg-10 - image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33" - -# DB migration, rollback, and seed jobs -.db-migrate-reset: &db-migrate-reset - extends: .dedicated-no-docs-and-no-qa-pull-cache-job - script: - - bundle exec rake db:migrate:reset - dependencies: - - setup-test-env +.rspec-base-pg: + extends: + - .rspec-base + - .use-pg -.migration-paths: &migration-paths - extends: .dedicated-no-docs-and-no-qa-pull-cache-job - variables: - SETUP_DB: "false" - script: - - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v11.11.0 - - git checkout -f FETCH_HEAD - - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile - - sed -i "s/gem 'bootsnap', '~> 1.0.0'/gem 'bootsnap'/" Gemfile - - bundle update google-protobuf grpc bootsnap - - bundle install $BUNDLE_INSTALL_FLAGS - - date - - cp config/gitlab.yml.example config/gitlab.yml - - bundle exec rake db:drop db:create db:schema:load db:seed_fu - - date - - git checkout -f $CI_COMMIT_SHA - - bundle install $BUNDLE_INSTALL_FLAGS - - date - - . scripts/prepare_build.sh - - date - - bundle exec rake db:migrate - dependencies: - - setup-test-env +.rspec-base-pg-10: + extends: + - .rspec-base + - .use-pg-10 setup-test-env: extends: - - .dedicated-runner-default-cache - - .no-docs + - .default-tags + - .default-retry + - .default-cache + - .default-before_script - .use-pg + - .except-docs stage: prepare script: - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init' @@ -141,67 +92,72 @@ setup-test-env: - vendor/gitaly-ruby rspec unit pg: - <<: *rspec-metadata-pg + extends: .rspec-base-pg parallel: 20 rspec integration pg: - <<: *rspec-metadata-pg + extends: .rspec-base-pg parallel: 6 rspec system pg: - <<: *rspec-metadata-pg + extends: .rspec-base-pg parallel: 24 rspec unit pg-10: - <<: *rspec-metadata-pg-10 - <<: *only-schedules-master + extends: + - .rspec-base-pg-10 + - .only-schedules-master parallel: 20 rspec integration pg-10: - <<: *rspec-metadata-pg-10 - <<: *only-schedules-master + extends: + - .rspec-base-pg-10 + - .only-schedules-master parallel: 6 rspec system pg-10: - <<: *rspec-metadata-pg-10 - <<: *only-schedules-master + extends: + - .rspec-base-pg-10 + - .only-schedules-master parallel: 24 rspec-fast-spec-helper: - <<: *rspec-metadata-pg + extends: .rspec-base-pg script: - bundle exec rspec spec/fast_spec_helper.rb -.rspec-quarantine: &rspec-quarantine - <<: *only-schedules-master +rspec quarantine pg: + extends: + - .default-before_script + - .rspec-base-pg + - .only-schedules-master script: - - export CACHE_CLASSES=true + - export NO_KNAPSACK=1 CACHE_CLASSES=true - scripts/gitaly-test-spawn - bin/rspec --color --format documentation --tag quarantine -- spec/ - -rspec quarantine pg: - <<: *rspec-metadata-pg - <<: *rspec-quarantine allow_failure: true static-analysis: - extends: .dedicated-no-docs-no-db-pull-cache-job - dependencies: - - compile-assets - - compile-assets pull-cache - - setup-test-env + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs + dependencies: ["setup-test-env", "compile-assets", "compile-assets pull-cache"] + variables: + SETUP_DB: "false" script: - scripts/static-analysis cache: - key: "debian-stretch-ruby-2.6.3-node-12.x-and-rubocop" + key: "debian-stretch-ruby-2.6.3-and-rubocop" paths: - vendor/ruby - - .yarn-cache/ - tmp/rubocop_cache policy: pull-push downtime_check: - <<: *rake-exec + extends: .rake-exec except: refs: - master @@ -209,22 +165,20 @@ downtime_check: - /^[\d-]+-stable(-ee)?$/ - /(^docs[\/-].+|.+-docs$)/ - /(^qa[\/-].*|.*-qa$)/ - dependencies: - - setup-test-env + dependencies: ["setup-test-env"] ee_compat_check: - <<: *rake-exec + extends: .rake-exec dependencies: [] except: refs: - master - tags - - /[\d-]+-stable(-ee)?/ - - /^security-/ - branches@gitlab-org/gitlab-ee - branches@gitlab/gitlab-ee + - /^[\d-]+-stable(-ee)?$/ - /(^docs[\/-].+|.+-docs$)/ - retry: 0 + - /^security-/ artifacts: name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}" when: always @@ -232,44 +186,106 @@ ee_compat_check: paths: - ee_compat_check/patches/*.patch -db:migrate:reset-pg: - <<: *db-migrate-reset - <<: *use-pg +# DB migration, rollback, and seed jobs +db:migrate:reset: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] + script: + - bundle exec rake db:migrate:reset -db:check-schema-pg: - <<: *db-migrate-reset - <<: *use-pg +db:check-schema: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] script: - source scripts/schema_changed.sh -migration:path-pg: - <<: *migration-paths - <<: *use-pg +db:migrate-from-v11.11.0: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] + variables: + SETUP_DB: "false" + script: + - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v11.11.0 + - git checkout -f FETCH_HEAD + - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile + - sed -i "s/gem 'bootsnap', '~> 1.0.0'/gem 'bootsnap'/" Gemfile + - bundle update google-protobuf grpc bootsnap + - bundle install $BUNDLE_INSTALL_FLAGS + - date + - cp config/gitlab.yml.example config/gitlab.yml + - bundle exec rake db:drop db:create db:schema:load db:seed_fu + - date + - git checkout -f $CI_COMMIT_SHA + - bundle install $BUNDLE_INSTALL_FLAGS + - date + - . scripts/prepare_build.sh + - date + - bundle exec rake db:migrate -.db-rollback: &db-rollback - extends: .dedicated-no-docs-and-no-qa-pull-cache-job +db:rollback: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] script: - bundle exec rake db:migrate VERSION=20180101160629 - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true - dependencies: - - setup-test-env - -db:rollback-pg: - <<: *db-rollback - <<: *use-pg -gitlab:setup-pg: - <<: *gitlab-setup - <<: *use-pg - dependencies: - - setup-test-env +gitlab:setup: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] + variables: + SETUP_DB: "false" + script: + # Manually clone gitlab-test and only seed this project in + # db/fixtures/development/04_project.rb thanks to SIZE=1 below + - git clone https://gitlab.com/gitlab-org/gitlab-test.git + /home/git/repositories/gitlab-org/gitlab-test.git + - scripts/gitaly-test-spawn + - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup + artifacts: + when: on_failure + expire_in: 1d + paths: + - log/development.log coverage: # Don't include dedicated-no-docs-no-db-pull-cache-job here since we need to # download artifacts from all the rspec jobs instead of from setup-test-env only extends: - - .dedicated-runner-default-cache - - .no-docs-and-no-qa + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs-qa cache: policy: pull variables: diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index e3768ecf2a2..905f1ede2c4 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -4,36 +4,29 @@ include: - template: Security/Dependency-Scanning.gitlab-ci.yml - template: Security/DAST.gitlab-ci.yml +.reports: + extends: + - .default-retry + - .except-docs + tags: + - gitlab-org + - docker + code_quality: - extends: .dedicated-no-docs - # gitlab-org runners set `privileged: false` but we need to have it set to true - # since we're using Docker in Docker - tags: [] - before_script: [] - cache: {} + extends: .reports sast: - extends: .dedicated-no-docs - tags: [] - before_script: [] - cache: {} + extends: .reports variables: SAST_BRAKEMAN_LEVEL: 2 SAST_EXCLUDED_PATHS: qa,spec,doc dependency_scanning: - extends: .dedicated-no-docs - tags: [] - before_script: [] - cache: {} + extends: .reports dast: - extends: - - .dedicated-runner - - .review-only + extends: .reports stage: qa - dependencies: - - review-deploy + dependencies: ["review-deploy"] before_script: - export DAST_WEBSITE="$(cat review_app_url.txt)" - cache: {} diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index beb049c0b3b..3415f1b6ab4 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -1,4 +1,4 @@ -.review-schedules-only: &review-schedules-only +.review-schedules-only: only: refs: - schedules@gitlab-org/gitlab-ce @@ -11,39 +11,39 @@ - tags - /(^docs[\/-].+|.+-docs$)/ -.review-base: &review-base +.review-base: extends: - - .dedicated-runner + - .default-tags + - .default-retry - .review-only image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base - cache: {} dependencies: [] before_script: - source scripts/utils.sh -.review-docker: &review-docker - <<: *review-base +.review-docker: + extends: .review-base image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine services: - docker:19.03.0-dind tags: - gitlab-org - docker - variables: &review-docker-variables + variables: DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 LATEST_QA_IMAGE: "gitlab/${CI_PROJECT_NAME}-qa:nightly" QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/${CI_PROJECT_NAME}-qa:${CI_COMMIT_REF_SLUG}" build-qa-image: - <<: *review-docker + extends: .review-docker stage: test script: - time docker build --cache-from ${LATEST_QA_IMAGE} --tag ${QA_IMAGE} --file ./qa/Dockerfile ./ - echo "${CI_JOB_TOKEN}" | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY} - time docker push ${QA_IMAGE} -.review-build-cng-base: &review-build-cng-base +.review-build-cng-base: image: ruby:2.6-alpine stage: test when: manual @@ -52,20 +52,21 @@ build-qa-image: - install_api_client_dependencies_with_apk - install_gitlab_gem dependencies: [] - cache: {} script: - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng review-build-cng: - extends: .review-only - <<: *review-build-cng-base + extends: + - .review-build-cng-base + - .review-only schedule:review-build-cng: - <<: *review-schedules-only - <<: *review-build-cng-base + extends: + - .review-build-cng-base + - .review-schedules-only -.review-deploy-base: &review-deploy-base - <<: *review-base +review-deploy: + extends: .review-base allow_failure: true retry: 1 stage: review @@ -73,7 +74,7 @@ schedule:review-build-cng: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" GITLAB_HELM_CHART_REF: "master" - environment: &review-environment + environment: name: review/${CI_COMMIT_REF_NAME} url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} on_stop: review-stop @@ -98,50 +99,45 @@ schedule:review-build-cng: expire_in: 2 days when: always -review-deploy: - <<: *review-deploy-base - schedule:review-deploy: - <<: *review-deploy-base - <<: *review-schedules-only + extends: + - review-deploy + - .review-schedules-only review-stop: - extends: - - .single-script-job-dedicated-runner - - .review-only - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base - stage: review + extends: review-deploy when: manual - allow_failure: true - variables: - SCRIPT_NAME: review_apps/review-apps.sh environment: - <<: *review-environment action: stop - script: + variables: + GIT_STRATEGY: none + before_script: + # We don't clone the repo by using GIT_STRATEGY: none and only download the + # single script we need here so it's much faster than cloning. + - apk add --update openssl + - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/review_apps/review-apps.sh - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/utils.sh - source utils.sh - - source $(basename $SCRIPT_NAME) + - source review-apps.sh + script: - delete + artifacts: {} -.review-qa-base: &review-qa-base - <<: *review-docker - allow_failure: true +.review-qa-base: + extends: .review-docker retry: 2 stage: qa variables: - <<: *review-docker-variables QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" QA_CAN_TEST_GIT_PROTOCOL_V2: "false" + QA_DEBUG: "true" GITLAB_USERNAME: "root" GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" GITLAB_ADMIN_USERNAME: "root" GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" - QA_DEBUG: "true" - dependencies: - - review-deploy + dependencies: ["review-deploy"] artifacts: paths: - ./qa/gitlab-qa-run-* @@ -156,12 +152,13 @@ review-stop: - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}} review-qa-smoke: - <<: *review-qa-base + extends: .review-qa-base + allow_failure: true script: - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" review-qa-all: - <<: *review-qa-base + extends: .review-qa-base allow_failure: true when: manual parallel: 5 @@ -172,21 +169,16 @@ review-qa-all: parallel-spec-reports: extends: - - .dedicated-runner - - .no-docs - dependencies: - - review-qa-all + - .default-tags + - .except-docs image: ruby:2.6-alpine - services: [] - before_script: [] + stage: post-test + dependencies: ["review-qa-all"] variables: - SETUP_DB: "false" NEW_PARALLEL_SPECS_REPORT: qa/report-new.html BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/" - stage: post-test allow_failure: true when: manual - retry: 0 artifacts: when: always paths: @@ -196,15 +188,15 @@ parallel-spec-reports: junit: qa/gitlab-qa-run-*/**/rspec-*.xml script: - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/* - - gem install nokogiri + - gem install nokogiri --no-document - cd qa/gitlab-qa-run-*/gitlab-* - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_) - - cd ../../.. + - cd - - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}' - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm -.review-performance-base: &review-performance-base - <<: *review-qa-base +review-performance: + extends: .review-qa-base allow_failure: true before_script: - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)" @@ -222,18 +214,16 @@ parallel-spec-reports: reports: performance: performance.json -review-performance: - <<: *review-performance-base - schedule:review-performance: - <<: *review-performance-base - <<: *review-schedules-only - dependencies: - - schedule:review-deploy + extends: + - review-performance + - .review-schedules-only + dependencies: ["schedule:review-deploy"] schedule:review-cleanup: - <<: *review-base - <<: *review-schedules-only + extends: + - .review-base + - .review-schedules-only stage: build allow_failure: true environment: @@ -246,11 +236,13 @@ schedule:review-cleanup: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb danger-review: - extends: .dedicated-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger stage: test dependencies: [] - before_script: [] only: variables: - $DANGER_GITLAB_API_TOKEN @@ -259,9 +251,8 @@ danger-review: - master - /^\d+-\d+-auto-deploy-\d+$/ - /^[\d-]+-stable(-ee)?$/ - variables: - - $CI_COMMIT_REF_NAME =~ /^ce-to-ee-.*/ - - $CI_COMMIT_REF_NAME =~ /.*-stable(-ee)?-prepare-.*/ + - /^ce-to-ee-.*/ + - /.*-stable(-ee)?-prepare-.*/ script: - git version - node --version diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index c1fc3a893ca..d9384780356 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -1,41 +1,42 @@ # Insurance in case a gem needed by one of our releases gets yanked from # rubygems.org in the future. cache gems: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs + dependencies: ["setup-test-env"] + variables: + SETUP_DB: "false" script: - bundle package --all --all-platforms artifacts: paths: - vendor/cache only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - tags - dependencies: - - setup-test-env + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - tags -gitlab_git_test: +.minimal-job: extends: - - .dedicated-runner - - .no-docs-and-no-qa - variables: - SETUP_DB: "false" - before_script: [] + - .default-tags + - .default-retry + - .except-docs-qa dependencies: [] - cache: {} + +gitlab_git_test: + extends: .minimal-job script: - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes no_ee_check: - extends: - - .dedicated-runner - - .no-docs-and-no-qa - variables: - SETUP_DB: "false" - before_script: [] - dependencies: [] - cache: {} + extends: .minimal-job script: - scripts/no-ee-check only: - - /.+/@gitlab-org/gitlab-ce + refs: + - branches@gitlab-org/gitlab-ce diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index 4c97a4feb18..b9dac64957e 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -1,5 +1,4 @@ -.tests-metadata-state: &tests-metadata-state - extends: .dedicated-runner +.tests-metadata-state: variables: TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache" before_script: @@ -14,7 +13,7 @@ retrieve-tests-metadata: extends: - .tests-metadata-state - - .no-docs-and-no-qa + - .except-docs-qa stage: prepare cache: key: tests_metadata @@ -29,7 +28,7 @@ retrieve-tests-metadata: - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}' update-tests-metadata: - <<: *tests-metadata-state + extends: .tests-metadata-state stage: post-test cache: key: tests_metadata @@ -49,25 +48,24 @@ update-tests-metadata: - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json - scripts/insert-rspec-profiling-data only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - master@gitlab/gitlabhq - - master@gitlab/gitlab-ee + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - master@gitlab/gitlabhq + - master@gitlab/gitlab-ee flaky-examples-check: - extends: .dedicated-runner + extends: + - .default-tags + - .default-retry image: ruby:2.6-alpine - services: [] - before_script: [] + stage: post-test variables: - SETUP_DB: "false" - USE_BUNDLE_INSTALL: "false" NEW_FLAKY_SPECS_REPORT: rspec_flaky/report-new.json - stage: post-test allow_failure: true - retry: 0 only: - - branches + refs: + - branches except: refs: - master diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml index b7aa418d8f7..3e107b475c9 100644 --- a/.gitlab/ci/yaml.gitlab-ci.yml +++ b/.gitlab/ci/yaml.gitlab-ci.yml @@ -1,9 +1,10 @@ # Yamllint of *.yml for .gitlab-ci.yml. # This uses rules from project root `.yamllint`. lint-ci-gitlab: - extends: .dedicated-runner - before_script: [] - dependencies: [] + extends: + - .default-tags + - .default-retry image: sdesbure/yamllint:latest + dependencies: [] script: - yamllint .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates changelogs diff --git a/app/assets/javascripts/main.js b/app/assets/javascripts/main.js index 0ddf40b0405..39f2097c174 100644 --- a/app/assets/javascripts/main.js +++ b/app/assets/javascripts/main.js @@ -35,7 +35,6 @@ import initPerformanceBar from './performance_bar'; import initSearchAutocomplete from './search_autocomplete'; import GlFieldErrors from './gl_field_errors'; import initUserPopovers from './user_popovers'; -import { initUserTracking } from './tracking'; import { __ } from './locale'; import 'ee_else_ce/main_ee'; @@ -95,7 +94,6 @@ function deferredInitialisation() { initLogoAnimation(); initUsagePingConsent(); initUserPopovers(); - initUserTracking(); if (document.querySelector('.search')) initSearchAutocomplete(); diff --git a/app/assets/javascripts/tracking.js b/app/assets/javascripts/tracking.js index 03281b5ef49..a852f937eec 100644 --- a/app/assets/javascripts/tracking.js +++ b/app/assets/javascripts/tracking.js @@ -1,23 +1,5 @@ import $ from 'jquery'; -const DEFAULT_SNOWPLOW_OPTIONS = { - namespace: 'gl', - hostname: window.location.hostname, - cookieDomain: window.location.hostname, - appId: '', - userFingerprint: false, - respectDoNotTrack: true, - forceSecureTracker: true, - eventMethod: 'post', - contexts: { webPage: true }, - // Page tracking tracks a single event when the page loads. - pageTrackingEnabled: false, - // Activity tracking tracks when a user is still interacting with the page. - // Events like scrolling and mouse movements are used to determine if the - // user has the tab active and is still actively engaging. - activityTrackingEnabled: false, -}; - const extractData = (el, opts = {}) => { const { trackEvent, trackLabel = '', trackProperty = '' } = el.dataset; let trackValue = el.dataset.trackValue || el.value || ''; @@ -89,13 +71,3 @@ export default class Tracking { }; } } - -export function initUserTracking() { - if (!Tracking.enabled()) return; - - const opts = Object.assign({}, DEFAULT_SNOWPLOW_OPTIONS, window.snowplowOptions); - window.snowplow('newTracker', opts.namespace, opts.hostname, opts); - - if (opts.activityTrackingEnabled) window.snowplow('enableActivityTracking', 30, 30); - if (opts.pageTrackingEnabled) window.snowplow('trackPageView'); // must be after enableActivityTracking -} diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 33f0be91632..85f5a2040c0 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -5,7 +5,7 @@ class DeployToken < ApplicationRecord include TokenAuthenticatable include PolicyActor include Gitlab::Utils::StrongMemoize - add_authentication_token_field :token + add_authentication_token_field :token, encrypted: :optional AVAILABLE_SCOPES = %i(read_repository read_registry).freeze GITLAB_DEPLOY_TOKEN_NAME = 'gitlab-deploy-token'.freeze diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index b8dee1b0789..e2634692dc7 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -502,6 +502,8 @@ class ProjectPolicy < BasePolicy end def feature_available?(feature) + return false unless project.project_feature + case project.project_feature.access_level(feature) when ProjectFeature::DISABLED false diff --git a/app/views/layouts/_piwik.html.haml b/app/views/layouts/_piwik.html.haml index 2cb2e23433d..361a7b03180 100644 --- a/app/views/layouts/_piwik.html.haml +++ b/app/views/layouts/_piwik.html.haml @@ -11,5 +11,5 @@ var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.type='text/javascript'; g.async=true; g.defer=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s); })(); - <noscript><p><img src="//#{extra_config.piwik_url}/piwik.php?idsite=#{extra_config.piwik_site_id}" style="border:0;" alt="" /></p></noscript> - <!-- End Piwik Code --> +<noscript><p><img src="//#{extra_config.piwik_url}/piwik.php?idsite=#{extra_config.piwik_site_id}" style="border:0;" alt="" /></p></noscript> +<!-- End Piwik Code --> diff --git a/app/views/layouts/_snowplow.html.haml b/app/views/layouts/_snowplow.html.haml index d7ff5ad1094..5f5c5e984c5 100644 --- a/app/views/layouts/_snowplow.html.haml +++ b/app/views/layouts/_snowplow.html.haml @@ -7,4 +7,23 @@ };p[i].q=p[i].q||[];n=l.createElement(o);g=l.getElementsByTagName(o)[0];n.async=1; n.src=w;g.parentNode.insertBefore(n,g)}}(window,document,"script","#{asset_url('snowplow/sp.js')}","snowplow")); - window.snowplowOptions = #{Gitlab::Tracking.snowplow_options(@group).to_json} + window.snowplow('newTracker', '#{Gitlab::SnowplowTracker::NAMESPACE}', '#{Gitlab::CurrentSettings.snowplow_collector_hostname}', { + appId: '#{Gitlab::CurrentSettings.snowplow_site_id}', + cookieDomain: '#{Gitlab::CurrentSettings.snowplow_cookie_domain}', + userFingerprint: false, + respectDoNotTrack: true, + forceSecureTracker: true, + post: true, + contexts: { webPage: true }, + stateStorageStrategy: "localStorage" + }); + + window.snowplow('enableActivityTracking', 30, 30); + window.snowplow('trackPageView'); + +- return unless Feature.enabled?(:additional_snowplow_tracking, @group) + += javascript_tag nonce: true do + :plain + window.snowplow('enableFormTracking'); + window.snowplow('enableLinkClickTracking'); diff --git a/app/views/projects/mirrors/_mirror_repos.html.haml b/app/views/projects/mirrors/_mirror_repos.html.haml index eb100e5cf47..6f8a93fbcf5 100644 --- a/app/views/projects/mirrors/_mirror_repos.html.haml +++ b/app/views/projects/mirrors/_mirror_repos.html.haml @@ -13,8 +13,6 @@ .settings-content = form_for @project, url: project_mirror_path(@project), html: { class: 'gl-show-field-errors js-mirror-form', autocomplete: 'new-password', data: mirrors_form_data_attributes } do |f| .panel.panel-default - .panel-heading - %h3.panel-title= _('Mirror a repository') .panel-body %div= form_errors(@project) diff --git a/changelogs/unreleased/63502-encrypt-deploy-token.yml b/changelogs/unreleased/63502-encrypt-deploy-token.yml new file mode 100644 index 00000000000..81ce1e6c3dd --- /dev/null +++ b/changelogs/unreleased/63502-encrypt-deploy-token.yml @@ -0,0 +1,5 @@ +--- +title: Encrypt existing and new deploy tokens +merge_request: 30679 +author: +type: other diff --git a/changelogs/unreleased/oauth_bypass_two_factor.yml b/changelogs/unreleased/oauth_bypass_two_factor.yml new file mode 100644 index 00000000000..7261e65a9f3 --- /dev/null +++ b/changelogs/unreleased/oauth_bypass_two_factor.yml @@ -0,0 +1,5 @@ +--- +title: Add option to allow OAuth providers to bypass two factor +merge_request: 31996 +author: Dodocat +type: added diff --git a/changelogs/unreleased/sh-fix-piwik-template.yml b/changelogs/unreleased/sh-fix-piwik-template.yml new file mode 100644 index 00000000000..f0baed6a2e0 --- /dev/null +++ b/changelogs/unreleased/sh-fix-piwik-template.yml @@ -0,0 +1,5 @@ +--- +title: Fix Piwik not working +merge_request: 32234 +author: +type: fixed diff --git a/changelogs/unreleased/sh-guard-against-orphaned-project-feature.yml b/changelogs/unreleased/sh-guard-against-orphaned-project-feature.yml new file mode 100644 index 00000000000..99c8732c5b0 --- /dev/null +++ b/changelogs/unreleased/sh-guard-against-orphaned-project-feature.yml @@ -0,0 +1,5 @@ +--- +title: Guard against deleted project feature entry in project permissions +merge_request: 32187 +author: +type: fixed diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 973c2747838..20b1020e025 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -780,6 +780,14 @@ production: &base # (default: []) external_providers: [] + # CAUTION! + # This allows users to login with the specified providers without two factor. Define the allowed providers + # using an array, e.g. ["twitter", 'google_oauth2'], or as true/false to allow all providers or none. + # This option should only be configured for providers which already have two factor. + # This configration dose not apply to SAML. + # (default: false) + allow_bypass_two_factor: ["twitter", 'google_oauth2'] + ## Auth providers # Uncomment the following lines and fill in the data of the auth provider you want to use # If your favorite auth provider is not listed you can use others: diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index fdc6b0a05ab..81433b620bc 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -84,6 +84,7 @@ Settings['omniauth'] ||= Settingslogic.new({}) Settings.omniauth['enabled'] = true if Settings.omniauth['enabled'].nil? Settings.omniauth['auto_sign_in_with_provider'] = false if Settings.omniauth['auto_sign_in_with_provider'].nil? Settings.omniauth['allow_single_sign_on'] = false if Settings.omniauth['allow_single_sign_on'].nil? +Settings.omniauth['allow_bypass_two_factor'] = false if Settings.omniauth['allow_bypass_two_factor'].nil? Settings.omniauth['external_providers'] = [] if Settings.omniauth['external_providers'].nil? Settings.omniauth['block_auto_created_users'] = true if Settings.omniauth['block_auto_created_users'].nil? Settings.omniauth['auto_link_ldap_user'] = false if Settings.omniauth['auto_link_ldap_user'].nil? diff --git a/db/migrate/20190711200053_change_deploy_tokens_token_not_null.rb b/db/migrate/20190711200053_change_deploy_tokens_token_not_null.rb new file mode 100644 index 00000000000..14ccf544d0b --- /dev/null +++ b/db/migrate/20190711200053_change_deploy_tokens_token_not_null.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class ChangeDeployTokensTokenNotNull < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + change_column_null :deploy_tokens, :token, true + end +end diff --git a/db/migrate/20190711200508_add_token_encrypted_to_deploy_tokens.rb b/db/migrate/20190711200508_add_token_encrypted_to_deploy_tokens.rb new file mode 100644 index 00000000000..ea0956fdf7f --- /dev/null +++ b/db/migrate/20190711200508_add_token_encrypted_to_deploy_tokens.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddTokenEncryptedToDeployTokens < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :deploy_tokens, :token_encrypted, :string, limit: 255 + end +end diff --git a/db/migrate/20190719174505_add_index_to_deploy_tokens_token_encrypted.rb b/db/migrate/20190719174505_add_index_to_deploy_tokens_token_encrypted.rb new file mode 100644 index 00000000000..d58d1d8348c --- /dev/null +++ b/db/migrate/20190719174505_add_index_to_deploy_tokens_token_encrypted.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddIndexToDeployTokensTokenEncrypted < ActiveRecord::Migration[5.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :deploy_tokens, :token_encrypted, unique: true, name: "index_deploy_tokens_on_token_encrypted" + end + + def down + remove_concurrent_index_by_name :deploy_tokens, "index_deploy_tokens_on_token_encrypted" + end +end diff --git a/db/post_migrate/20190711201818_encrypt_deploy_tokens_tokens.rb b/db/post_migrate/20190711201818_encrypt_deploy_tokens_tokens.rb new file mode 100644 index 00000000000..2eb8d1ee11c --- /dev/null +++ b/db/post_migrate/20190711201818_encrypt_deploy_tokens_tokens.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class EncryptDeployTokensTokens < ActiveRecord::Migration[5.1] + DOWNTIME = false + + class DeploymentTokens < ActiveRecord::Base + self.table_name = 'deploy_tokens' + end + + def up + say_with_time("Encrypting tokens from deploy_tokens") do + DeploymentTokens.where('token_encrypted is NULL AND token IS NOT NULL').find_each(batch_size: 10000) do |deploy_token| + token_encrypted = Gitlab::CryptoHelper.aes256_gcm_encrypt(deploy_token.token) + deploy_token.update!(token_encrypted: token_encrypted) + end + end + end + + def down + say_with_time("Decrypting tokens from deploy_tokens") do + DeploymentTokens.where('token_encrypted IS NOT NULL AND token IS NULL').find_each(batch_size: 10000) do |deploy_token| + token = Gitlab::CryptoHelper.aes256_gcm_decrypt(deploy_token.token_encrypted) + deploy_token.update!(token: token) + end + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 36bf2371994..f30dad3d030 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1121,10 +1121,12 @@ ActiveRecord::Schema.define(version: 2019_08_20_163320) do t.datetime_with_timezone "expires_at", null: false t.datetime_with_timezone "created_at", null: false t.string "name", null: false - t.string "token", null: false + t.string "token" t.string "username" + t.string "token_encrypted", limit: 255 t.index ["token", "expires_at", "id"], name: "index_deploy_tokens_on_token_and_expires_at_and_id", where: "(revoked IS FALSE)" t.index ["token"], name: "index_deploy_tokens_on_token", unique: true + t.index ["token_encrypted"], name: "index_deploy_tokens_on_token_encrypted", unique: true end create_table "deployments", id: :serial, force: :cascade do |t| diff --git a/doc/api/events.md b/doc/api/events.md index 6dca8e52f69..1cd7047b867 100644 --- a/doc/api/events.md +++ b/doc/api/events.md @@ -70,7 +70,7 @@ Parameters: Example request: -``` +```bash curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/events?target_type=issue&action=created&after=2017-01-31&before=2017-03-01 ``` @@ -275,7 +275,7 @@ Parameters: Example request: -``` +```bash curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/:project_id/events?target_type=issue&action=created&after=2017-01-31&before=2017-03-01 ``` @@ -343,8 +343,8 @@ Example response: "username": "root", "id": 1, "state": "active", - "avatar_url": "http://localhost:3000/uploads/user/avatar/1/fox_avatar.png", - "web_url": "http://localhost:3000/root" + "avatar_url": "https://gitlab.example.com/uploads/user/avatar/1/fox_avatar.png", + "web_url": "https://gitlab.example.com/root" }, "created_at": "2015-12-04T10:33:56.698Z", "system": false, @@ -357,8 +357,8 @@ Example response: "username": "root", "id": 1, "state": "active", - "avatar_url": "http://localhost:3000/uploads/user/avatar/1/fox_avatar.png", - "web_url": "http://localhost:3000/root" + "avatar_url": "https://gitlab.example.com/uploads/user/avatar/1/fox_avatar.png", + "web_url": "https://gitlab.example.com/root" }, "author_username": "root" } diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 89a61b2a9e3..663fa1395a6 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -100,6 +100,7 @@ The following table lists available parameters for jobs: | [`stage`](#stage) | Defines a job stage (default: `test`). | | [`only`](#onlyexcept-basic) | Limit when jobs are created. Also available: [`only:refs`, `only:kubernetes`, `only:variables`, and `only:changes`](#onlyexcept-advanced). | | [`except`](#onlyexcept-basic) | Limit when jobs are not created. Also available: [`except:refs`, `except:kubernetes`, `except:variables`, and `except:changes`](#onlyexcept-advanced). | +| [`rules`](#rules) | List of coniditions to evaluate and determine selected attributes of a build and whether or not it is created. May not be used alongside `only`/`except`. | [`tags`](#tags) | List of tags which are used to select Runner. | | [`allow_failure`](#allow_failure) | Allow job to fail. Failed job doesn't contribute to commit status. | | [`when`](#when) | When to run job. Also available: `when:manual` and `when:delayed`. | @@ -690,6 +691,125 @@ In the scenario above, if a merge request is created or updated that changes either files in `service-one` directory or the `Dockerfile`, GitLab creates and triggers the `docker build service one` job. +### `rules` + +Using `rules` allows for a list of individual rule objects to be evaluated +*in order*, until one matches and dynamically provides attributes to the job. + +Available rule clauses include: + +- `if` (similar to [`only:variables`](#onlyvariablesexceptvariables)). +- `changes` (same as [`only:changes`](#onlychangesexceptchanges)). + +For example, using `if`: + +```yaml +job: + script: "echo Hello, Rules!" + rules: + - if: '$CI_MERGE_REQUEST_TARGET_BRANCH == "master"' # This rule will be evaluated + when: always + - if: '$VAR =~ /pattern/' # This rule will only be evaluated if the first does not match + when: manual + - when: on_success # A Rule entry with no conditional clauses evaluates to true. If neither of the first two Rules match, this one will and set job:when to "on_success" +``` + +If the first rule does not match, further rules will be evaluated sequentially +until a match is found. The above configuration will specify that `job` should +be built and run for every pipeline on merge requests targeting `master`, +regardless of the status of other builds. + +#### `rules:if` + +`rules:if` differs slightly from `only:variables` by accepting only a single +expression string, rather than an array of them. Any set of expressions to be +evaluated should be conjoined into a single expression using `&&` or `||`. For example: + +```yaml +job: + script: "echo Hello, Rules!" + rules: + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH =~ /^feature/ && $CI_MERGE_REQUEST_TARGET_BRANCH == "master"' # This rule will be evaluated + when: always + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH =~ /^feature/' # This rule will only be evaluated if the target branch is not "master" + when: manual + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH' # If neither of the first two match but the simple presence does, we set to "on_success" by default +``` + +If none of the provided rules match, the job will be set to `when:never`, and +not included in the pipeline. If `rules:when` is not included in the configuration +at all, the behavior defaults to `job:when`, which continues to default to +`on_success`. + +#### `rules:changes` + +`changes` works exactly the same way as [`only`/`except`](#onlychangesexceptchanges), +accepting an array of paths. The following configuration configures a job to be +run manually if `Dockerfile` has changed OR `$VAR == "string value"`. Otherwise +it is set to `when:on_success` by the last rule, where 0 clauses evaluate as +vacuously true. + +```yaml +docker build: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - changes: # Will include the job and set to when:manual if any of the follow paths match a modified file. + - Dockerfile + when: manual + - if: '$VAR == "string value"' + when: manual # Will include the job and set to when:manual if the expression evaluates to true, after the `changes:` rule fails to match. + - when: on_success # If neither of the first rules match, set to on_success + +``` + +#### Complex Rule Clauses + +To conjoin `if` and `changes` clauses with an AND, use them in the same rule. +Here we run the job manually if `Dockerfile` or any file in `docker/scripts/` +has changed AND `$VAR == "string value"`. Otherwise, the job will not be +included in the pipeline. + +```yaml +docker build: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - if: '$VAR == "string value"' + changes: # Will include the job and set to when:manual if any of the follow paths match a modified file. + - Dockerfile + - docker/scripts/* + when: manual + # - when: never would be redundant here, this is implied any time rules are listed. +``` + +The only clauses currently available are `if` and `changes`. Keywords such as +`branches` or `refs` that are currently available for `only`/`except` are not +yet available in `rules` as they are being individually considered for their +usage and behavior in the newer context. + +#### Permitted attributes + +The only job attributes currently set by `rules` are `when` and `start_in`, if +`when` is set to `delayed`. A job will be included in a pipeline if `when` is +evaluated to any value except `never`. + +Delayed jobs require a `start_in` value, so rule objects do as well. For example: + +```yaml +docker build: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - changes: # Will include the job and delay 3 hours when the Dockerfile has changed + - Dockerfile + when: delayed + start_in: '3 hours' + - when: on_success # Otherwise include the job and set to run normally + +``` + +Additional Job configuration may be added to rules in the future, if something +useful isn't available, please open an issue on +[Gitlab CE](https://www.gitlab.com/gitlab-org/gitlab-ce/issues). + ### `tags` `tags` is used to select specific Runners from the list of all Runners that are diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index 7a92ed994c7..0314b5a808d 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -277,3 +277,24 @@ omniauth: sync_profile_from_provider: ['twitter', 'google_oauth2'] sync_profile_attributes: ['email', 'location'] ``` + +## Bypassing two factor authentication + +Starting with GitLab 12.3, this allows users to login with the specified +providers without two factor authentication. + +Define the allowed providers using an array, e.g. `["twitter", 'google_oauth2']`, or as +`true`/`false` to allow all providers or none. This option should only be configured +for providers which already have two factor authentication (default: false). +This configration dose not apply to SAML. + +```ruby +gitlab_rails['omniauth_allow_bypass_two_factor'] = ['twitter', 'google_oauth2'] +``` + +**For installations from source** + +```yaml +omniauth: + allow_bypass_two_factor: ['twitter', 'google_oauth2'] +``` diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 26d5221acc5..6021e8cff1d 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -111,9 +111,8 @@ To make full use of Auto DevOps, you will need: a domain configured with wildcard DNS which is going to be used by all of your Auto DevOps applications. [Read the specifics](#auto-devops-base-domain). - **Kubernetes** (needed for Auto Review Apps, Auto Deploy, and Auto Monitoring) - - To enable deployments, you will need Kubernetes 1.5+. You need a [Kubernetes cluster][kubernetes-clusters] - for the project, or a Kubernetes [default service template](../../user/project/integrations/services_templates.md) - for the entire GitLab installation. + To enable deployments, you will need Kubernetes 1.5+. You will need a [Kubernetes cluster][kubernetes-clusters] + for the project. - **A load balancer** - You can use NGINX ingress by deploying it to your Kubernetes cluster using the [`nginx-ingress`](https://github.com/kubernetes/charts/tree/master/stable/nginx-ingress) diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index e17a096ef19..6769bd95c2b 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -198,12 +198,10 @@ module Gitlab end.uniq end - # rubocop: disable CodeReuse/ActiveRecord def deploy_token_check(login, password) return unless password.present? - token = - DeployToken.active.find_by(token: password) + token = DeployToken.active.find_by_token(password) return unless token && login return if login != token.username @@ -214,7 +212,6 @@ module Gitlab Gitlab::Auth::Result.new(token, token.project, :deploy_token, scopes) end end - # rubocop: enable CodeReuse/ActiveRecord def lfs_token_check(login, encoded_token, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 09d1d79fefc..f121dce4cbb 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -77,7 +77,12 @@ module Gitlab end def bypass_two_factor? - false + providers = Gitlab.config.omniauth.allow_bypass_two_factor + if providers.is_a?(Array) + providers.include?(auth_hash.provider) + else + providers + end end protected diff --git a/lib/gitlab/snowplow_tracker.rb b/lib/gitlab/snowplow_tracker.rb new file mode 100644 index 00000000000..9f12513e09e --- /dev/null +++ b/lib/gitlab/snowplow_tracker.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'snowplow-tracker' + +module Gitlab + module SnowplowTracker + NAMESPACE = 'cf' + + class << self + def track_event(category, action, label: nil, property: nil, value: nil, context: nil) + tracker&.track_struct_event(category, action, label, property, value, context, Time.now.to_i) + end + + private + + def tracker + return unless enabled? + + @tracker ||= ::SnowplowTracker::Tracker.new(emitter, subject, NAMESPACE, Gitlab::CurrentSettings.snowplow_site_id) + end + + def subject + ::SnowplowTracker::Subject.new + end + + def emitter + ::SnowplowTracker::Emitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname) + end + + def enabled? + Gitlab::CurrentSettings.snowplow_enabled? + end + end + end +end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb deleted file mode 100644 index ef669b03c87..00000000000 --- a/lib/gitlab/tracking.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -require 'snowplow-tracker' - -module Gitlab - module Tracking - SNOWPLOW_NAMESPACE = 'gl' - - class << self - def enabled? - Gitlab::CurrentSettings.snowplow_enabled? - end - - def event(category, action, label: nil, property: nil, value: nil, context: nil) - return unless enabled? - - snowplow.track_struct_event(category, action, label, property, value, context, Time.now.to_i) - end - - def snowplow_options(group) - additional_features = Feature.enabled?(:additional_snowplow_tracking, group) - { - namespace: SNOWPLOW_NAMESPACE, - hostname: Gitlab::CurrentSettings.snowplow_collector_hostname, - cookie_domain: Gitlab::CurrentSettings.snowplow_cookie_domain, - app_id: Gitlab::CurrentSettings.snowplow_site_id, - page_tracking_enabled: additional_features, - activity_tracking_enabled: additional_features - }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym } - end - - private - - def snowplow - @snowplow ||= SnowplowTracker::Tracker.new( - SnowplowTracker::Emitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname), - SnowplowTracker::Subject.new, - SNOWPLOW_NAMESPACE, - Gitlab::CurrentSettings.snowplow_site_id - ) - end - end - end -end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index e1059032be5..e44c18979cd 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -7113,9 +7113,6 @@ msgstr "" msgid "Minimum length is %{minimum_password_length} characters." msgstr "" -msgid "Mirror a repository" -msgstr "" - msgid "Mirror direction" msgstr "" diff --git a/spec/factories/deploy_tokens.rb b/spec/factories/deploy_tokens.rb index a96258f5cbe..99486acc2ab 100644 --- a/spec/factories/deploy_tokens.rb +++ b/spec/factories/deploy_tokens.rb @@ -2,7 +2,8 @@ FactoryBot.define do factory :deploy_token do - token { SecureRandom.hex(50) } + token nil + token_encrypted { Gitlab::CryptoHelper.aes256_gcm_encrypt( SecureRandom.hex(50) ) } sequence(:name) { |n| "PDT #{n}" } read_repository true read_registry true diff --git a/spec/features/oauth_login_spec.rb b/spec/features/oauth_login_spec.rb index a47eaa9bda7..c6e69fa3fb0 100644 --- a/spec/features/oauth_login_spec.rb +++ b/spec/features/oauth_login_spec.rb @@ -55,6 +55,18 @@ describe 'OAuth Login', :js, :allow_forgery_protection do expect(current_path).to eq root_path end + + it 'when bypass-two-factor is enabled' do + allow(Gitlab.config.omniauth).to receive_messages(allow_bypass_two_factor: true) + login_via(provider.to_s, user, uid, remember_me: false) + expect(current_path).to eq root_path + end + + it 'when bypass-two-factor is disabled' do + allow(Gitlab.config.omniauth).to receive_messages(allow_bypass_two_factor: false) + login_with_provider(provider, enter_two_factor: true) + expect(current_path).to eq root_path + end end context 'when "remember me" is checked' do diff --git a/spec/frontend/tracking_spec.js b/spec/frontend/tracking_spec.js index 7c98a1a66c9..0e862c683d3 100644 --- a/spec/frontend/tracking_spec.js +++ b/spec/frontend/tracking_spec.js @@ -1,56 +1,20 @@ import $ from 'jquery'; import { setHTMLFixture } from './helpers/fixtures'; -import Tracking, { initUserTracking } from '~/tracking'; +import Tracking from '~/tracking'; describe('Tracking', () => { - let snowplowSpy; - beforeEach(() => { window.snowplow = window.snowplow || (() => {}); - window.snowplowOptions = { - namespace: '_namespace_', - hostname: 'app.gitfoo.com', - cookieDomain: '.gitfoo.com', - }; - snowplowSpy = jest.spyOn(window, 'snowplow'); }); - describe('initUserTracking', () => { - it('calls through to get a new tracker with the expected options', () => { - initUserTracking(); - expect(snowplowSpy).toHaveBeenCalledWith('newTracker', '_namespace_', 'app.gitfoo.com', { - namespace: '_namespace_', - hostname: 'app.gitfoo.com', - cookieDomain: '.gitfoo.com', - appId: '', - userFingerprint: false, - respectDoNotTrack: true, - forceSecureTracker: true, - eventMethod: 'post', - contexts: { webPage: true }, - activityTrackingEnabled: false, - pageTrackingEnabled: false, - }); - }); - - it('should activate features based on what has been enabled', () => { - initUserTracking(); - expect(snowplowSpy).not.toHaveBeenCalledWith('enableActivityTracking', 30, 30); - expect(snowplowSpy).not.toHaveBeenCalledWith('trackPageView'); - - window.snowplowOptions = Object.assign({}, window.snowplowOptions, { - activityTrackingEnabled: true, - pageTrackingEnabled: true, - }); + describe('.event', () => { + let snowplowSpy = null; - initUserTracking(); - expect(snowplowSpy).toHaveBeenCalledWith('enableActivityTracking', 30, 30); - expect(snowplowSpy).toHaveBeenCalledWith('trackPageView'); + beforeEach(() => { + snowplowSpy = jest.spyOn(window, 'snowplow'); }); - }); - describe('.event', () => { afterEach(() => { window.doNotTrack = undefined; navigator.doNotTrack = undefined; diff --git a/spec/lib/gitlab/auth/o_auth/user_spec.rb b/spec/lib/gitlab/auth/o_auth/user_spec.rb index a9b15c411dc..1e3da4f7c2d 100644 --- a/spec/lib/gitlab/auth/o_auth/user_spec.rb +++ b/spec/lib/gitlab/auth/o_auth/user_spec.rb @@ -787,11 +787,25 @@ describe Gitlab::Auth::OAuth::User do end end - describe '#bypass_two_factor?' do - subject { oauth_user.bypass_two_factor? } + describe "#bypass_two_factor?" do + it "when with allow_bypass_two_factor disabled (Default)" do + stub_omniauth_config(allow_bypass_two_factor: false) + expect(oauth_user.bypass_two_factor?).to be_falsey + end + + it "when with allow_bypass_two_factor enabled" do + stub_omniauth_config(allow_bypass_two_factor: true) + expect(oauth_user.bypass_two_factor?).to be_truthy + end + + it "when provider in allow_bypass_two_factor array" do + stub_omniauth_config(allow_bypass_two_factor: [provider]) + expect(oauth_user.bypass_two_factor?).to be_truthy + end - it 'returns always false' do - is_expected.to be_falsey + it "when provider not in allow_bypass_two_factor array" do + stub_omniauth_config(allow_bypass_two_factor: ["foo"]) + expect(oauth_user.bypass_two_factor?).to be_falsey end end end diff --git a/spec/lib/gitlab/snowplow_tracker_spec.rb b/spec/lib/gitlab/snowplow_tracker_spec.rb new file mode 100644 index 00000000000..073a33e5973 --- /dev/null +++ b/spec/lib/gitlab/snowplow_tracker_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe Gitlab::SnowplowTracker do + let(:timestamp) { Time.utc(2017, 3, 22) } + + around do |example| + Timecop.freeze(timestamp) { example.run } + end + + subject { described_class.track_event('epics', 'action', property: 'what', value: 'doit') } + + context '.track_event' do + context 'when Snowplow tracker is disabled' do + it 'does not track the event' do + expect(SnowplowTracker::Tracker).not_to receive(:new) + + subject + end + end + + context 'when Snowplow tracker is enabled' do + before do + stub_application_setting(snowplow_enabled: true) + stub_application_setting(snowplow_site_id: 'awesome gitlab') + stub_application_setting(snowplow_collector_hostname: 'url.com') + end + + it 'tracks the event' do + tracker = double + + expect(::SnowplowTracker::Tracker).to receive(:new) + .with( + an_instance_of(::SnowplowTracker::Emitter), + an_instance_of(::SnowplowTracker::Subject), + 'cf', 'awesome gitlab' + ).and_return(tracker) + expect(tracker).to receive(:track_struct_event) + .with('epics', 'action', nil, 'what', 'doit', nil, timestamp.to_i) + + subject + end + end + end +end diff --git a/spec/lib/gitlab/tracking_spec.rb b/spec/lib/gitlab/tracking_spec.rb deleted file mode 100644 index f14e74427e1..00000000000 --- a/spec/lib/gitlab/tracking_spec.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: true -require 'spec_helper' - -describe Gitlab::Tracking do - let(:timestamp) { Time.utc(2017, 3, 22) } - - before do - stub_application_setting(snowplow_enabled: true) - stub_application_setting(snowplow_collector_hostname: 'gitfoo.com') - stub_application_setting(snowplow_cookie_domain: '.gitfoo.com') - stub_application_setting(snowplow_site_id: '_abc123_') - end - - describe '.snowplow_options' do - subject(&method(:described_class)) - - it 'returns useful client options' do - expect(subject.snowplow_options(nil)).to eq( - namespace: 'gl', - hostname: 'gitfoo.com', - cookieDomain: '.gitfoo.com', - appId: '_abc123_', - pageTrackingEnabled: true, - activityTrackingEnabled: true - ) - end - - it 'enables features using feature flags' do - stub_feature_flags(additional_snowplow_tracking: true) - allow(Feature).to receive(:enabled?).with( - :additional_snowplow_tracking, - '_group_' - ).and_return(false) - - expect(subject.snowplow_options('_group_')).to include( - pageTrackingEnabled: false, - activityTrackingEnabled: false - ) - end - end - - describe '.event' do - subject(&method(:described_class)) - - around do |example| - Timecop.freeze(timestamp) { example.run } - end - - it 'can track events' do - tracker = double - - expect(SnowplowTracker::Emitter).to receive(:new).with( - 'gitfoo.com' - ).and_return('_emitter_') - - expect(SnowplowTracker::Tracker).to receive(:new).with( - '_emitter_', - an_instance_of(SnowplowTracker::Subject), - 'gl', - '_abc123_' - ).and_return(tracker) - - expect(tracker).to receive(:track_struct_event).with( - 'category', - 'action', - '_label_', - '_property_', - '_value_', - '_context_', - timestamp.to_i - ) - - subject.event('category', 'action', - label: '_label_', - property: '_property_', - value: '_value_', - context: '_context_' - ) - end - - it 'does not track when not enabled' do - stub_application_setting(snowplow_enabled: false) - expect(SnowplowTracker::Tracker).not_to receive(:new) - - subject.event('epics', 'action', property: 'what', value: 'doit') - end - end -end diff --git a/spec/migrations/encrypt_deploy_tokens_tokens_spec.rb b/spec/migrations/encrypt_deploy_tokens_tokens_spec.rb new file mode 100644 index 00000000000..a398e079731 --- /dev/null +++ b/spec/migrations/encrypt_deploy_tokens_tokens_spec.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require Rails.root.join('db', 'post_migrate', '20190711201818_encrypt_deploy_tokens_tokens.rb') + +describe EncryptDeployTokensTokens, :migration do + let(:migration) { described_class.new } + let(:deployment_tokens) { table(:deploy_tokens) } + let(:plaintext) { "secret-token" } + let(:expires_at) { DateTime.now + 1.year } + let(:ciphertext) { Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext) } + + describe '#up' do + it 'keeps plaintext token the same and populates token_encrypted if not present' do + deploy_token = deployment_tokens.create!( + name: 'test_token', + read_repository: true, + expires_at: expires_at, + username: 'gitlab-token-1', + token: plaintext + ) + + migration.up + + expect(deploy_token.reload.token).to eq(plaintext) + expect(deploy_token.reload.token_encrypted).to eq(ciphertext) + end + end + + describe '#down' do + it 'decrypts encrypted token and saves it' do + deploy_token = deployment_tokens.create!( + name: 'test_token', + read_repository: true, + expires_at: expires_at, + username: 'gitlab-token-1', + token_encrypted: ciphertext + ) + + migration.down + + expect(deploy_token.reload.token).to eq(plaintext) + expect(deploy_token.reload.token_encrypted).to eq(ciphertext) + end + end +end diff --git a/spec/policies/project_policy_spec.rb b/spec/policies/project_policy_spec.rb index 8fd54e0bf1d..71ba73d5661 100644 --- a/spec/policies/project_policy_spec.rb +++ b/spec/policies/project_policy_spec.rb @@ -94,6 +94,19 @@ describe ProjectPolicy do permissions.each { |p| is_expected.not_to be_allowed(p) } end + context 'with no project feature' do + subject { described_class.new(owner, project) } + + before do + project.project_feature.destroy + project.reload + end + + it 'returns false' do + is_expected.to be_disallowed(:read_build) + end + end + it 'does not include the read_issue permission when the issue author is not a member of the private project' do project = create(:project, :private) issue = create(:issue, project: project, author: create(:user)) diff --git a/spec/views/layouts/_head.html.haml_spec.rb b/spec/views/layouts/_head.html.haml_spec.rb index 70cdc08b4b6..d7f24950e6f 100644 --- a/spec/views/layouts/_head.html.haml_spec.rb +++ b/spec/views/layouts/_head.html.haml_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' describe 'layouts/_head' do + include StubConfiguration + before do allow(view).to receive(:current_application_settings).and_return(Gitlab::CurrentSettings.current_application_settings) end @@ -87,6 +89,24 @@ describe 'layouts/_head' do end end + context 'when a Piwik config is set' do + let(:piwik_host) { 'piwik.example.com' } + + before do + stub_config(extra: { + piwik_url: piwik_host, + piwik_site_id: 12345 + }) + end + + it 'add a Piwik Javascript' do + render + + expect(rendered).to match(/<script.*>.*var u="\/\/#{piwik_host}\/".*<\/script>/m) + expect(rendered).to match(%r(<noscript>.*<img src="//#{piwik_host}/piwik.php.*</noscript>)) + end + end + def stub_helper_with_safe_string(method) allow_any_instance_of(PageLayoutHelper).to receive(method) .and_return(%q{foo" http-equiv="refresh}.html_safe) |