diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-17 14:16:34 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-17 14:16:34 +0000 |
commit | 4ab54c2233e91f60a80e5b6fa2181e6899fdcc3e (patch) | |
tree | 2b256ff8dfe63dafe7f42b0d995f9e74fd1dc48b /.gitlab | |
parent | bd860c22f6a4b9473cbddd34a53eead8235a7ea1 (diff) | |
download | gitlab-ce-4ab54c2233e91f60a80e5b6fa2181e6899fdcc3e.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to '.gitlab')
-rw-r--r-- | .gitlab/ci/cng.gitlab-ci.yml | 5 | ||||
-rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 39 | ||||
-rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 69 | ||||
-rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 95 | ||||
-rw-r--r-- | .gitlab/ci/memory.gitlab-ci.yml | 14 | ||||
-rw-r--r-- | .gitlab/ci/pages.gitlab-ci.yml | 7 | ||||
-rw-r--r-- | .gitlab/ci/qa.gitlab-ci.yml | 21 | ||||
-rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 214 | ||||
-rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 227 | ||||
-rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 57 | ||||
-rw-r--r-- | .gitlab/ci/setup.gitlab-ci.yml | 14 | ||||
-rw-r--r-- | .gitlab/ci/test-metadata.gitlab-ci.yml | 30 | ||||
-rw-r--r-- | .gitlab/ci/yaml.gitlab-ci.yml | 5 |
13 files changed, 466 insertions, 331 deletions
diff --git a/.gitlab/ci/cng.gitlab-ci.yml b/.gitlab/ci/cng.gitlab-ci.yml index 3ba3b221df0..35859a1ab33 100644 --- a/.gitlab/ci/cng.gitlab-ci.yml +++ b/.gitlab/ci/cng.gitlab-ci.yml @@ -11,5 +11,6 @@ cloud-native-image: - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng only: refs: - - tags@gitlab-org/gitlab-foss - - tags@gitlab-org/gitlab + - tags + variables: + - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index 22e752ae0b0..87023da3c24 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -2,6 +2,12 @@ extends: - .default-tags - .default-retry + - .only-docs-changes + only: + refs: + - merge_requests + variables: + - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" image: ruby:2.6-alpine stage: review dependencies: [] @@ -19,55 +25,32 @@ - apk add --update openssl - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/trigger-build-docs - chmod 755 trigger-build-docs - -# Trigger a manual docs build in gitlab-docs only on non docs-only branches. -# Useful to preview the docs changes live. -review-docs-deploy-manual: - extends: - - .review-docs - - .except-docs-qa - script: - gem install gitlab --no-document - - ./trigger-build-docs deploy - when: manual - only: - - branches@gitlab-org/gitlab-foss - - branches@gitlab-org/gitlab # Always trigger a docs build in gitlab-docs only on docs-only branches. # Useful to preview the docs changes live. review-docs-deploy: - extends: - - .review-docs - - .except-qa + extends: .review-docs script: - - gem install gitlab --no-document - ./trigger-build-docs deploy - only: - - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-foss - - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab + when: manual # Cleanup remote environment of gitlab-docs review-docs-cleanup: - extends: - - .review-docs - - .except-qa + extends: .review-docs environment: name: review-docs/$CI_COMMIT_REF_SLUG action: stop script: - - gem install gitlab --no-document - ./trigger-build-docs cleanup when: manual - only: - - branches@gitlab-org/gitlab-foss - - branches@gitlab-org/gitlab docs lint: extends: - .default-tags - .default-retry - - .except-qa + - .default-only + - .only-docs-changes image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint" stage: test dependencies: [] diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index a61601597c3..f72396e4edf 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -9,9 +9,10 @@ extends: - .default-tags - .default-retry - - .assets-compile-cache + - .default-only - .default-before_script - - .except-docs + - .assets-compile-cache + - .only-code-qa-changes image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1 stage: test dependencies: ["setup-test-env"] @@ -45,10 +46,9 @@ - scripts/clean-old-cached-assets - rm -f /etc/apt/sources.list.d/google*.list # We don't need to update Chrome here only: - - /.+/@gitlab-org/gitlab-foss - - /.+/@gitlab-org/gitlab - - /.+/@gitlab/gitlabhq - - /.+/@gitlab/gitlab-ee + variables: + - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" + - $CI_SERVER_HOST == "dev.gitlab.org" tags: - gitlab-org - docker @@ -57,8 +57,7 @@ gitlab:assets:compile: extends: .gitlab:assets:compile-metadata only: refs: - - master@gitlab-org/gitlab-foss - - master@gitlab-org/gitlab + - master cache: policy: pull-push @@ -66,9 +65,7 @@ gitlab:assets:compile pull-cache: extends: .gitlab:assets:compile-metadata except: refs: - - master@gitlab-org/gitlab-foss - - master@gitlab-org/gitlab - - /(^docs[\/-].+|.+-docs$)/ + - master cache: policy: pull @@ -76,8 +73,10 @@ gitlab:assets:compile pull-cache: extends: - .default-tags - .default-retry - - .assets-compile-cache + - .default-only - .default-before_script + - .assets-compile-cache + - .only-code-qa-changes - .use-pg stage: prepare script: @@ -98,12 +97,10 @@ gitlab:assets:compile pull-cache: - public/assets compile-assets: - extends: - - .compile-assets-metadata + extends: .compile-assets-metadata only: refs: - - master@gitlab-org/gitlab-foss - - master@gitlab-org/gitlab + - master cache: policy: pull-push @@ -111,21 +108,23 @@ compile-assets pull-cache: extends: .compile-assets-metadata except: refs: - - master@gitlab-org/gitlab-foss - - master@gitlab-org/gitlab - - /(^docs[\/-].+|.+-docs$)/ + - master cache: policy: pull -karma: +.only-code-frontend-job-base: extends: - .default-tags - .default-retry - .default-cache + - .default-only - .default-before_script + - .only-code-changes - .use-pg - - .except-docs dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"] + +karma: + extends: .only-code-frontend-job-base variables: # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 @@ -148,14 +147,7 @@ karma: junit: junit_karma.xml jest: - extends: - - .default-tags - - .default-retry - - .default-cache - - .default-before_script - - .use-pg - - .except-docs-qa - dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"] + extends: .only-code-frontend-job-base script: - scripts/gitaly-test-spawn - date @@ -178,27 +170,26 @@ jest: - tmp/jest/jest/ policy: pull-push -.qa: +.qa-job-base: extends: - .default-tags - .default-retry - .default-cache - - .except-docs + - .default-only + - .only-code-qa-changes dependencies: [] stage: test - variables: - SETUP_DB: "false" before_script: - cd qa/ - bundle install qa:internal: - extends: .qa + extends: .qa-job-base script: - bundle exec rspec qa:selectors: - extends: .qa + extends: .qa-job-base script: - bundle exec bin/qa Test::Sanity::Selectors @@ -207,7 +198,8 @@ qa:selectors: - .default-tags - .default-retry - .default-cache - - .except-docs + - .default-only + - .only-code-changes dependencies: [] cache: key: "$CI_JOB_NAME" @@ -238,10 +230,9 @@ webpack-dev-server: - .default-tags - .default-retry - .default-cache - - .except-docs-qa - dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"] + - .only-code-changes + dependencies: ["setup-test-env", "compile-assets", "compile-assets pull-cache"] variables: - SETUP_DB: "false" WEBPACK_MEMORY_TEST: "true" script: - node --version diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 8287390c80e..94d7d2cdaa4 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -27,40 +27,83 @@ - vendor/gitaly-ruby policy: pull -.except-docs: - except: +.default-only: + only: refs: - - /(^docs[\/-].+|.+-docs$)/ + - master + - /^[\d-]+-stable(-ee)?$/ + - /^\d+-\d+-auto-deploy-\d+$/ + - merge_requests + - tags -.except-qa: - except: - refs: - - /(^qa[\/-].*|.*-qa$)/ +.only-code-changes: + only: + changes: + - ".gitlab/ci/**/*" + - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" + - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" + - ".csscomb.json" + - "Dangerfile" + - "Dockerfile.assets" + - "*_VERSION" + - "Gemfile{,.lock}" + - "Rakefile" + - "{babel.config,jest.config}.js" + - "config.ru" + - "{package.json,yarn.lock}" + - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,public,rubocop,scripts,spec,symbol,vendor}/**/*" + - "doc/README.md" # Some RSpec test rely on this file -.except-docs-qa: - except: - refs: - - /(^docs[\/-].+|.+-docs$)/ - - /(^qa[\/-].*|.*-qa$)/ +.only-qa-changes: + only: + changes: + - ".dockerignore" + - "qa/**/*" -.except-docs-qa-geo: - except: - refs: - - /(^docs[\/-].+|.+-docs$)/ - - /(^qa[\/-].*|.*-qa$)/ - - /(^geo[\/-].*|.*-geo$)/ +.only-docs-changes: + only: + changes: + - ".gitlab/route-map.yml" + - "doc/**/*" + - ".markdownlint.json" -.review-only: +.only-code-qa-changes: only: - refs: - - branches@gitlab-org/gitlab-foss - - branches@gitlab-org/gitlab + changes: + - ".gitlab/ci/**/*" + - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" + - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" + - ".csscomb.json" + - "Dangerfile" + - "Dockerfile.assets" + - "*_VERSION" + - "Gemfile{,.lock}" + - "Rakefile" + - "{babel.config,jest.config}.js" + - "config.ru" + - "{package.json,yarn.lock}" + - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,public,rubocop,scripts,spec,symbol,vendor}/**/*" + - "doc/README.md" # Some RSpec test rely on this file + - ".dockerignore" + - "qa/**/*" + +.only-review: + only: + variables: + - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" kubernetes: active except: refs: - master - /^\d+-\d+-auto-deploy-\d+$/ - - /(^docs[\/-].+|.+-docs$)/ + +.only-review-schedules: + only: + refs: + - schedules + variables: + - $REVIEW_APP_CLEANUP && $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" + kubernetes: active .use-pg: services: @@ -74,3 +117,9 @@ - name: postgres:10.9 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:alpine + +.only-ee: + only: + variables: + - $CI_PROJECT_NAME == "gitlab-ee" + - $CI_PROJECT_NAME == "gitlab" # New name of gitlab-ee after the single codebase migration diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml index 1936933cca4..d990c7eefa2 100644 --- a/.gitlab/ci/memory.gitlab-ci.yml +++ b/.gitlab/ci/memory.gitlab-ci.yml @@ -1,10 +1,14 @@ -memory-static: +.only-code-memory-job-base: extends: - .default-tags - .default-retry - .default-cache + - .default-only - .default-before_script - - .except-docs + - .only-code-changes + +memory-static: + extends: .only-code-memory-job-base variables: SETUP_DB: "false" script: @@ -31,12 +35,8 @@ memory-static: # All tests are run without a webserver (directly using Rack::Mock by default). memory-on-boot: extends: - - .default-tags - - .default-retry - - .default-cache - - .default-before_script + - .only-code-memory-job-base - .use-pg-10 - - .except-docs-qa variables: NODE_ENV: "production" RAILS_ENV: "production" diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index cf77ec24090..47f4117e34a 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -3,11 +3,12 @@ pages: - .default-tags - .default-retry - .default-cache - - .except-docs + - .default-only only: refs: - - master@gitlab-org/gitlab-foss - - master@gitlab-org/gitlab + - master + variables: + - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" stage: pages dependencies: ["coverage", "karma", "gitlab:assets:compile"] script: diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index a868316c353..e7362f913a9 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -1,4 +1,5 @@ .package-and-qa-base: + extends: .default-only image: ruby:2.6-alpine stage: qa dependencies: [] @@ -10,17 +11,16 @@ - install_gitlab_gem - ./scripts/trigger-build omnibus only: - refs: - - branches@gitlab-org/gitlab-foss - - branches@gitlab-org/gitlab + variables: + - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" package-and-qa-manual: - extends: .package-and-qa-base + extends: + - .package-and-qa-base + - .only-code-changes except: refs: - master - - /(^docs[\/-].+|.+-docs$)/ - - /(^qa[\/-].*|.*-qa$)/ when: manual needs: ["build-qa-image", "gitlab:assets:compile pull-cache"] @@ -34,10 +34,11 @@ package-and-qa-manual:master: needs: ["build-qa-image", "gitlab:assets:compile"] package-and-qa: - extends: .package-and-qa-base - only: + extends: + - .package-and-qa-base + - .only-qa-changes + except: refs: - - /(^qa[\/-].*|.*-qa$)/@gitlab-org/gitlab-foss - - /(^qa[\/-].*|.*-qa$)/@gitlab-org/gitlab + - master needs: ["build-qa-image", "gitlab:assets:compile pull-cache"] allow_failure: true diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 2363c0e5f8a..e9222c4b719 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -1,36 +1,40 @@ -.only-schedules-master: +.only-master: only: refs: - - schedules@gitlab-org/gitlab-foss - - schedules@gitlab-org/gitlab - - master@gitlab-org/gitlab-foss - - master@gitlab-org/gitlab - - master@gitlab/gitlabhq - - master@gitlab/gitlab-ee - -.only-gitlab-ee: - only: - - branches@gitlab-org/gitlab - - tags@gitlab-org/gitlab + - master .rake-exec: extends: - .default-tags - .default-retry - .default-cache + - .default-only - .default-before_script variables: SETUP_DB: "false" script: - bundle exec rake $CI_JOB_NAME -.rspec-base: +.only-code-rails-job-base: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-only + - .default-before_script + - .only-code-changes + +.only-code-qa-rails-job-base: extends: - .default-tags - .default-retry - .default-cache + - .default-only - .default-before_script - - .except-docs-qa + - .only-code-qa-changes + +.rspec-base: + extends: .only-code-rails-job-base stage: test script: - JOB_NAME=( $CI_JOB_NAME ) @@ -79,12 +83,8 @@ setup-test-env: extends: - - .default-tags - - .default-retry - - .default-cache - - .default-before_script + - .only-code-qa-rails-job-base - .use-pg - - .except-docs stage: prepare script: - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init' @@ -111,19 +111,19 @@ rspec system pg: rspec unit pg-10: extends: - .rspec-base-pg-10 - - .only-schedules-master + - .only-master parallel: 20 rspec integration pg-10: extends: - .rspec-base-pg-10 - - .only-schedules-master + - .only-master parallel: 6 rspec system pg-10: extends: - .rspec-base-pg-10 - - .only-schedules-master + - .only-master parallel: 24 rspec-fast-spec-helper: @@ -133,9 +133,8 @@ rspec-fast-spec-helper: rspec quarantine pg: extends: - - .default-before_script - .rspec-base-pg - - .only-schedules-master + - .only-master script: - export NO_KNAPSACK=1 CACHE_CLASSES=true - scripts/gitaly-test-spawn @@ -143,12 +142,7 @@ rspec quarantine pg: allow_failure: true static-analysis: - extends: - - .default-tags - - .default-retry - - .default-cache - - .default-before_script - - .except-docs + extends: .only-code-qa-rails-job-base dependencies: ["setup-test-env", "compile-assets", "compile-assets pull-cache"] variables: SETUP_DB: "false" @@ -162,81 +156,48 @@ static-analysis: policy: pull-push downtime_check: - extends: .rake-exec + extends: + - .rake-exec + - .only-code-changes except: refs: - master - tags - - /^[\d-]+-stable(-ee)?$/ - - /(^docs[\/-].+|.+-docs$)/ - - /(^qa[\/-].*|.*-qa$)/ + variables: + - $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/ stage: test dependencies: ["setup-test-env"] needs: ["setup-test-env"] -ee_compat_check: - extends: .rake-exec - dependencies: [] - except: - refs: - - master - - tags - - branches@gitlab-org/gitlab - - branches@gitlab/gitlab-ee - - /^[\d-]+-stable(-ee)?$/ - - /(^docs[\/-].+|.+-docs$)/ - - /^security-/ - artifacts: - name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}" - when: always - expire_in: 10d - paths: - - ee_compat_check/patches/*.patch - -# DB migration, rollback, and seed jobs -db:migrate:reset: +.db-job-base: extends: - - .default-tags - - .default-retry - - .default-cache - - .default-before_script + - .only-code-rails-job-base - .use-pg - - .except-docs-qa stage: test dependencies: ["setup-test-env"] needs: ["setup-test-env"] + +# DB migration, rollback, and seed jobs +db:migrate:reset: + extends: .db-job-base script: - bundle exec rake db:migrate:reset db:check-schema: - extends: - - .default-tags - - .default-retry - - .default-cache - - .default-before_script - - .use-pg - - .except-docs-qa - stage: test - dependencies: ["setup-test-env"] - needs: ["setup-test-env"] + extends: .db-job-base script: - source scripts/schema_changed.sh db:migrate-from-v11.11.0: - extends: - - .default-tags - - .default-retry - - .default-cache - - .default-before_script - - .use-pg - - .except-docs-qa - stage: test - dependencies: ["setup-test-env"] - needs: ["setup-test-env"] + extends: .db-job-base variables: SETUP_DB: "false" script: - - git fetch https://gitlab.com/gitlab-org/gitlab.git v11.11.0-ee + - export PROJECT_TO_CHECKOUT="gitlab-foss" + - export TAG_TO_CHECKOUT="v11.11.0" + - '[[ ! -d "ee/" ]] || export PROJECT_TO_CHECKOUT="gitlab"' + - '[[ ! -d "ee/" ]] || export TAG_TO_CHECKOUT="v11.11.0-ee"' + - git fetch https://gitlab.com/gitlab-org/$PROJECT_TO_CHECKOUT.git $TAG_TO_CHECKOUT - git checkout -f FETCH_HEAD - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile - sed -i "s/gem 'bootsnap', '~> 1.0.0'/gem 'bootsnap'/" Gemfile @@ -254,31 +215,13 @@ db:migrate-from-v11.11.0: - bundle exec rake db:migrate db:rollback: - extends: - - .default-tags - - .default-retry - - .default-cache - - .default-before_script - - .use-pg - - .except-docs-qa - stage: test - dependencies: ["setup-test-env"] - needs: ["setup-test-env"] + extends: .db-job-base script: - bundle exec rake db:migrate VERSION=20180101160629 - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true gitlab:setup: - extends: - - .default-tags - - .default-retry - - .default-cache - - .default-before_script - - .use-pg - - .except-docs-qa - stage: test - dependencies: ["setup-test-env"] - needs: ["setup-test-env"] + extends: .db-job-base variables: SETUP_DB: "false" script: @@ -295,14 +238,7 @@ gitlab:setup: - log/development.log coverage: - # Don't include dedicated-no-docs-no-db-pull-cache-job here since we need to - # download artifacts from all the rspec jobs instead of from setup-test-env only - extends: - - .default-tags - - .default-retry - - .default-cache - - .default-before_script - - .except-docs-qa + extends: .only-code-rails-job-base cache: policy: pull variables: @@ -324,7 +260,7 @@ coverage: .rspec-base-ee: extends: - .rspec-base - - .only-gitlab-ee + - .only-ee script: - JOB_NAME=( $CI_JOB_NAME ) - TEST_TOOL=${JOB_NAME[0]} @@ -362,10 +298,9 @@ rspec system pg ee: extends: .rspec-base-pg-ee parallel: 5 -.rspec-base-pg-geo: - extends: - - .rspec-base - - .only-gitlab-ee +.rspec-base-geo: + extends: .rspec-base-ee + parallel: 3 script: - JOB_NAME=( $CI_JOB_NAME ) - TEST_TOOL=${JOB_NAME[0]} @@ -382,33 +317,37 @@ rspec system pg ee: rspec geo pg ee: extends: - - .rspec-base-pg-geo + - .rspec-base-geo - .use-pg - - .except-docs-qa-geo - parallel: 3 + except: + variables: + - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/ rspec geo pg-10 ee: extends: - - .rspec-base-pg-geo + - .rspec-base-geo - .use-pg-10 - - .except-docs-qa-geo - parallel: 3 + except: + variables: + - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/ quick-rspec geo pg ee: extends: - - .rspec-base-pg-geo + - .rspec-base-geo - .use-pg stage: quick-test only: - - /(^geo[\/-].*|.*-geo$)/ + variables: + - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/ quick-rspec geo pg-10 ee: extends: - - .rspec-base-pg-geo + - .rspec-base-geo - .use-pg-10 stage: quick-test only: - - /(^geo[\/-].*|.*-geo$)/ + variables: + - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/ rspec quarantine pg ee: extends: rspec quarantine pg @@ -417,35 +356,10 @@ rspec quarantine pg ee: - scripts/gitaly-test-spawn - bin/rspec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag quarantine -- ee/spec/ -migration:upgrade-pg-ce-to-ee: - extends: - - .default-tags - - .default-retry - - .default-cache - - .default-before_script - - .use-pg - - .except-docs-qa - dependencies: ["setup-test-env"] - variables: - SETUP_DB: "false" - script: - - ruby -r./scripts/ee_specific_check/ee_specific_check -e'EESpecificCheck.fetch_remote_ce_branch' - - git checkout -f FETCH_HEAD - - . scripts/utils.sh - - . scripts/prepare_build.sh - - date - - setup_db - - date - - git checkout -f $CI_COMMIT_SHA - - date - - . scripts/prepare_build.sh - - date - - bundle exec rake db:migrate - db:rollback geo: extends: - db:rollback - - .only-gitlab-ee + - .only-ee script: - bundle exec rake geo:db:migrate VERSION=20170627195211 - bundle exec rake geo:db:migrate diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 5622cd232ca..38da73034d5 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -1,39 +1,236 @@ -include: - - template: Code-Quality.gitlab-ci.yml - - template: Security/SAST.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/DAST.gitlab-ci.yml +# include: +# - template: Code-Quality.gitlab-ci.yml +# - template: Security/SAST.gitlab-ci.yml +# - template: Security/Dependency-Scanning.gitlab-ci.yml +# - template: Security/DAST.gitlab-ci.yml -.reports: +# We need to duplicate this job's definition because it seems it's impossible to +# override an included `only.refs`. +# See https://gitlab.com/gitlab-org/gitlab/issues/31371. +code_quality: extends: - .default-retry - - .except-docs - -code_quality: - extends: .reports + - .default-only + - .only-code-changes + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + script: + - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - docker run + --env SOURCE_CODE="$PWD" + --volume "$PWD":/code + --volume /var/run/docker.sock:/var/run/docker.sock + "registry.gitlab.com/gitlab-org/security-products/codequality:12-0-stable" /code + artifacts: + reports: + codequality: gl-code-quality-report.json + expire_in: 1 week + dependencies: [] + except: + variables: + - $CODE_QUALITY_DISABLED +# We need to duplicate this job's definition because it seems it's impossible to +# override an included `only.refs`. +# See https://gitlab.com/gitlab-org/gitlab/issues/31371. sast: - extends: .reports + extends: + - .default-retry + - .default-only + - .only-code-changes + stage: test + image: docker:stable variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" SAST_BRAKEMAN_LEVEL: 2 - SAST_EXCLUDED_PATHS: qa,spec,doc + SAST_EXCLUDED_PATHS: qa,spec,doc,ee/spec + allow_failure: true + services: + - docker:stable-dind + script: + - export SAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} + - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | + docker run \ + $(propagate_env_vars \ + SAST_BANDIT_EXCLUDED_PATHS \ + SAST_ANALYZER_IMAGES \ + SAST_ANALYZER_IMAGE_PREFIX \ + SAST_ANALYZER_IMAGE_TAG \ + SAST_DEFAULT_ANALYZERS \ + SAST_PULL_ANALYZER_IMAGES \ + SAST_BRAKEMAN_LEVEL \ + SAST_FLAWFINDER_LEVEL \ + SAST_GITLEAKS_ENTROPY_LEVEL \ + SAST_GOSEC_LEVEL \ + SAST_EXCLUDED_PATHS \ + SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ + SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ + SAST_RUN_ANALYZER_TIMEOUT \ + SAST_JAVA_VERSION \ + ANT_HOME \ + ANT_PATH \ + GRADLE_PATH \ + JAVA_OPTS \ + JAVA_PATH \ + JAVA_8_VERSION \ + JAVA_11_VERSION \ + MAVEN_CLI_OPTS \ + MAVEN_PATH \ + MAVEN_REPO_PATH \ + SBT_PATH \ + FAIL_NEVER \ + ) \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code artifacts: expire_in: 7 days paths: - gl-sast-report.json + reports: + sast: gl-sast-report.json + dependencies: [] + only: + variables: + - $GITLAB_FEATURES =~ /\bsast\b/ + except: + variables: + - $SAST_DISABLED +# We need to duplicate this job's definition because it seems it's impossible to +# override an included `only.refs`. +# See https://gitlab.com/gitlab-org/gitlab/issues/31371. dependency_scanning: - extends: .reports + extends: + - .default-retry + - .default-only + - .only-code-changes + stage: test + image: docker:stable + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + allow_failure: true + services: + - docker:stable-dind + script: + - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} + - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | + docker run \ + $(propagate_env_vars \ + DS_ANALYZER_IMAGES \ + DS_ANALYZER_IMAGE_PREFIX \ + DS_ANALYZER_IMAGE_TAG \ + DS_DEFAULT_ANALYZERS \ + DS_EXCLUDED_PATHS \ + DEP_SCAN_DISABLE_REMOTE_CHECKS \ + DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ + DS_PULL_ANALYZER_IMAGE_TIMEOUT \ + DS_RUN_ANALYZER_TIMEOUT \ + DS_PYTHON_VERSION \ + DS_PIP_DEPENDENCY_PATH \ + PIP_INDEX_URL \ + PIP_EXTRA_INDEX_URL \ + ) \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_VERSION" /code + artifacts: + reports: + dependency_scanning: gl-dependency-scanning-report.json + dependencies: [] + only: + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + except: + variables: + - $DEPENDENCY_SCANNING_DISABLED + +# We need to duplicate this job's definition because it seems it's impossible to +# override an included `only.refs`. +# See https://gitlab.com/gitlab-org/gitlab/issues/31371. dast: extends: - - .reports - - .review-only + - .default-retry + - .default-only + - .only-code-qa-changes + - .only-review stage: qa dependencies: ["review-deploy"] before_script: - export DAST_WEBSITE="$(cat review_app_url.txt)" + image: + name: "registry.gitlab.com/gitlab-org/security-products/dast:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" + variables: + # URL to scan: + # DAST_WEBSITE: https://example.com/ + # + # Time limit for target availability (scan is attempted even when timeout): + # DAST_TARGET_AVAILABILITY_TIMEOUT: 60 + # + # Set these variables to scan with an authenticated user: + # DAST_AUTH_URL: https://example.com/sign-in + # DAST_USERNAME: john.doe@example.com + # DAST_PASSWORD: john-doe-password + # DAST_USERNAME_FIELD: session[user] # the name of username field at the sign-in HTML form + # DAST_PASSWORD_FIELD: session[password] # the name of password field at the sign-in HTML form + # DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional: URLs to skip during the authenticated scan; comma-separated, no spaces in between + # + # Perform ZAP Full Scan, which includes both passive and active scanning: + # DAST_FULL_SCAN_ENABLED: "true" + allow_failure: true + script: + - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} + - /analyze -t $DAST_WEBSITE artifacts: expire_in: 7 days paths: - gl-dast-report.json + reports: + dast: gl-dast-report.json + only: + variables: + - $GITLAB_FEATURES =~ /\bdast\b/ + except: + variables: + - $DAST_DISABLED diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index f35d3ce19a4..c7b79dac9dd 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -1,21 +1,10 @@ -.review-schedules-only: - only: - refs: - - schedules@gitlab-org/gitlab-foss - - schedules@gitlab-org/gitlab - kubernetes: active - variables: - - $REVIEW_APP_CLEANUP - except: - refs: - - tags - - /(^docs[\/-].+|.+-docs$)/ - .review-base: extends: - .default-tags - .default-retry - - .review-only + - .default-only + - .only-review + - .only-code-qa-changes image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base dependencies: [] before_script: @@ -25,6 +14,7 @@ extends: - .default-tags - .default-retry + - .default-only image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine services: - docker:19.03.0-dind @@ -39,12 +29,11 @@ build-qa-image: extends: - .review-docker - - .except-docs + - .only-code-qa-changes only: - refs: - - branches@gitlab-org/gitlab-foss - - branches@gitlab-org/gitlab - stage: test + variables: + - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" + stage: prepare script: - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/gitlab-${GITLAB_EDITION}-qa:${CI_COMMIT_REF_SLUG}" @@ -53,6 +42,9 @@ build-qa-image: - time docker push ${QA_IMAGE} .review-build-cng-base: + extends: + - .default-only + - .only-code-qa-changes image: ruby:2.6-alpine stage: review-prepare before_script: @@ -66,13 +58,13 @@ build-qa-image: review-build-cng: extends: - .review-build-cng-base - - .review-only + - .only-review needs: ["gitlab:assets:compile pull-cache"] schedule:review-build-cng: extends: - .review-build-cng-base - - .review-schedules-only + - .only-review-schedules needs: ["gitlab:assets:compile"] .review-deploy-base: @@ -118,7 +110,7 @@ review-deploy: schedule:review-deploy: extends: - .review-deploy-base - - .review-schedules-only + - .only-review-schedules needs: ["schedule:review-build-cng"] review-stop: @@ -153,7 +145,8 @@ review-cleanup-failed-deployment: .review-qa-base: extends: - .review-docker - - .review-only + - .only-review + - .only-code-qa-changes stage: qa variables: QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" @@ -200,7 +193,9 @@ review-qa-all: parallel-spec-reports: extends: - .default-tags - - .except-docs + - .default-only + - .only-code-qa-changes + - .only-review image: ruby:2.6-alpine stage: post-test dependencies: ["review-qa-all"] @@ -247,14 +242,14 @@ review-performance: schedule:review-performance: extends: - review-performance - - .review-schedules-only + - .only-review-schedules dependencies: ["schedule:review-deploy"] schedule:review-cleanup: extends: - .review-base - - .review-schedules-only - stage: build + - .only-review-schedules + stage: prepare allow_failure: true environment: name: review/auto-cleanup @@ -270,6 +265,7 @@ danger-review: - .default-tags - .default-retry - .default-cache + - .default-only image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger stage: test dependencies: [] @@ -279,10 +275,9 @@ danger-review: except: refs: - master - - /^\d+-\d+-auto-deploy-\d+$/ - - /^[\d-]+-stable(-ee)?$/ - - /^ce-to-ee-.*/ - - /.*-stable(-ee)?-prepare-.*/ + variables: + - $CI_COMMIT_REF_NAME =~ /^\d+-\d+-auto-deploy-\d+$/ + - $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/ script: - git version - node --version diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index d29907b3683..e673eb9c7f3 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -6,7 +6,6 @@ cache gems: - .default-retry - .default-cache - .default-before_script - - .except-docs stage: test dependencies: ["setup-test-env"] needs: ["setup-test-env"] @@ -19,15 +18,17 @@ cache gems: - vendor/cache only: refs: - - master@gitlab-org/gitlab-foss - - master@gitlab-org/gitlab + - master - tags + variables: + - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" .minimal-job: extends: - .default-tags - .default-retry - - .except-docs-qa + - .default-only + - .only-code-changes dependencies: [] gitlab_git_test: @@ -40,5 +41,6 @@ no_ee_check: script: - scripts/no-ee-check only: - refs: - - branches@gitlab-org/gitlab-foss + variables: + - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAME == "gitlab-foss" + - $CI_SERVER_HOST == "dev.gitlab.org" && $CI_PROJECT_NAME == "gitlabhq" diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index 14c230deb47..fe3cde2bb16 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -1,4 +1,5 @@ .tests-metadata-state: + extends: .default-only variables: TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache" before_script: @@ -13,7 +14,7 @@ retrieve-tests-metadata: extends: - .tests-metadata-state - - .except-docs-qa + - .only-code-changes stage: prepare cache: key: tests_metadata @@ -26,8 +27,8 @@ retrieve-tests-metadata: - mkdir -p rspec_profiling/ - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}' - - wget -O $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH - - '[[ -f $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH}' + - '[[ ! -d "ee/" ]] || wget -O $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH' + - '[[ ! -d "ee/" ]] || [[ -f $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH}' update-tests-metadata: extends: .tests-metadata-state @@ -43,9 +44,9 @@ update-tests-metadata: - echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_*_pg_node_*.json - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH' - - echo "{}" > ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} - - scripts/merge-reports ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_*_pg_ee_*node_*.json - - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH' + - '[[ ! -d "ee/" ]] || echo "{}" > ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH}' + - '[[ ! -d "ee/" ]] || scripts/merge-reports ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_*_pg_ee_*node_*.json' + - '[[ ! -d "ee/" ]] || [[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH' - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH} @@ -54,15 +55,17 @@ update-tests-metadata: - scripts/insert-rspec-profiling-data only: refs: - - master@gitlab-org/gitlab-foss - - master@gitlab-org/gitlab - - master@gitlab/gitlabhq - - master@gitlab/gitlab-ee + - master + variables: + - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" + - $CI_SERVER_HOST == "dev.gitlab.org" flaky-examples-check: extends: - .default-tags - .default-retry + - .default-only + - .only-code-changes image: ruby:2.6-alpine stage: post-test variables: @@ -70,12 +73,7 @@ flaky-examples-check: allow_failure: true only: refs: - - branches - except: - refs: - - master - - /(^docs[\/-].+|.+-docs$)/ - - /(^qa[\/-].*|.*-qa$)/ + - merge_requests artifacts: expire_in: 30d paths: diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml index dd61cb3f035..323f94b6d04 100644 --- a/.gitlab/ci/yaml.gitlab-ci.yml +++ b/.gitlab/ci/yaml.gitlab-ci.yml @@ -4,7 +4,10 @@ lint-ci-gitlab: extends: - .default-tags - .default-retry - - .except-docs + - .default-only + only: + changes: + - "**/*.yml" image: sdesbure/yamllint:latest dependencies: [] script: |