diff options
Diffstat (limited to '.gitlab/ci')
| -rw-r--r-- | .gitlab/ci/cng.gitlab-ci.yml | 16 | ||||
| -rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 76 | ||||
| -rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 237 | ||||
| -rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 62 | ||||
| -rw-r--r-- | .gitlab/ci/pages.gitlab-ci.yml | 27 | ||||
| -rw-r--r-- | .gitlab/ci/qa.gitlab-ci.yml | 20 | ||||
| -rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 292 | ||||
| -rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 60 | ||||
| -rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 232 | ||||
| -rw-r--r-- | .gitlab/ci/setup.gitlab-ci.yml | 43 | ||||
| -rw-r--r-- | .gitlab/ci/test-metadata.gitlab-ci.yml | 82 |
11 files changed, 1147 insertions, 0 deletions
diff --git a/.gitlab/ci/cng.gitlab-ci.yml b/.gitlab/ci/cng.gitlab-ci.yml new file mode 100644 index 00000000000..e15f8ed91e0 --- /dev/null +++ b/.gitlab/ci/cng.gitlab-ci.yml @@ -0,0 +1,16 @@ +cloud-native-image: + image: ruby:2.5-alpine + before_script: [] + dependencies: [] + stage: post-test + allow_failure: true + variables: + GIT_DEPTH: "1" + cache: {} + when: manual + script: + - gem install gitlab --no-document + - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng + only: + - tags@gitlab-org/gitlab-ce + - tags@gitlab-org/gitlab-ee diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml new file mode 100644 index 00000000000..5aa1a856405 --- /dev/null +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -0,0 +1,76 @@ +.review-docs: &review-docs + extends: .single-script-job-dedicated-runner + variables: + SCRIPT_NAME: trigger-build-docs + environment: + name: review-docs/$CI_COMMIT_REF_SLUG + # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are CI variables + # Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693 + url: http://$CI_ENVIRONMENT_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX + on_stop: review-docs-cleanup + +# Trigger a manual docs build in gitlab-docs only on non docs-only branches. +# Useful to preview the docs changes live. +review-docs-deploy-manual: + <<: *review-docs + stage: build + script: + - gem install gitlab --no-document + - ./$SCRIPT_NAME deploy + when: manual + only: + - branches@gitlab-org/gitlab-ce + - branches@gitlab-org/gitlab-ee + except: + - /(^docs[\/-].*|.*-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ + +# Always trigger a docs build in gitlab-docs only on docs-only branches. +# Useful to preview the docs changes live. +review-docs-deploy: + <<: *review-docs + stage: post-test + script: + - gem install gitlab --no-document + - ./$SCRIPT_NAME deploy + only: + - /(^docs[\/-].*|.*-docs$)/@gitlab-org/gitlab-ce + - /(^docs[\/-].*|.*-docs$)/@gitlab-org/gitlab-ee + except: + - /(^qa[\/-].*|.*-qa$)/ + +# Cleanup remote environment of gitlab-docs +review-docs-cleanup: + <<: *review-docs + stage: post-cleanup + environment: + name: review-docs/$CI_COMMIT_REF_SLUG + action: stop + script: + - gem install gitlab --no-document + - ./$SCRIPT_NAME cleanup + when: manual + only: + - branches@gitlab-org/gitlab-ce + - branches@gitlab-org/gitlab-ee + +docs lint: + extends: .dedicated-runner + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint" + stage: test + cache: {} + dependencies: [] + before_script: [] + script: + - scripts/lint-doc.sh + - scripts/lint-changelog-yaml + - mv doc/ /tmp/gitlab-docs/content/$DOCS_GITLAB_REPO_SUFFIX + - cd /tmp/gitlab-docs + # Build HTML from Markdown + - bundle exec nanoc + # Check the internal links + - bundle exec nanoc check internal_links + # Check the internal anchor links + - bundle exec nanoc check internal_anchors + except: + - /(^qa[\/-].*|.*-qa$)/ diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml new file mode 100644 index 00000000000..e38b878972d --- /dev/null +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -0,0 +1,237 @@ +.assets-compile-cache: &assets-compile-cache + cache: + key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v5" + paths: + - vendor/ruby/ + - .yarn-cache/ + - tmp/cache/assets/sprockets + policy: pull-push + +.use-pg: &use-pg + services: + - name: postgres:9.6 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:alpine + +gitlab:assets:compile: + <<: *assets-compile-cache + extends: .dedicated-no-docs-pull-cache-job + image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.5.3-git-2.18-chrome-71.0-node-8.x-yarn-1.12-graphicsmagick-1.3.29-docker-18.06.1 + dependencies: + - setup-test-env + services: + - docker:stable-dind + variables: + NODE_ENV: "production" + RAILS_ENV: "production" + SETUP_DB: "false" + SKIP_STORAGE_VALIDATION: "true" + WEBPACK_REPORT: "true" + # we override the max_old_space_size to prevent OOM errors + NODE_OPTIONS: --max_old_space_size=3584 + DOCKER_DRIVER: overlay2 + DOCKER_HOST: tcp://docker:2375 + script: + - node --version + - yarn install --frozen-lockfile --production --cache-folder .yarn-cache + - free -m + - bundle exec rake gitlab:assets:compile + - time scripts/build_assets_image + - scripts/clean-old-cached-assets + artifacts: + name: webpack-report + expire_in: 31d + paths: + - webpack-report/ + - public/assets/ + only: + - /.+/@gitlab-org/gitlab-ce + - /.+/@gitlab-org/gitlab-ee + - /.+/@gitlab/gitlabhq + - /.+/@gitlab/gitlab-ee + tags: + - docker + - gitlab-org + +compile-assets: + extends: .dedicated-runner + <<: *use-pg + <<: *assets-compile-cache + stage: prepare + script: + - node --version + - yarn install --frozen-lockfile --cache-folder .yarn-cache + - free -m + - bundle exec rake gitlab:assets:compile + - scripts/clean-old-cached-assets + variables: + # we override the max_old_space_size to prevent OOM errors + NODE_OPTIONS: --max_old_space_size=3584 + artifacts: + expire_in: 7d + paths: + - node_modules + - public/assets + except: + refs: + - /(^docs[\/-].*|.*-docs$)/ + +gitlab:ui:visual: + extends: .dedicated-runner + before_script: [] + allow_failure: true + dependencies: + - compile-assets + script: + # Remove node modules from GitLab that may conflict with gitlab-ui + - rm -r node_modules + - git clone https://gitlab.com/gitlab-org/gitlab-ui.git + - cp public/assets/application-*.css gitlab-ui/styles/application.css + - cd gitlab-ui + - yarn install + - CSS_URL=./application.css yarn test + only: + changes: + - app/assets/stylesheets/*.scss + - app/assets/stylesheets/**/*.scss + - app/assets/stylesheets/**/**/*.scss + except: + refs: + - /(^docs[\/-].*|.*-docs$)/ + - master + variables: + - $CI_COMMIT_MESSAGE =~ /\[skip visual\]/i + artifacts: + paths: + - tests/__image_snapshots__/ + +karma: + extends: .dedicated-no-docs-pull-cache-job + <<: *use-pg + dependencies: + - compile-assets + - setup-test-env + variables: + # we override the max_old_space_size to prevent OOM errors + NODE_OPTIONS: --max_old_space_size=3584 + script: + - export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log + - date + - scripts/gitaly-test-spawn + - date + - bundle exec rake karma + coverage: '/^Statements *: (\d+\.\d+%)/' + artifacts: + name: coverage-javascript + expire_in: 31d + when: always + paths: + - chrome_debug.log + - coverage-javascript/ + reports: + junit: junit_karma.xml + +jest: + extends: .dedicated-no-docs-and-no-qa-pull-cache-job + <<: *use-pg + dependencies: + - compile-assets + - setup-test-env + script: + - scripts/gitaly-test-spawn + - date + - bundle exec rake karma:fixtures + - date + - yarn jest --ci --coverage + artifacts: + name: coverage-frontend + expire_in: 31d + when: always + paths: + - coverage-frontend/ + - junit_jest.xml + reports: + junit: junit_jest.xml + cache: + key: jest + paths: + - tmp/jest/jest/ + policy: pull-push + +qa:internal: + extends: .dedicated-no-docs-no-db-pull-cache-job + services: [] + script: + - cd qa/ + - bundle install + - bundle exec rspec + dependencies: + - setup-test-env + +qa:selectors: + extends: .dedicated-no-docs-no-db-pull-cache-job + services: [] + script: + - cd qa/ + - bundle install + - bundle exec bin/qa Test::Sanity::Selectors + dependencies: + - setup-test-env + +.qa-frontend-node: &qa-frontend-node + extends: .dedicated-no-docs-no-db-pull-cache-job + stage: test + cache: + key: "$CI_JOB_NAME" + paths: + - .yarn-cache/ + policy: pull-push + dependencies: [] + before_script: [] + script: + - date + - yarn install --frozen-lockfile --cache-folder .yarn-cache + - date + - yarn run webpack-prod + +qa-frontend-node:8: + <<: *qa-frontend-node + image: node:8-alpine + +qa-frontend-node:10: + <<: *qa-frontend-node + image: node:10-alpine + +qa-frontend-node:latest: + <<: *qa-frontend-node + image: node:alpine + allow_failure: true + +lint:javascript:report: + extends: .dedicated-no-docs-no-db-pull-cache-job + stage: post-test + dependencies: [] + before_script: [] + script: + - date + - yarn run eslint-report || true # ignore exit code + artifacts: + name: eslint-report + expire_in: 31d + paths: + - eslint-report.html + +jsdoc: + extends: .dedicated-no-docs-no-db-pull-cache-job + stage: post-test + dependencies: + - compile-assets + before_script: [] + script: + - date + - yarn run jsdoc || true # ignore exit code + artifacts: + name: jsdoc + expire_in: 31d + paths: + - jsdoc/
\ No newline at end of file diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml new file mode 100644 index 00000000000..c48672183b9 --- /dev/null +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -0,0 +1,62 @@ +.dedicated-runner: + retry: + max: 1 # This is confusing but this means "2 runs at max". + when: + - unknown_failure + - api_failure + - runner_system_failure + tags: + - gitlab-org + +.default-cache: &default-cache + key: "debian-stretch-ruby-2.5.3-node-10.x" + paths: + - vendor/ruby + - .yarn-cache/ + - vendor/gitaly-ruby + +.dedicated-runner-default-cache: + extends: .dedicated-runner + cache: + <<: *default-cache + +# Jobs that only need to pull cache +.dedicated-pull-cache-job: + extends: .dedicated-runner + cache: + <<: *default-cache + policy: pull + stage: test + +.dedicated-no-docs-pull-cache-job: + extends: .dedicated-pull-cache-job + except: + - /(^docs[\/-].*|.*-docs$)/ + +.dedicated-no-docs-and-no-qa-pull-cache-job: + extends: .dedicated-pull-cache-job + except: + - /(^docs[\/-].*|.*-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ + +# Jobs that do not need a DB +.dedicated-no-docs-no-db-pull-cache-job: + extends: .dedicated-no-docs-pull-cache-job + variables: + SETUP_DB: "false" + +.single-script-job-dedicated-runner: + extends: .dedicated-runner + image: ruby:2.5-alpine + stage: test + cache: {} + dependencies: [] + variables: + GIT_STRATEGY: none + before_script: + # We don't clone the repo by using GIT_STRATEGY: none and only download the + # single script we need here so it's much faster than cloning. + - export SCRIPT_NAME="${SCRIPT_NAME:-$CI_JOB_NAME}" + - apk add --update openssl + - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/$SCRIPT_NAME + - chmod 755 $(basename $SCRIPT_NAME) diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml new file mode 100644 index 00000000000..7d9136b8213 --- /dev/null +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -0,0 +1,27 @@ + +pages: + extends: .dedicated-no-docs-no-db-pull-cache-job + before_script: [] + stage: pages + dependencies: + - coverage + - karma + - gitlab:assets:compile + - lint:javascript:report + - jsdoc + script: + - mv public/ .public/ + - mkdir public/ + - mv coverage/ public/coverage-ruby/ || true + - mv coverage-javascript/ public/coverage-javascript/ || true + - mv eslint-report.html public/ || true + - mv webpack-report/ public/webpack-report/ || true + - cp .public/assets/application-*.css public/application.css || true + - cp .public/assets/application-*.css.gz public/application.css.gz || true + - mv jsdoc/ public/jsdoc/ || true + artifacts: + paths: + - public + only: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml new file mode 100644 index 00000000000..07b38c9aa85 --- /dev/null +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -0,0 +1,20 @@ +package-and-qa: + image: ruby:2.5-alpine + stage: test + before_script: [] + dependencies: [] + cache: {} + variables: + GIT_DEPTH: "1" + API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}" + retry: 0 + script: + - apk add --update openssl curl jq + - gem install gitlab --no-document + - source ./scripts/review_apps/review-apps.sh + - wait_for_job_to_be_done "gitlab:assets:compile" + - ./scripts/trigger-build omnibus + when: manual + only: + - /.+/@gitlab-org/gitlab-ce + - /.+/@gitlab-org/gitlab-ee diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml new file mode 100644 index 00000000000..b926fb77a08 --- /dev/null +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -0,0 +1,292 @@ +.use-pg: &use-pg + services: + - name: postgres:9.6 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:alpine + +.use-pg-10: &use-pg-10 + services: + - postgres:10.0 + - redis:alpine + +.use-mysql: &use-mysql + services: + - mysql:5.7 + - redis:alpine + +.only-schedules-master: &only-schedules-master + only: + - schedules@gitlab-org/gitlab-ce + - schedules@gitlab-org/gitlab-ee + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - master@gitlab/gitlabhq + - master@gitlab/gitlab-ee + +.gitlab-setup: &gitlab-setup + extends: .dedicated-no-docs-and-no-qa-pull-cache-job + <<: *use-pg + variables: + SETUP_DB: "false" + script: + # Manually clone gitlab-test and only seed this project in + # db/fixtures/development/04_project.rb thanks to SIZE=1 below + - git clone https://gitlab.com/gitlab-org/gitlab-test.git + /home/git/repositories/gitlab-org/gitlab-test.git + - scripts/gitaly-test-spawn + - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup + artifacts: + when: on_failure + expire_in: 1d + paths: + - log/development.log + +.rake-exec: &rake-exec + extends: .dedicated-no-docs-no-db-pull-cache-job + script: + - bundle exec rake $CI_JOB_NAME + +.rspec-metadata: &rspec-metadata + extends: .dedicated-pull-cache-job + stage: test + script: + - JOB_NAME=( $CI_JOB_NAME ) + - TEST_TOOL=${JOB_NAME[0]} + - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json + - export KNAPSACK_GENERATE_REPORT=true + - export SUITE_FLAKY_RSPEC_REPORT_PATH=${FLAKY_RSPEC_SUITE_REPORT_PATH} + - export FLAKY_RSPEC_REPORT_PATH=rspec_flaky/all_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json + - export NEW_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/new_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json + - export FLAKY_RSPEC_GENERATE_REPORT=true + - export CACHE_CLASSES=true + - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} + - '[[ -f $FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_REPORT_PATH}' + - '[[ -f $NEW_FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${NEW_FLAKY_RSPEC_REPORT_PATH}' + - scripts/gitaly-test-spawn + - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml" + artifacts: + expire_in: 31d + when: always + paths: + - coverage/ + - knapsack/ + - rspec_flaky/ + - rspec_profiling/ + - tmp/capybara/ + reports: + junit: junit_rspec.xml + except: + - /(^docs[\/-].*|.*-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ + +.rspec-metadata-pg: &rspec-metadata-pg + <<: *rspec-metadata + <<: *use-pg + +.rspec-metadata-pg-10: &rspec-metadata-pg-10 + <<: *rspec-metadata + <<: *use-pg-10 + image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.5.3-golang-1.11-git-2.18-chrome-71.0-node-10.x-yarn-1.12-postgresql-10-graphicsmagick-1.3.29" + +.rspec-metadata-mysql: &rspec-metadata-mysql + <<: *rspec-metadata + <<: *use-mysql + +# DB migration, rollback, and seed jobs +.db-migrate-reset: &db-migrate-reset + extends: .dedicated-no-docs-and-no-qa-pull-cache-job + script: + - bundle exec rake db:migrate:reset + dependencies: + - setup-test-env + +.migration-paths: &migration-paths + extends: .dedicated-no-docs-and-no-qa-pull-cache-job + variables: + SETUP_DB: "false" + script: + - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v9.3.0 + - git checkout -f FETCH_HEAD + - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile + - bundle update google-protobuf grpc + - bundle install $BUNDLE_INSTALL_FLAGS + - date + - cp config/gitlab.yml.example config/gitlab.yml + - bundle exec rake db:drop db:create db:schema:load db:seed_fu + - date + - git checkout -f $CI_COMMIT_SHA + - bundle install $BUNDLE_INSTALL_FLAGS + - date + - . scripts/prepare_build.sh + - date + - bundle exec rake db:migrate + dependencies: + - setup-test-env + +setup-test-env: + extends: .dedicated-runner-default-cache + <<: *use-pg + stage: prepare + script: + - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init' + - scripts/gitaly-test-build # Do not use 'bundle exec' here + artifacts: + expire_in: 7d + paths: + - tmp/tests + - config/secrets.yml + - vendor/gitaly-ruby + except: + - /(^docs[\/-].*|.*-docs$)/ + +rspec-pg: + <<: *rspec-metadata-pg + parallel: 50 + +rspec-pg-10: + <<: *rspec-metadata-pg-10 + <<: *only-schedules-master + parallel: 50 + +rspec-mysql: + <<: *rspec-metadata-mysql + <<: *only-schedules-master + parallel: 50 + +rspec-fast-spec-helper: + <<: *rspec-metadata-pg + script: + - bundle exec rspec spec/fast_spec_helper.rb + +.rspec-quarantine: &rspec-quarantine + <<: *only-schedules-master + script: + - export CACHE_CLASSES=true + - scripts/gitaly-test-spawn + - bin/rspec --color --format documentation --tag quarantine spec/ + +rspec-pg-quarantine: + <<: *rspec-metadata-pg + <<: *rspec-quarantine + allow_failure: true + +rspec-mysql-quarantine: + <<: *rspec-metadata-mysql + <<: *rspec-quarantine + allow_failure: true + +static-analysis: + extends: .dedicated-no-docs-no-db-pull-cache-job + dependencies: + - compile-assets + - setup-test-env + script: + - scripts/static-analysis + cache: + key: "debian-stretch-ruby-2.5.3-node-10.x-and-rubocop" + paths: + - vendor/ruby + - .yarn-cache/ + - tmp/rubocop_cache + policy: pull-push + +downtime_check: + <<: *rake-exec + except: + - master + - tags + - /^[\d-]+-stable(-ee)?$/ + - /(^docs[\/-].*|.*-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ + dependencies: + - setup-test-env + +ee_compat_check: + <<: *rake-exec + dependencies: [] + except: + - master + - tags + - /[\d-]+-stable(-ee)?/ + - /^security-/ + - branches@gitlab-org/gitlab-ee + - branches@gitlab/gitlab-ee + retry: 0 + artifacts: + name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}" + when: always + expire_in: 10d + paths: + - ee_compat_check/patches/*.patch + +db:migrate:reset-pg: + <<: *db-migrate-reset + <<: *use-pg + +db:migrate:reset-mysql: + <<: *db-migrate-reset + <<: *use-mysql + +db:check-schema-pg: + <<: *db-migrate-reset + <<: *use-pg + script: + - source scripts/schema_changed.sh + +migration:path-pg: + <<: *migration-paths + <<: *use-pg + +migration:path-mysql: + <<: *migration-paths + <<: *use-mysql + +.db-rollback: &db-rollback + extends: .dedicated-no-docs-and-no-qa-pull-cache-job + script: + - bundle exec rake db:migrate VERSION=20170523121229 + - bundle exec rake db:migrate + dependencies: + - setup-test-env + +db:rollback-pg: + <<: *db-rollback + <<: *use-pg + +db:rollback-mysql: + <<: *db-rollback + <<: *use-mysql + +gitlab:setup-pg: + <<: *gitlab-setup + <<: *use-pg + dependencies: + - setup-test-env + +gitlab:setup-mysql: + <<: *gitlab-setup + <<: *use-mysql + dependencies: + - setup-test-env + +coverage: + # Don't include dedicated-no-docs-no-db-pull-cache-job here since we need to + # download artifacts from all the rspec jobs instead of from setup-test-env only + extends: .dedicated-runner-default-cache + cache: + policy: pull + variables: + SETUP_DB: "false" + stage: post-test + script: + - bundle exec scripts/merge-simplecov + coverage: '/LOC \((\d+\.\d+%)\) covered.$/' + artifacts: + name: coverage + expire_in: 31d + paths: + - coverage/index.html + - coverage/assets/ + except: + - /(^docs[\/-].*|.*-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml new file mode 100644 index 00000000000..9655498073e --- /dev/null +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -0,0 +1,60 @@ +include: + - local: /lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml + +code_quality: + extends: .dedicated-runner + # gitlab-org runners set `privileged: false` but we need to have it set to true + # since we're using Docker in Docker + tags: [] + before_script: [] + cache: {} + dependencies: [] + variables: + SETUP_DB: "false" + +sast: + extends: .dedicated-runner + image: docker:stable + variables: + SAST_CONFIDENCE_LEVEL: 2 + DOCKER_DRIVER: overlay2 + allow_failure: true + tags: [] + before_script: [] + cache: {} + dependencies: [] + services: + - docker:stable-dind + script: + - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + - docker run + --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}" + --volume "$PWD:/code" + --volume /var/run/docker.sock:/var/run/docker.sock + "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code + artifacts: + reports: + sast: gl-sast-report.json + +dependency_scanning: + extends: .dedicated-runner + image: docker:stable + variables: + DOCKER_DRIVER: overlay2 + allow_failure: true + tags: [] + before_script: [] + cache: {} + dependencies: [] + services: + - docker:stable-dind + script: + - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + - docker run + --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}" + --volume "$PWD:/code" + --volume /var/run/docker.sock:/var/run/docker.sock + "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code + artifacts: + reports: + dependency_scanning: gl-dependency-scanning-report.json diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml new file mode 100644 index 00000000000..a7f6caedd37 --- /dev/null +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -0,0 +1,232 @@ +.review-only: &review-only + only: + refs: + - branches@gitlab-org/gitlab-ce + - branches@gitlab-org/gitlab-ee + kubernetes: active + except: + refs: + - master + - /(^docs[\/-].*|.*-docs$)/ + +.review-schedules-only: &review-schedules-only + only: + refs: + - schedules@gitlab-org/gitlab-ce + - schedules@gitlab-org/gitlab-ee + kubernetes: active + variables: + - $REVIEW_APP_CLEANUP + except: + refs: + - tags + - /(^docs[\/-].*|.*-docs$)/ + +.review-base: &review-base + extends: .dedicated-runner + <<: *review-only + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base + stage: test + cache: {} + dependencies: [] + environment: &review-environment + name: review/${CI_COMMIT_REF_NAME} + url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} + before_script: [] + +.review-docker: &review-docker + <<: *review-base + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine + services: + - docker:stable-dind + tags: + - gitlab-org + - docker + variables: &review-docker-variables + GIT_DEPTH: "1" + DOCKER_DRIVER: overlay2 + DOCKER_HOST: tcp://docker:2375 + LATEST_QA_IMAGE: "gitlab/${CI_PROJECT_NAME}-qa:nightly" + QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/${CI_PROJECT_NAME}-qa:${CI_COMMIT_REF_SLUG}" + +build-qa-image: + <<: *review-docker + variables: + <<: *review-docker-variables + GIT_DEPTH: "20" + stage: prepare + script: + - time docker build --cache-from ${LATEST_QA_IMAGE} --tag ${QA_IMAGE} ./qa/ + - echo "${CI_JOB_TOKEN}" | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY} + - time docker push ${QA_IMAGE} + +.review-build-cng-base: &review-build-cng-base + image: ruby:2.5-alpine + stage: test + before_script: [] + dependencies: [] + cache: {} + variables: + API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}" + script: + - apk add --update openssl curl jq + - gem install gitlab --no-document + - source ./scripts/review_apps/review-apps.sh + - wait_for_job_to_be_done "gitlab:assets:compile" + - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng + +review-build-cng: + <<: *review-only + <<: *review-build-cng-base + +schedule:review-build-cng: + <<: *review-schedules-only + <<: *review-build-cng-base + + +.review-deploy-base: &review-deploy-base + <<: *review-base + allow_failure: true + variables: + HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" + DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" + GITLAB_HELM_CHART_REF: "master" + API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}" + environment: + <<: *review-environment + on_stop: review-stop + before_script: + - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) + - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) + - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) + - apk update && apk add jq + - gem install gitlab --no-document + - source ./scripts/review_apps/review-apps.sh + script: + - wait_for_job_to_be_done "review-build-cng" + - perform_review_app_deployment + +review-deploy: + <<: *review-deploy-base + +schedule:review-deploy: + <<: *review-deploy-base + <<: *review-schedules-only + script: + - wait_for_job_to_be_done "schedule:review-build-cng" + - perform_review_app_deployment + +.review-qa-base: &review-qa-base + <<: *review-docker + allow_failure: true + variables: + <<: *review-docker-variables + API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}" + QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" + QA_CAN_TEST_GIT_PROTOCOL_V2: "false" + GITLAB_USERNAME: "root" + GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" + GITLAB_ADMIN_USERNAME: "root" + GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" + GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" + EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" + QA_DEBUG: "true" + artifacts: + paths: + - ./qa/gitlab-qa-run-* + expire_in: 7 days + when: always + before_script: + - echo "${QA_IMAGE}" + - echo "${CI_ENVIRONMENT_URL}" + - apk update && apk add curl jq + - source ./scripts/review_apps/review-apps.sh + - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}} + +review-qa-smoke: + <<: *review-qa-base + script: + - wait_for_job_to_be_done "review-deploy" + - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" + +review-qa-all: + <<: *review-qa-base + script: + - wait_for_job_to_be_done "review-deploy" + - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" + when: manual + +.review-performance-base: &review-performance-base + <<: *review-qa-base + script: + - wait_for_job_to_be_done "review-deploy" + - mkdir -p gitlab-exporter + - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js + - mkdir sitespeed-results + - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" + - mv sitespeed-results/data/performance.json performance.json + artifacts: + paths: + - sitespeed-results/ + reports: + performance: performance.json + +review-performance: + <<: *review-performance-base + +review-stop: + extends: .single-script-job-dedicated-runner + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base + allow_failure: true + variables: + SCRIPT_NAME: "review_apps/review-apps.sh" + when: manual + environment: + <<: *review-environment + action: stop + script: + - source $(basename "${SCRIPT_NAME}") + - delete + - cleanup + +schedule:review-cleanup: + <<: *review-base + <<: *review-schedules-only + stage: build + allow_failure: true + variables: + GIT_DEPTH: "1" + environment: + name: review/auto-cleanup + before_script: + - gem install gitlab --no-document + script: + - ruby -rrubygems scripts/review_apps/automated_cleanup.rb + +schedule:review-performance: + <<: *review-performance-base + <<: *review-schedules-only + script: + - wait_for_job_to_be_done "schedule:review-deploy" + +danger-review: + extends: .dedicated-pull-cache-job + image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger + stage: test + dependencies: [] + before_script: [] + only: + variables: + - $DANGER_GITLAB_API_TOKEN + except: + refs: + - master + variables: + - $CI_COMMIT_REF_NAME =~ /^ce-to-ee-.*/ + - $CI_COMMIT_REF_NAME =~ /.*-stable(-ee)?-prepare-.*/ + script: + - git version + - node --version + - yarn install --frozen-lockfile --cache-folder .yarn-cache + - danger --fail-on-errors=true + diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml new file mode 100644 index 00000000000..debc90a1cb0 --- /dev/null +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -0,0 +1,43 @@ +# Insurance in case a gem needed by one of our releases gets yanked from +# rubygems.org in the future. +cache gems: + extends: .dedicated-no-docs-no-db-pull-cache-job + script: + - bundle package --all --all-platforms + artifacts: + paths: + - vendor/cache + only: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - tags + dependencies: + - setup-test-env + +gitlab_git_test: + extends: .dedicated-runner + variables: + SETUP_DB: "false" + before_script: [] + dependencies: [] + cache: {} + script: + - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes + except: + - /(^docs[\/-].*|.*-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ + +no_ee_check: + extends: .dedicated-runner + variables: + SETUP_DB: "false" + before_script: [] + dependencies: [] + cache: {} + script: + - scripts/no-ee-check + only: + - /.+/@gitlab-org/gitlab-ce + except: + - /(^docs[\/-].*|.*-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml new file mode 100644 index 00000000000..3ba7af956b5 --- /dev/null +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -0,0 +1,82 @@ +.tests-metadata-state: &tests-metadata-state + extends: .dedicated-runner + variables: + TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache" + before_script: + - source scripts/utils.sh + artifacts: + expire_in: 31d + paths: + - knapsack/ + - rspec_flaky/ + - rspec_profiling/ + +retrieve-tests-metadata: + <<: *tests-metadata-state + stage: prepare + cache: + key: tests_metadata + policy: pull + script: + - mkdir -p knapsack/${CI_PROJECT_NAME}/ + - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH + - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}' + - mkdir -p rspec_flaky/ + - mkdir -p rspec_profiling/ + - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH + - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}' + except: + - /(^docs[\/-].*|.*-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ + +update-tests-metadata: + <<: *tests-metadata-state + stage: post-test + cache: + key: tests_metadata + paths: + - knapsack/ + - rspec_flaky/ + policy: push + script: + - retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document + - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json + - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json + - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH} + - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH' + - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH' + - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json + - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json + - scripts/insert-rspec-profiling-data + only: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - master@gitlab/gitlabhq + - master@gitlab/gitlab-ee + +flaky-examples-check: + extends: .dedicated-runner + image: ruby:2.5-alpine + services: [] + before_script: [] + variables: + SETUP_DB: "false" + USE_BUNDLE_INSTALL: "false" + NEW_FLAKY_SPECS_REPORT: rspec_flaky/report-new.json + stage: post-test + allow_failure: true + retry: 0 + only: + - branches + except: + - master + - /(^docs[\/-].*|.*-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ + artifacts: + expire_in: 30d + paths: + - rspec_flaky/ + script: + - '[[ -f $NEW_FLAKY_SPECS_REPORT ]] || echo "{}" > ${NEW_FLAKY_SPECS_REPORT}' + - scripts/merge-reports ${NEW_FLAKY_SPECS_REPORT} rspec_flaky/new_*_*.json + - scripts/detect-new-flaky-examples $NEW_FLAKY_SPECS_REPORT |