Optimize the use of cache and dependencies in CI pipelineoptimize-job-dependencies-and-cache

The idea is to only pull & push from/to cache from the `setup-test-env`
and `compile-assets` jobs, and only pull the required artifacts from all
the other jobs.

Signed-off-by: Rémy Coutable <remy@rymai.me>

9 files changed, 204 insertions, 137 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f926cbc2939..de29963b1ce 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -12,7 +12,6 @@ variables:
   BUILD_ASSETS_IMAGE: "false"
 
 before_script:
-  - date
   - source scripts/utils.sh
   - source scripts/prepare_build.sh
   - date
diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml
index 5c3278fcf53..49eb62f4a8f 100644
--- a/.gitlab/ci/frontend.gitlab-ci.yml
+++ b/.gitlab/ci/frontend.gitlab-ci.yml
@@ -1,20 +1,11 @@
-.assets-compile-cache: &assets-compile-cache
-  cache:
-    key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6"
-    paths:
-      - vendor/ruby/
-      - .yarn-cache/
-      - tmp/cache/assets/sprockets
-
 .use-pg: &use-pg
   services:
     - name: postgres:9.6.14
       command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
     - name: redis:alpine
 
-.gitlab:assets:compile-metadata:
-  <<: *assets-compile-cache
-  extends: .dedicated-no-docs-pull-cache-job
+gitlab:assets:compile:
+  extends: .dedicated-no-docs-no-cache
   image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1
   dependencies:
     - setup-test-env
@@ -42,6 +33,10 @@
     - install_api_client_dependencies_with_apt
     - play_job "review-build-cng" || true  # this job might not exist so ignore the failure if it cannot be played
     - play_job "schedule:review-build-cng" || true  # this job might not exist so ignore the failure if it cannot be played
+  cache:
+    key: "assets-compile:tmp_cache_assets_sprockets"
+    paths:
+      - tmp/cache/assets/sprockets
   artifacts:
     name: webpack-report
     expire_in: 31d
@@ -57,68 +52,49 @@
     - docker
     - gitlab-org
 
-gitlab:assets:compile:
-  extends: .gitlab:assets:compile-metadata
-  cache:
-    policy: pull-push
-  only:
-    - master@gitlab-org/gitlab-ce
-    - master@gitlab-org/gitlab-ee
-
-gitlab:assets:compile pull-cache:
-  extends: .gitlab:assets:compile-metadata
-  cache:
-    policy: pull
-  except:
-    refs:
-      - master@gitlab-org/gitlab-ce
-      - master@gitlab-org/gitlab-ee
-      - /(^docs[\/-].+|.+-docs$)/
-
-.compile-assets-metadata:
-  extends: .dedicated-runner
+compile-assets:
+  extends:
+    - .dedicated-runner-default-cache
+    - .no-docs
   <<: *use-pg
-  <<: *assets-compile-cache
   stage: prepare
   script:
     - node --version
     - retry yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline
     - free -m
     - retry bundle exec rake gitlab:assets:compile
-    - scripts/clean-old-cached-assets
   variables:
     # we override the max_old_space_size to prevent OOM errors
     NODE_OPTIONS: --max_old_space_size=3584
   artifacts:
     expire_in: 7d
     paths:
-      - node_modules
-      - public/assets
+      - node_modules/
+      - public/assets/
 
-compile-assets:
-  extends: .compile-assets-metadata
-  cache:
-    policy: pull-push
+gitlab:ui:visual:
+  extends: .dedicated-no-docs-no-cache
+  before_script: []
+  allow_failure: true
+  dependencies:
+    - compile-assets
+  script:
+    # Remove node modules from GitLab that may conflict with gitlab-ui
+    - rm -r node_modules
+    - git clone https://gitlab.com/gitlab-org/gitlab-ui.git
+    - cp public/assets/application-*.css gitlab-ui/styles/application.css
+    - cd gitlab-ui
+    - yarn install
+    - CSS_URL=./application.css yarn test
   only:
     - master@gitlab-org/gitlab-ce
     - master@gitlab-org/gitlab-ee
 
-compile-assets pull-cache:
-  extends: .compile-assets-metadata
-  cache:
-    policy: pull
-  except:
-    refs:
-      - master@gitlab-org/gitlab-ce
-      - master@gitlab-org/gitlab-ee
-      - /(^docs[\/-].+|.+-docs$)/
-
 karma:
-  extends: .dedicated-no-docs-pull-cache-job
+  extends: .dedicated-no-docs-no-cache
   <<: *use-pg
   dependencies:
     - compile-assets
-    - compile-assets pull-cache
     - setup-test-env
   variables:
     # we override the max_old_space_size to prevent OOM errors
@@ -142,11 +118,10 @@ karma:
       junit: junit_karma.xml
 
 jest:
-  extends: .dedicated-no-docs-and-no-qa-pull-cache-job
+  extends: .dedicated-no-docs-no-qa-no-cache
   <<: *use-pg
   dependencies:
     - compile-assets
-    - compile-assets pull-cache
     - setup-test-env
   script:
     - scripts/gitaly-test-spawn
@@ -168,42 +143,39 @@ jest:
     key: jest
     paths:
       - tmp/jest/jest/
-    policy: pull-push
 
 qa:internal:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
-  services: []
+  extends: .dedicated-no-docs-no-db-no-cache
+  before_script: []
+  dependencies:
+    - setup-test-env
   script:
+    - mkdir -p qa/vendor && mv vendor/ruby qa/vendor/ruby
     - cd qa/
-    - bundle install
+    - time bundle install --without=production --jobs=$(nproc) --path=vendor --retry=3 --quiet
     - bundle exec rspec
-  dependencies:
-    - setup-test-env
 
 qa:selectors:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
-  services: []
+  extends: .dedicated-no-docs-no-db-no-cache
+  before_script: []
+  dependencies:
+    - setup-test-env
   script:
+    - mkdir -p qa/vendor && mv vendor/ruby qa/vendor/ruby
     - cd qa/
-    - bundle install
+    - time bundle install --without=production --jobs=$(nproc) --path=vendor --retry=3 --quiet
     - bundle exec bin/qa Test::Sanity::Selectors
-  dependencies:
-    - setup-test-env
 
 .qa-frontend-node: &qa-frontend-node
-  extends: .dedicated-no-docs-no-db-pull-cache-job
-  stage: test
+  extends: .dedicated-no-docs-no-db-no-cache
+  before_script: []
+  dependencies: []
   cache:
     key: "$CI_JOB_NAME"
     paths:
       - .yarn-cache/
-    policy: pull-push
-  dependencies: []
-  before_script: []
   script:
-    - date
     - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline
-    - date
     - yarn run webpack-prod
 
 qa-frontend-node:8:
@@ -220,12 +192,12 @@ qa-frontend-node:latest:
   allow_failure: true
 
 lint:javascript:report:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends: .dedicated-no-docs-no-db-no-cache
   stage: post-test
-  dependencies: []
   before_script: []
+  dependencies:
+    - compile-assets
   script:
-    - date
     - yarn run eslint-report || true  # ignore exit code
   artifacts:
     name: eslint-report
@@ -234,14 +206,12 @@ lint:javascript:report:
       - eslint-report.html
 
 jsdoc:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends: .dedicated-no-docs-no-db-no-cache
   stage: post-test
+  before_script: []
   dependencies:
     - compile-assets
-    - compile-assets pull-cache
-  before_script: []
   script:
-    - date
     - yarn run jsdoc || true  # ignore exit code
   artifacts:
     name: jsdoc
diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml
index 78ef346d417..6e1be400f54 100644
--- a/.gitlab/ci/global.gitlab-ci.yml
+++ b/.gitlab/ci/global.gitlab-ci.yml
@@ -9,23 +9,22 @@
     - gitlab-org
 
 .default-cache: &default-cache
-  key: "debian-stretch-ruby-2.6.3-node-12.x"
-  paths:
-    - vendor/ruby
-    - .yarn-cache/
-    - vendor/gitaly-ruby
+  cache:
+    key: "debian-stretch-ruby-2.6.3-node-12.x"
+    paths:
+      - vendor/ruby/
+      - .yarn-cache/
+      - vendor/gitaly-ruby/
+      - tmp/cache/assets/sprockets
 
 .dedicated-runner-default-cache:
   extends: .dedicated-runner
-  cache:
-    <<: *default-cache
+  <<: *default-cache
 
-# Jobs that only need to pull cache
-.dedicated-pull-cache-job:
+# Jobs that do not need cache
+.dedicated-no-cache:
   extends: .dedicated-runner
-  cache:
-    <<: *default-cache
-    policy: pull
+  cache: {}
   stage: test
 
 .no-docs:
@@ -33,12 +32,24 @@
     refs:
       - /(^docs[\/-].+|.+-docs$)/
 
+# Jobs that shouldn't run for docs branches and do not need cache
+.dedicated-no-docs-no-cache:
+  extends:
+    - .dedicated-no-cache
+    - .no-docs
+
 .no-docs-and-no-qa:
   except:
     refs:
       - /(^docs[\/-].+|.+-docs$)/
       - /(^qa[\/-].*|.*-qa$)/
 
+# Jobs that shouldn't run for docs nor qa branches and do not need cache
+.dedicated-no-docs-no-qa-no-cache:
+  extends:
+    - .dedicated-no-cache
+    - .no-docs-and-no-qa
+
 .dedicated-no-docs-pull-cache-job:
   extends:
     - .dedicated-pull-cache-job
@@ -49,9 +60,9 @@
     - .dedicated-pull-cache-job
     - .no-docs-and-no-qa
 
-# Jobs that do not need a DB
-.dedicated-no-docs-no-db-pull-cache-job:
-  extends: .dedicated-no-docs-pull-cache-job
+# Jobs that shouldn't run for docs and do not need DB nor cache
+.dedicated-no-docs-no-db-no-cache:
+  extends: .dedicated-no-docs-no-cache
   variables:
     SETUP_DB: "false"
 
diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml
index f7b18b809b4..cab84642828 100644
--- a/.gitlab/ci/pages.gitlab-ci.yml
+++ b/.gitlab/ci/pages.gitlab-ci.yml
@@ -1,5 +1,5 @@
 pages:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends: .dedicated-no-docs-no-db-no-cache
   before_script: []
   stage: pages
   dependencies:
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index 50476b43dd6..74bfe37b4bc 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -21,8 +21,10 @@
 
 .gitlab-setup: &gitlab-setup
   extends:
-    - .dedicated-no-docs-and-no-qa-pull-cache-job
+    - .dedicated-no-docs-no-qa-no-cache
     - .use-pg
+  dependencies:
+    - setup-test-env
   variables:
     SETUP_DB: "false"
   script:
@@ -39,13 +41,13 @@
       - log/development.log
 
 .rake-exec: &rake-exec
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends: .dedicated-no-docs-no-db-no-cache
   script:
     - bundle exec rake $CI_JOB_NAME
 
 .rspec-metadata: &rspec-metadata
   extends:
-    - .dedicated-pull-cache-job
+    - .dedicated-no-cache
     - .no-docs-and-no-qa
   stage: test
   script:
@@ -94,14 +96,16 @@
 
 # DB migration, rollback, and seed jobs
 .db-migrate-reset: &db-migrate-reset
-  extends: .dedicated-no-docs-and-no-qa-pull-cache-job
+  extends: .dedicated-no-docs-no-qa-no-cache
   script:
     - bundle exec rake db:migrate:reset
   dependencies:
     - setup-test-env
 
 .migration-paths: &migration-paths
-  extends: .dedicated-no-docs-and-no-qa-pull-cache-job
+  extends: .dedicated-no-docs-no-qa-no-cache
+  dependencies:
+    - setup-test-env
   variables:
     SETUP_DB: "false"
   script:
@@ -121,8 +125,6 @@
     - . scripts/prepare_build.sh
     - date
     - bundle exec rake db:migrate
-  dependencies:
-    - setup-test-env
 
 setup-test-env:
   extends:
@@ -131,14 +133,18 @@ setup-test-env:
     - .use-pg
   stage: prepare
   script:
+    - date
     - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
+    - date
     - scripts/gitaly-test-build  # Do not use 'bundle exec' here
+    - date
   artifacts:
     expire_in: 7d
     paths:
       - tmp/tests
       - config/secrets.yml
-      - vendor/gitaly-ruby
+      - vendor/ruby/
+      - vendor/gitaly-ruby/
 
 rspec unit pg:
   <<: *rspec-metadata-pg
@@ -185,32 +191,28 @@ rspec quarantine pg:
   allow_failure: true
 
 static-analysis:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends: .dedicated-no-docs-no-db-no-cache
   dependencies:
     - compile-assets
     - compile-assets pull-cache
     - setup-test-env
-  script:
-    - scripts/static-analysis
   cache:
-    key: "debian-stretch-ruby-2.6.3-node-12.x-and-rubocop"
+    key: "rubocop_cache"
     paths:
-      - vendor/ruby
-      - .yarn-cache/
       - tmp/rubocop_cache
-    policy: pull-push
+  script:
+    - scripts/static-analysis
 
 downtime_check:
   <<: *rake-exec
-  except:
-    refs:
-      - master
-      - tags
-      - /^[\d-]+-stable(-ee)?$/
-      - /(^docs[\/-].+|.+-docs$)/
-      - /(^qa[\/-].*|.*-qa$)/
   dependencies:
     - setup-test-env
+  except:
+    - master
+    - tags
+    - /^[\d-]+-stable(-ee)?$/
+    - /(^docs[\/-].+|.+-docs$)/
+    - /(^qa[\/-].*|.*-qa$)/
 
 ee_compat_check:
   <<: *rake-exec
@@ -247,7 +249,7 @@ migration:path-pg:
   <<: *use-pg
 
 .db-rollback: &db-rollback
-  extends: .dedicated-no-docs-and-no-qa-pull-cache-job
+  extends: .dedicated-no-docs-no-qa-no-cache
   script:
     - bundle exec rake db:migrate VERSION=20180101160629
     - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true
@@ -261,20 +263,12 @@ db:rollback-pg:
 gitlab:setup-pg:
   <<: *gitlab-setup
   <<: *use-pg
-  dependencies:
-    - setup-test-env
 
 coverage:
-  # Don't include dedicated-no-docs-no-db-pull-cache-job here since we need to
-  # download artifacts from all the rspec jobs instead of from setup-test-env only
-  extends:
-    - .dedicated-runner-default-cache
-    - .no-docs-and-no-qa
-  cache:
-    policy: pull
+  extends: .dedicated-no-docs-no-qa-no-cache
+  stage: post-test
   variables:
     SETUP_DB: "false"
-  stage: post-test
   script:
     - bundle exec scripts/merge-simplecov
     - bundle exec scripts/gather-test-memory-data
diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml
index ca55bbd32a7..35956c302f7 100644
--- a/.gitlab/ci/reports.gitlab-ci.yml
+++ b/.gitlab/ci/reports.gitlab-ci.yml
@@ -4,11 +4,16 @@ include:
   - template: Security/Dependency-Scanning.gitlab-ci.yml
 
 code_quality:
+<<<<<<< HEAD
   extends: .dedicated-no-docs
+=======
+  extends: .dedicated-no-docs-no-db-no-cache
+>>>>>>> Optimize the use of cache and dependencies in CI pipeline
   # gitlab-org runners set `privileged: false` but we need to have it set to true
   # since we're using Docker in Docker
   tags: []
   before_script: []
+<<<<<<< HEAD
   cache: {}
 
 sast:
@@ -24,3 +29,89 @@ dependency_scanning:
   tags: []
   before_script: []
   cache: {}
+=======
+  dependencies: []
+
+sast:
+  extends: .dedicated-no-docs-no-db-no-cache
+  image: docker:stable
+  variables:
+    SAST_CONFIDENCE_LEVEL: 2
+    DOCKER_DRIVER: overlay2
+  allow_failure: true
+  tags: []
+  before_script: []
+  dependencies: []
+  services:
+    - docker:stable-dind
+  script:
+    - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage
+      function propagate_env_vars() {
+        CURRENT_ENV=$(printenv)
+
+        for VAR_NAME; do
+          echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME "
+        done
+      }
+    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
+    - |
+      docker run \
+        $(propagate_env_vars \
+          SAST_ANALYZER_IMAGES \
+          SAST_ANALYZER_IMAGE_PREFIX \
+          SAST_ANALYZER_IMAGE_TAG \
+          SAST_DEFAULT_ANALYZERS \
+          SAST_BRAKEMAN_LEVEL \
+          SAST_GOSEC_LEVEL \
+          SAST_FLAWFINDER_LEVEL \
+          SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \
+          SAST_PULL_ANALYZER_IMAGE_TIMEOUT \
+          SAST_RUN_ANALYZER_TIMEOUT \
+        ) \
+        --volume "$PWD:/code" \
+        --volume /var/run/docker.sock:/var/run/docker.sock \
+        "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code
+  artifacts:
+    reports:
+      sast: gl-sast-report.json
+
+dependency_scanning:
+  extends: .dedicated-no-docs-no-db-no-cache
+  image: docker:stable
+  variables:
+    DOCKER_DRIVER: overlay2
+  allow_failure: true
+  tags: []
+  before_script: []
+  dependencies: []
+  services:
+    - docker:stable-dind
+  script:
+    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
+    - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage
+      function propagate_env_vars() {
+        CURRENT_ENV=$(printenv)
+
+        for VAR_NAME; do
+          echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME "
+        done
+      }
+    - |
+      docker run \
+        $(propagate_env_vars \
+          DS_ANALYZER_IMAGES \
+          DS_ANALYZER_IMAGE_PREFIX \
+          DS_ANALYZER_IMAGE_TAG \
+          DS_DEFAULT_ANALYZERS \
+          DEP_SCAN_DISABLE_REMOTE_CHECKS \
+          DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \
+          DS_PULL_ANALYZER_IMAGE_TIMEOUT \
+          DS_RUN_ANALYZER_TIMEOUT \
+        ) \
+        --volume "$PWD:/code" \
+        --volume /var/run/docker.sock:/var/run/docker.sock \
+        "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code
+  artifacts:
+    reports:
+      dependency_scanning: gl-dependency-scanning-report.json
+>>>>>>> Optimize the use of cache and dependencies in CI pipeline
diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml
index 4a9269ffd82..33ec6a9a98b 100644
--- a/.gitlab/ci/review.gitlab-ci.yml
+++ b/.gitlab/ci/review.gitlab-ci.yml
@@ -27,8 +27,8 @@
   extends: .dedicated-runner
   <<: *review-only
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
-  cache: {}
   dependencies: []
+  cache: {}
   before_script:
     - source scripts/utils.sh
 
@@ -254,11 +254,16 @@ schedule:review-cleanup:
     - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
 
 danger-review:
-  extends: .dedicated-pull-cache-job
+  extends: .dedicated-no-cache
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
   stage: test
-  dependencies: []
-  before_script: []
+  dependencies:
+    - setup-test-env
+    - compile-assets
+  before_script:
+    - git version
+    - node --version
+    - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline
   only:
     variables:
       - $DANGER_GITLAB_API_TOKEN
@@ -271,7 +276,4 @@ danger-review:
       - $CI_COMMIT_REF_NAME =~ /^ce-to-ee-.*/
       - $CI_COMMIT_REF_NAME =~ /.*-stable(-ee)?-prepare-.*/
   script:
-    - git version
-    - node --version
-    - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline
     - danger --fail-on-errors=true
diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml
index c1fc3a893ca..635507c8ebf 100644
--- a/.gitlab/ci/setup.gitlab-ci.yml
+++ b/.gitlab/ci/setup.gitlab-ci.yml
@@ -1,7 +1,7 @@
 # Insurance in case a gem needed by one of our releases gets yanked from
 # rubygems.org in the future.
 cache gems:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends: .dedicated-no-docs-no-db-no-cache
   script:
     - bundle package --all --all-platforms
   artifacts:
diff --git a/scripts/gitaly_test.rb b/scripts/gitaly_test.rb
index b5cc5118530..b528e7cbbb8 100644
--- a/scripts/gitaly_test.rb
+++ b/scripts/gitaly_test.rb
@@ -31,7 +31,7 @@ module GitalyTest
 
     if ENV['CI']
       bundle_path = File.expand_path('../vendor/gitaly-ruby', __dir__)
-      env_hash['BUNDLE_FLAGS'] << " --path=#{bundle_path}"
+      env_hash['BUNDLE_FLAGS'] << " --path=#{bundle_path} --quiet"
     end
 
     env_hash