Add latest changes from gitlab-org/gitlab@master

2 files changed, 176 insertions, 105 deletions

diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml
index 4d6c02336fe..797ec4f5860 100644
--- a/.gitlab/ci/reports.gitlab-ci.yml
+++ b/.gitlab/ci/reports.gitlab-ci.yml
@@ -1,5 +1,120 @@
+# Make sure to update all the similar conditions in other CI config files if you modify these conditions
+.if-canonical-gitlab-merge-request: &if-canonical-gitlab-merge-request
+  if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_MERGE_REQUEST_IID'
+
+# Make sure to update all the similar conditions in other CI config files if you modify these conditions
+.if-canonical-dot-com-gitlab-org-group-schedule: &if-canonical-dot-com-gitlab-org-group-schedule
+  if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_PIPELINE_SOURCE == "schedule"'
+
+# Make sure to update all the similar conditions in other CI config files if you modify these conditions
+.if-master-refs: &if-master-refs
+  if: '$CI_COMMIT_REF_NAME == "master"'
+
+# Make sure to update all the similar conditions in other CI config files if you modify these conditions
+.if-default-refs: &if-default-refs
+  if: '$CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/ || $CI_COMMIT_REF_NAME =~ /^\d+-\d+-auto-deploy-\d+$/ || $CI_COMMIT_REF_NAME =~ /^security\// || $CI_MERGE_REQUEST_IID || $CI_COMMIT_TAG'
+
+# Make sure to update all the similar patterns in other CI config files if you modify these patterns
+.code-backstage-patterns: &code-backstage-patterns
+  - ".gitlab/ci/**/*"
+  - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
+  - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
+  - ".csscomb.json"
+  - "Dockerfile.assets"
+  - "*_VERSION"
+  - "Gemfile{,.lock}"
+  - "Rakefile"
+  - "{babel.config,jest.config}.js"
+  - "config.ru"
+  - "{package.json,yarn.lock}"
+  - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
+  - "doc/api/graphql/reference/*" # Files in this folder are auto-generated
+  # Backstage changes
+  - "Dangerfile"
+  - "danger/**/*"
+  - "{,ee/}fixtures/**/*"
+  - "{,ee/}rubocop/**/*"
+  - "{,ee/}spec/**/*"
+  - "doc/README.md"  # Some RSpec test rely on this file
+
+# Make sure to update all the similar patterns in other CI config files if you modify these patterns
+.code-qa-patterns: &code-qa-patterns
+  - ".gitlab/ci/**/*"
+  - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
+  - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
+  - ".csscomb.json"
+  - "Dockerfile.assets"
+  - "*_VERSION"
+  - "Gemfile{,.lock}"
+  - "Rakefile"
+  - "{babel.config,jest.config}.js"
+  - "config.ru"
+  - "{package.json,yarn.lock}"
+  - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
+  - "doc/api/graphql/reference/*" # Files in this folder are auto-generated
+  # QA changes
+  - ".dockerignore"
+  - "qa/**/*"
+
+# Make sure to update all the similar patterns in other CI config files if you modify these patterns
+.code-backstage-qa-patterns: &code-backstage-qa-patterns
+  - ".gitlab/ci/**/*"
+  - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
+  - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
+  - ".csscomb.json"
+  - "Dockerfile.assets"
+  - "*_VERSION"
+  - "Gemfile{,.lock}"
+  - "Rakefile"
+  - "{babel.config,jest.config}.js"
+  - "config.ru"
+  - "{package.json,yarn.lock}"
+  - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
+  - "doc/api/graphql/reference/*" # Files in this folder are auto-generated
+  # Backstage changes
+  - "Dangerfile"
+  - "danger/**/*"
+  - "{,ee/}fixtures/**/*"
+  - "{,ee/}rubocop/**/*"
+  - "{,ee/}spec/**/*"
+  - "doc/README.md"  # Some RSpec test rely on this file
+  # QA changes
+  - ".dockerignore"
+  - "qa/**/*"
+
+.reports:rules:code_quality:
+  rules:
+    - if: '$CODE_QUALITY_DISABLED'
+      when: never
+    # - <<: *if-master-refs  # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
+    - <<: *if-default-refs
+      changes: *code-backstage-patterns
+
+.reports:rules:sast:
+  rules:
+    - if: '$SAST_DISABLED || $GITLAB_FEATURES !~ /\bsast\b/'
+      when: never
+    # - <<: *if-master-refs  # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
+    - <<: *if-default-refs
+      changes: *code-backstage-qa-patterns
+
+.reports:rules:dependency_scanning:
+  rules:
+    - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/'
+      when: never
+    # - <<: *if-master-refs  # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
+    - <<: *if-default-refs
+      changes: *code-backstage-qa-patterns
+
+.reports:rules:dast:
+  rules:
+    - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/'
+      when: never
+    - <<: *if-canonical-gitlab-merge-request
+      changes: *code-qa-patterns
+
 # include:
-#   - template: Code-Quality.gitlab-ci.yml
+#   - template: Jobs/Code-Quality.gitlab-ci.yml
 #   - template: Security/SAST.gitlab-ci.yml
 #   - template: Security/Dependency-Scanning.gitlab-ci.yml
 #   - template: Security/DAST.gitlab-ci.yml
@@ -10,8 +125,7 @@
 code_quality:
   extends:
     - .default-retry
-    - .default-only
-    - .only:changes-code-backstage
+    - .reports:rules:code_quality
   stage: test
   image: docker:stable
   allow_failure: true
@@ -38,12 +152,9 @@ code_quality:
     reports:
       codequality: gl-code-quality-report.json
     paths:
-      - gl-code-quality-report.json
-    expire_in: 1 week
+      - gl-code-quality-report.json  # GitLab-specific
+    expire_in: 1 week  # GitLab-specific
   dependencies: []
-  except:
-    variables:
-      - $CODE_QUALITY_DISABLED
 
 # We need to duplicate this job's definition because it seems it's impossible to
 # override an included `only.refs`.
@@ -53,16 +164,22 @@ code_quality:
 sast:
   extends:
     - .default-retry
-    - .default-only
-    - .only:changes-code-backstage-qa
+    - .reports:rules:sast
   stage: test
+  allow_failure: true
+  dependencies: []  # GitLab-specific
+  artifacts:
+    paths:
+      - gl-sast-report.json  # GitLab-specific
+    reports:
+      sast: gl-sast-report.json
+    expire_in: 1 week  # GitLab-specific
   image: docker:stable
   variables:
     DOCKER_DRIVER: overlay2
     DOCKER_TLS_CERTDIR: ""
-    SAST_BRAKEMAN_LEVEL: 2
-    SAST_EXCLUDED_PATHS: qa,spec,doc,ee/spec
-  allow_failure: true
+    SAST_BRAKEMAN_LEVEL: 2  # GitLab-specific
+    SAST_EXCLUDED_PATHS: qa,spec,doc,ee/spec  # GitLab-specific
   services:
     - docker:stable-dind
   script:
@@ -73,61 +190,12 @@ sast:
           export DOCKER_HOST='tcp://localhost:2375'
         fi
       fi
-    - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage
-      function propagate_env_vars() {
-        CURRENT_ENV=$(printenv)
-
-        for VAR_NAME; do
-          echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME "
-        done
-      }
     - |
-      docker run \
-        $(propagate_env_vars \
-          SAST_BANDIT_EXCLUDED_PATHS \
-          SAST_ANALYZER_IMAGES \
-          SAST_ANALYZER_IMAGE_PREFIX \
-          SAST_ANALYZER_IMAGE_TAG \
-          SAST_DEFAULT_ANALYZERS \
-          SAST_PULL_ANALYZER_IMAGES \
-          SAST_BRAKEMAN_LEVEL \
-          SAST_FLAWFINDER_LEVEL \
-          SAST_GITLEAKS_ENTROPY_LEVEL \
-          SAST_GOSEC_LEVEL \
-          SAST_EXCLUDED_PATHS \
-          SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \
-          SAST_PULL_ANALYZER_IMAGE_TIMEOUT \
-          SAST_RUN_ANALYZER_TIMEOUT \
-          SAST_JAVA_VERSION \
-          ANT_HOME \
-          ANT_PATH \
-          GRADLE_PATH \
-          JAVA_OPTS \
-          JAVA_PATH \
-          JAVA_8_VERSION \
-          JAVA_11_VERSION \
-          MAVEN_CLI_OPTS \
-          MAVEN_PATH \
-          MAVEN_REPO_PATH \
-          SBT_PATH \
-          FAIL_NEVER \
-        ) \
+      ENVS=`printenv | grep -vE '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | sed -n '/^[^\t]/s/=.*//p' | sed '/^$/d' | sed 's/^/-e /g' | tr '\n' ' '`
+      docker run "$ENVS" \
         --volume "$PWD:/code" \
         --volume /var/run/docker.sock:/var/run/docker.sock \
         "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code
-  artifacts:
-    expire_in: 7 days
-    paths:
-      - gl-sast-report.json
-    reports:
-      sast: gl-sast-report.json
-  dependencies: []
-  only:
-    variables:
-      - $GITLAB_FEATURES =~ /\bsast\b/
-  except:
-    variables:
-      - $SAST_DISABLED
 
 # We need to duplicate this job's definition because it seems it's impossible to
 # override an included `only.refs`.
@@ -135,14 +203,13 @@ sast:
 dependency_scanning:
   extends:
     - .default-retry
-    - .default-only
-    - .only:changes-code-backstage-qa
+    - .reports:rules:dependency_scanning
   stage: test
   image: docker:stable
   variables:
     DOCKER_DRIVER: overlay2
     DOCKER_TLS_CERTDIR: ""
-    DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports,spec,ee/spec"
+    DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports,spec,ee/spec"  # GitLab-specific
   allow_failure: true
   services:
     - docker:stable-dind
@@ -174,23 +241,29 @@ dependency_scanning:
           DS_PULL_ANALYZER_IMAGE_TIMEOUT \
           DS_RUN_ANALYZER_TIMEOUT \
           DS_PYTHON_VERSION \
+          DS_PIP_VERSION \
           DS_PIP_DEPENDENCY_PATH \
+          GEMNASIUM_DB_LOCAL_PATH \
+          GEMNASIUM_DB_REMOTE_URL \
+          GEMNASIUM_DB_REF_NAME \
           PIP_INDEX_URL \
           PIP_EXTRA_INDEX_URL \
+          PIP_REQUIREMENTS_FILE \
+          MAVEN_CLI_OPTS \
+          BUNDLER_AUDIT_UPDATE_DISABLED \
+          BUNDLER_AUDIT_ADVISORY_DB_URL \
+          BUNDLER_AUDIT_ADVISORY_DB_REF_NAME \
         ) \
         --volume "$PWD:/code" \
         --volume /var/run/docker.sock:/var/run/docker.sock \
         "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_VERSION" /code
   artifacts:
+    paths:
+      - gl-dependency-scanning-report.json  # GitLab-specific
     reports:
       dependency_scanning: gl-dependency-scanning-report.json
+    expire_in: 1 week  # GitLab-specific
   dependencies: []
-  only:
-    variables:
-      - $GITLAB_FEATURES =~ /\bdependency_scanning\b/
-  except:
-    variables:
-      - $DEPENDENCY_SCANNING_DISABLED
 
 # We need to duplicate this job's definition because it seems it's impossible to
 # override an included `only.refs`.
@@ -198,40 +271,38 @@ dependency_scanning:
 dast:
   extends:
     - .default-retry
-    - .default-only
-    - .only:changes-code-qa
-    - .only-review
-  stage: qa
-  needs: ["review-deploy"]
-  dependencies: ["review-deploy"]
-  before_script:
-    - export DAST_WEBSITE="$(cat review_app_url.txt)"
+    - .reports:rules:dast
+  needs:
+    - job: review-deploy
+      artifacts: true
+  stage: qa  # GitLab-specific
   image:
-    name: "registry.gitlab.com/gitlab-org/security-products/dast:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable"
+    name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION"
   variables:
-  # URL to scan:
-  # DAST_WEBSITE: https://example.com/
-  #
-  # Time limit for target availability (scan is attempted even when timeout):
-  # DAST_TARGET_AVAILABILITY_TIMEOUT: 60
-  #
-  # Set these variables to scan with an authenticated user:
-  # DAST_AUTH_URL: https://example.com/sign-in
-  # DAST_USERNAME: john.doe@example.com
-  # DAST_PASSWORD: john-doe-password
-  # DAST_USERNAME_FIELD: session[user] # the name of username field at the sign-in HTML form
-  # DAST_PASSWORD_FIELD: session[password] # the name of password field at the sign-in HTML form
-  # DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional: URLs to skip during the authenticated scan; comma-separated, no spaces in between
-  #
-  # Perform ZAP Full Scan, which includes both passive and active scanning:
-  # DAST_FULL_SCAN_ENABLED: "true"
+    # To be done in a later iteration
+    # DAST_USERNAME: "root"
+    # DAST_USERNAME_FIELD: "user[login]"
+    # DAST_PASSWORD_FIELD: "user[passowrd]"
   allow_failure: true
   script:
-    - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
+    - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"'
+    # To be done in a later iteration
+    # - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"'
+    # - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"'
     - /analyze -t $DAST_WEBSITE
   artifacts:
-    expire_in: 7 days
     paths:
-      - gl-dast-report.json
+      - gl-dast-report.json  # GitLab-specific
     reports:
       dast: gl-dast-report.json
+    expire_in: 1 week  # GitLab-specific
+
+# To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
+# schedule:dast:
+#   extends: dast
+#   rules:
+#     - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/'
+#       when: never
+#     - <<: *if-canonical-dot-com-gitlab-org-group-schedule
+#   variables:
+#     DAST_FULL_SCAN_ENABLED: "true"
diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml
index 5d2c33d06b4..682ed12e199 100644
--- a/.gitlab/ci/review.gitlab-ci.yml
+++ b/.gitlab/ci/review.gitlab-ci.yml
@@ -148,7 +148,7 @@ review-deploy:
     - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION)
     - export GITALY_VERSION=$(<GITALY_SERVER_VERSION)
     - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION)
-    - echo "${CI_ENVIRONMENT_URL}" > review_app_url.txt
+    - echo "${CI_ENVIRONMENT_URL}" > environment_url.txt
     - source ./scripts/utils.sh
     - install_api_client_dependencies_with_apk
     - source scripts/review_apps/review-apps.sh
@@ -161,7 +161,7 @@ review-deploy:
     - date
     - deploy || (display_deployment_debug && exit 1)
   artifacts:
-    paths: [review_app_url.txt]
+    paths: [environment_url.txt]
     expire_in: 2 days
     when: always
 
@@ -216,7 +216,7 @@ review-stop:
   before_script:
     - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"'
     - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/gitlab-${GITLAB_EDITION}-qa:${CI_COMMIT_REF_SLUG}"
-    - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)"
+    - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
     - echo "${CI_ENVIRONMENT_URL}"
     - echo "${QA_IMAGE}"
     - source scripts/utils.sh
@@ -255,7 +255,7 @@ review-performance:
       artifacts: true
   allow_failure: true
   before_script:
-    - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)"
+    - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
     - echo "${CI_ENVIRONMENT_URL}"
     - mkdir -p gitlab-exporter
     - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js