diff options
7 files changed, 86 insertions, 253 deletions
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index e1de3e7c914..f2ee4a30f07 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -78,217 +78,8 @@ include: - template: Jobs/Code-Quality.gitlab-ci.yml - template: Jobs/Deploy.gitlab-ci.yml - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml - -license_management: - stage: test - image: - name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" - entrypoint: [""] - allow_failure: true - script: - - license_management - artifacts: - paths: [gl-license-management-report.json] - only: - refs: - - branches - - tags - variables: - - $GITLAB_FEATURES =~ /\blicense_management\b/ - except: - variables: - - $LICENSE_MANAGEMENT_DISABLED - -sast: - stage: test - image: docker:stable - allow_failure: true - services: - - docker:stable-dind - script: - - setup_docker - - sast - artifacts: - reports: - sast: gl-sast-report.json - only: - refs: - - branches - - tags - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ - except: - variables: - - $SAST_DISABLED - -dependency_scanning: - stage: test - image: docker:stable - allow_failure: true - services: - - docker:stable-dind - script: - - setup_docker - - dependency_scanning - artifacts: - reports: - dependency_scanning: gl-dependency-scanning-report.json - only: - refs: - - branches - - tags - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ - except: - variables: - - $DEPENDENCY_SCANNING_DISABLED - -container_scanning: - stage: test - image: docker:stable - allow_failure: true - services: - - docker:stable-dind - script: - - setup_docker - - container_scanning - artifacts: - paths: [gl-container-scanning-report.json] - only: - refs: - - branches - - tags - variables: - - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ - except: - variables: - - $CONTAINER_SCANNING_DISABLED - -dast: - stage: dast - allow_failure: true - image: registry.gitlab.com/gitlab-org/security-products/zaproxy - variables: - POSTGRES_DB: "false" - script: - - dast - artifacts: - paths: [gl-dast-report.json] - only: - refs: - - branches - - tags - kubernetes: active - variables: - - $GITLAB_FEATURES =~ /\bdast\b/ - except: - refs: - - master - variables: - - $DAST_DISABLED - -# --------------------------------------------------------------------------- - -.auto_devops: &auto_devops | - # Auto DevOps variables and functions - [[ "$TRACE" ]] && set -x - auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} - export DATABASE_URL=${DATABASE_URL-$auto_database_url} - if [[ -z "$CI_COMMIT_TAG" ]]; then - export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG - export CI_APPLICATION_TAG=$CI_COMMIT_SHA - else - export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE - export CI_APPLICATION_TAG=$CI_COMMIT_TAG - fi - export TILLER_NAMESPACE=$KUBE_NAMESPACE - # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - - function registry_login() { - if [[ -n "$CI_REGISTRY_USER" ]]; then - echo "Logging to GitLab Container Registry with CI credentials..." - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" - echo "" - fi - } - - function container_scanning() { - registry_login - - docker run -d --name db arminc/clair-db:latest - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.6 - apk add -U wget ca-certificates - docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64 - mv clair-scanner_linux_amd64 clair-scanner - chmod +x clair-scanner - touch clair-whitelist.yml - retries=0 - echo "Waiting for clair daemon to start" - while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done - ./clair-scanner -c http://${DOCKER_SERVICE}:6060 --ip $(hostname -i) -r gl-container-scanning-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true - } - - function license_management() { - /run.sh analyze . - } - - function sast() { - case "$CI_SERVER_VERSION" in - *-ee) - - # Deprecation notice for CONFIDENCE_LEVEL variable - if [ -z "$SAST_CONFIDENCE_LEVEL" -a "$CONFIDENCE_LEVEL" ]; then - SAST_CONFIDENCE_LEVEL="$CONFIDENCE_LEVEL" - echo "WARNING: CONFIDENCE_LEVEL is deprecated and MUST be replaced with SAST_CONFIDENCE_LEVEL" - fi - - docker run --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}" \ - --volume "$PWD:/code" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code - ;; - *) - echo "GitLab EE is required" - ;; - esac - } - - function dependency_scanning() { - case "$CI_SERVER_VERSION" in - *-ee) - docker run --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}" \ - --volume "$PWD:/code" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code - ;; - *) - echo "GitLab EE is required" - ;; - esac - } - - # With the Kubernetes executor, 'localhost' must be used instead - # https://docs.gitlab.com/runner/executors/kubernetes.html - function setup_docker() { - if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then - export DOCKER_HOST='tcp://localhost:2375' - export DOCKER_SERVICE="localhost" - else - export DOCKER_SERVICE="docker" - fi - fi - } - - function dast() { - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) - - mkdir /zap/wrk/ - /zap/zap-baseline.py -J gl-dast-report.json -t "$CI_ENVIRONMENT_URL" || true - cp /zap/wrk/gl-dast-report.json . - } - -before_script: - - *auto_devops + - template: Jobs/DAST.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml + - template: Security/License-Management.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Jobs/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST.gitlab-ci.yml new file mode 100644 index 00000000000..aedbbb21674 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/DAST.gitlab-ci.yml @@ -0,0 +1,54 @@ +dast: + stage: dast + image: docker:stable + variables: + DOCKER_DRIVER: overlay2 + allow_failure: true + services: + - docker:stable-dind + script: + - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} + - export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} + - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - | + function dast_run() { + docker run \ + --env DAST_TARGET_AVAILABILITY_TIMEOUT \ + --volume "$PWD:/output" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + -w /output \ + "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \ + /analyze -t $DAST_WEBSITE \ + "$@" + } + - | + if [ -n "$DAST_AUTH_URL" ] + then + dast_run \ + --auth-url $DAST_AUTH_URL \ + --auth-username $DAST_USERNAME \ + --auth-password $DAST_PASSWORD \ + --auth-username-field $DAST_USERNAME_FIELD \ + --auth-password-field $DAST_PASSWORD_FIELD + else + dast_run + fi + artifacts: + reports: + dast: gl-dast-report.json + only: + refs: + - branches + - tags + variables: + - $GITLAB_FEATURES =~ /\bdast\b/ + except: + refs: + - master + variables: + - $DAST_DISABLED diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 0b7a531682b..eef361c19e9 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -28,6 +28,12 @@ container_scanning: - docker:stable-dind script: - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then { export DOCKER_SERVICE="localhost" ; export DOCKER_HOST="tcp://${DOCKER_SERVICE}:2375" ; } fi + - | + if [[ -n "$CI_REGISTRY_USER" ]]; then + echo "Logging to GitLab Container Registry with CI credentials..." + docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" + echo "" + fi - docker run -d --name db arminc/clair-db:latest - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:${CLAIR_LOCAL_SCAN_VERSION} - apk add -U wget ca-certificates @@ -36,7 +42,6 @@ container_scanning: - mv clair-scanner_linux_amd64 clair-scanner - chmod +x clair-scanner - touch clair-whitelist.yml - - while( ! wget -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; done - retries=0 - echo "Waiting for clair daemon to start" - while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index ef6d7866e85..770340de16f 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -4,6 +4,9 @@ # List of the variables: https://gitlab.com/gitlab-org/security-products/dast#settings # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +include: + - template: Jobs/DAST.gitlab-ci.yml + variables: DAST_WEBSITE: http://example.com # Please edit to be your website to scan for vulnerabilities @@ -14,46 +17,10 @@ stages: - dast dast: - stage: dast - image: docker:stable - variables: - DOCKER_DRIVER: overlay2 - allow_failure: true - services: - - docker:stable-dind - script: - - export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - - | - function dast_run() { - docker run \ - --env DAST_TARGET_AVAILABILITY_TIMEOUT \ - --volume "$PWD:/output" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - -w /output \ - "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \ - /analyze -t $DAST_WEBSITE \ - "$@" - } - - | - if [ -n "$DAST_AUTH_URL" ] - then - dast_run \ - --auth-url $DAST_AUTH_URL \ - --auth-username $DAST_USERNAME \ - --auth-password $DAST_PASSWORD \ - --auth-username-field $DAST_USERNAME_FIELD \ - --auth-password-field $DAST_PASSWORD_FIELD - else - dast_run - fi - artifacts: - reports: - dast: gl-dast-report.json only: refs: - branches - variables: - - $GITLAB_FEATURES =~ /\bdast\b/ except: + refs: [] # Override default from template variables: - $DAST_DISABLED diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index fd666541d41..7f80a6e9285 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -15,6 +15,12 @@ dependency_scanning: script: - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - | docker run \ --env DS_ANALYZER_IMAGES \ --env DS_ANALYZER_IMAGE_PREFIX \ diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 034fba5499c..b941e89991e 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -15,6 +15,12 @@ sast: script: - export SAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - | docker run \ --env SAST_ANALYZER_IMAGES \ --env SAST_ANALYZER_IMAGE_PREFIX \ diff --git a/spec/lib/gitlab/ci/templates/templates_spec.rb b/spec/lib/gitlab/ci/templates/templates_spec.rb index fbbd58280a9..e20a621168c 100644 --- a/spec/lib/gitlab/ci/templates/templates_spec.rb +++ b/spec/lib/gitlab/ci/templates/templates_spec.rb @@ -4,6 +4,7 @@ require 'spec_helper' describe "CI YML Templates" do ABSTRACT_TEMPLATES = %w[Serverless].freeze + PROJECT_DEPENDENT_TEMPLATES = %w[Auto-DevOps].freeze def self.concrete_templates Gitlab::Template::GitlabCiYmlTemplate.all.reject do |template| @@ -20,7 +21,10 @@ describe "CI YML Templates" do describe 'concrete templates with CI/CD jobs' do concrete_templates.each do |template| it "#{template.name} template should be valid" do - expect { Gitlab::Ci::YamlProcessor.new(template.content) } + # Trigger processing of included files + project = create(:project, :test_repo) if PROJECT_DEPENDENT_TEMPLATES.include?(template.name) + + expect { Gitlab::Ci::YamlProcessor.new(template.content, project: project) } .not_to raise_error end end |