diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/frontend/merge_coverage_frontend.js | 2 | ||||
| -rw-r--r-- | scripts/frontend/prettier.js | 2 | ||||
| -rw-r--r-- | scripts/gitaly_test.rb | 23 | ||||
| -rwxr-xr-x | scripts/lint-doc.sh | 5 | ||||
| -rwxr-xr-x | scripts/merge-simplecov | 1 | ||||
| -rw-r--r-- | scripts/prepare_build.sh | 4 | ||||
| -rwxr-xr-x | scripts/review_apps/automated_cleanup.rb | 16 | ||||
| -rw-r--r-- | scripts/review_apps/base-config.yaml | 14 | ||||
| -rwxr-xr-x | scripts/review_apps/gcp_cleanup.sh | 2 | ||||
| -rwxr-xr-x | scripts/review_apps/review-apps.sh | 74 | ||||
| -rw-r--r-- | scripts/rspec_helpers.sh | 4 | ||||
| -rwxr-xr-x | scripts/trigger-build | 1 |
12 files changed, 115 insertions, 33 deletions
diff --git a/scripts/frontend/merge_coverage_frontend.js b/scripts/frontend/merge_coverage_frontend.js index 507695b45e5..99034176b29 100644 --- a/scripts/frontend/merge_coverage_frontend.js +++ b/scripts/frontend/merge_coverage_frontend.js @@ -26,6 +26,6 @@ reportFiles const context = createContext({ coverageMap: coverageMap, dir: 'coverage-frontend' }); -['json', 'lcov', 'text-summary', 'clover'].forEach(reporter => { +['json', 'lcov', 'text-summary', 'clover', 'cobertura'].forEach(reporter => { create(reporter, {}).execute(context); }); diff --git a/scripts/frontend/prettier.js b/scripts/frontend/prettier.js index 45cdef2ba86..7772f80c233 100644 --- a/scripts/frontend/prettier.js +++ b/scripts/frontend/prettier.js @@ -3,7 +3,7 @@ const prettier = require('prettier'); const fs = require('fs'); const { getStagedFiles } = require('./frontend_script_utils'); -const matchExtensions = ['js', 'vue']; +const matchExtensions = ['js', 'vue', 'graphql']; // This will improve glob performance by excluding certain directories. // The .prettierignore file will also be respected, but after the glob has executed. diff --git a/scripts/gitaly_test.rb b/scripts/gitaly_test.rb index c69c4ea747b..54bf07b3773 100644 --- a/scripts/gitaly_test.rb +++ b/scripts/gitaly_test.rb @@ -6,8 +6,16 @@ require 'securerandom' require 'socket' +require 'logger' module GitalyTest + LOGGER = begin + default_name = ENV['CI'] ? 'DEBUG' : 'WARN' + level_name = ENV['GITLAB_TESTING_LOG_LEVEL']&.upcase + level = Logger.const_get(level_name || default_name, true) # rubocop: disable Gitlab/ConstGetInheritFalse + Logger.new(STDOUT, level: level, formatter: ->(_, _, _, msg) { msg }) + end + def tmp_tests_gitaly_dir File.expand_path('../tmp/tests/gitaly', __dir__) end @@ -98,7 +106,7 @@ module GitalyTest end def check_gitaly_config! - puts "Checking gitaly-ruby Gemfile..." + LOGGER.debug "Checking gitaly-ruby Gemfile...\n" unless File.exist?(gemfile) message = "#{gemfile} does not exist." @@ -106,8 +114,9 @@ module GitalyTest abort message end - puts 'Checking gitaly-ruby bundle...' - abort 'bundle check failed' unless system(env, 'bundle', 'check', chdir: File.dirname(gemfile)) + LOGGER.debug "Checking gitaly-ruby bundle...\n" + out = ENV['CI'] ? STDOUT : '/dev/null' + abort 'bundle check failed' unless system(env, 'bundle', 'check', out: out, chdir: File.dirname(gemfile)) end def read_socket_path(service) @@ -126,22 +135,22 @@ module GitalyTest end def try_connect!(service) - print "Trying to connect to #{service}: " + LOGGER.debug "Trying to connect to #{service}: " timeout = 20 delay = 0.1 socket = read_socket_path(service) Integer(timeout / delay).times do UNIXSocket.new(socket) - puts ' OK' + LOGGER.debug " OK\n" return rescue Errno::ENOENT, Errno::ECONNREFUSED - print '.' + LOGGER.debug '.' sleep delay end - puts ' FAILED' + LOGGER.warn " FAILED to connect to #{service}\n" raise "could not connect to #{socket}" end diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh index ca9747987df..72e6334d0fc 100755 --- a/scripts/lint-doc.sh +++ b/scripts/lint-doc.sh @@ -41,14 +41,15 @@ fi # Do not use 'README.md', instead use 'index.md' # Number of 'README.md's as of 2020-05-28 -NUMBER_READMES=45 +NUMBER_READMES=44 FIND_READMES=$(find doc/ -name "README.md" | wc -l) echo '=> Checking for new README.md files...' echo if [ ${FIND_READMES} -ne $NUMBER_READMES ] then echo - echo ' ✖ ERROR: New README.md file(s) detected, prefer index.md over README.md.' >&2 + echo ' ✖ ERROR: The number of README.md file(s) has changed. Use index.md instead of README.md.' >&2 + echo ' ✖ If removing a README.md file, update NUMBER_READMES in lint-doc.sh.' >&2 echo ' https://docs.gitlab.com/ee/development/documentation/styleguide.html#work-with-directories-and-files' echo ((ERRORCODE++)) diff --git a/scripts/merge-simplecov b/scripts/merge-simplecov index 746be3317a7..c00dae81c4d 100755 --- a/scripts/merge-simplecov +++ b/scripts/merge-simplecov @@ -2,6 +2,7 @@ require_relative '../spec/simplecov_env' SimpleCovEnv.configure_profile +SimpleCovEnv.configure_formatter module SimpleCov module ResultMerger diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index d2cc16f4f8b..1243609dc24 100644 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -2,7 +2,7 @@ export SETUP_DB=${SETUP_DB:-true} export USE_BUNDLE_INSTALL=${USE_BUNDLE_INSTALL:-true} -export BUNDLE_INSTALL_FLAGS="--without=production --jobs=$(nproc) --path=vendor --retry=3 --quiet" +export BUNDLE_INSTALL_FLAGS=${BUNDLE_INSTALL_FLAGS:-"--without=production --without=development --jobs=$(nproc) --path=vendor --retry=3 --quiet"} if [ "$USE_BUNDLE_INSTALL" != "false" ]; then bundle --version @@ -11,7 +11,7 @@ if [ "$USE_BUNDLE_INSTALL" != "false" ]; then # When we test multiple versions of PG in the same pipeline, we have a single `setup-test-env` # job but the `pg` gem needs to be rebuilt since it includes extensions (https://guides.rubygems.org/gems-with-extensions). # Uncomment the following line if multiple versions of PG are tested in the same pipeline. - # run_timed_command "bundle pristine pg" + run_timed_command "bundle pristine pg" fi # Only install knapsack after bundle install! Otherwise oddly some native diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb index a9659071a2f..e40c6cd7276 100755 --- a/scripts/review_apps/automated_cleanup.rb +++ b/scripts/review_apps/automated_cleanup.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true require 'gitlab' -require_relative File.expand_path('../../lib/quality/helm3_client.rb', __dir__) -require_relative File.expand_path('../../lib/quality/kubernetes_client.rb', __dir__) +require_relative File.expand_path('../../tooling/lib/tooling/helm3_client.rb', __dir__) +require_relative File.expand_path('../../tooling/lib/tooling/kubernetes_client.rb', __dir__) class AutomatedCleanup attr_reader :project_path, :gitlab_token @@ -40,15 +40,15 @@ class AutomatedCleanup end def review_apps_namespace - self.class.ee? ? 'review-apps-ee' : 'review-apps-ce' + 'review-apps' end def helm - @helm ||= Quality::Helm3Client.new(namespace: review_apps_namespace) + @helm ||= Tooling::Helm3Client.new(namespace: review_apps_namespace) end def kubernetes - @kubernetes ||= Quality::KubernetesClient.new(namespace: review_apps_namespace) + @kubernetes ||= Tooling::KubernetesClient.new(namespace: review_apps_namespace) end def perform_gitlab_environment_cleanup!(days_for_stop:, days_for_delete:) @@ -76,7 +76,7 @@ class AutomatedCleanup if deployed_at < delete_threshold deleted_environment = delete_environment(environment, deployment) if deleted_environment - release = Quality::Helm3Client::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace) + release = Tooling::Helm3Client::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace) releases_to_delete << release end else @@ -157,11 +157,11 @@ class AutomatedCleanup helm.delete(release_name: releases_names) kubernetes.cleanup(release_name: releases_names, wait: false) - rescue Quality::Helm3Client::CommandFailedError => ex + rescue Tooling::Helm3Client::CommandFailedError => ex raise ex unless ignore_exception?(ex.message, IGNORED_HELM_ERRORS) puts "Ignoring the following Helm error:\n#{ex}\n" - rescue Quality::KubernetesClient::CommandFailedError => ex + rescue Tooling::KubernetesClient::CommandFailedError => ex raise ex unless ignore_exception?(ex.message, IGNORED_KUBERNETES_ERRORS) puts "Ignoring the following Kubernetes error:\n#{ex}\n" diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml index 6fb6943fb90..82be2d3a691 100644 --- a/scripts/review_apps/base-config.yaml +++ b/scripts/review_apps/base-config.yaml @@ -7,7 +7,7 @@ global: external-dns.alpha.kubernetes.io/ttl: 10 configureCertmanager: false tls: - secretName: tls-cert + secretName: review-apps-tls initialRootPassword: secret: shared-gitlab-initial-root-password certmanager: @@ -61,11 +61,11 @@ gitlab: task-runner: resources: requests: - cpu: 50m - memory: 350M + cpu: 300m + memory: 800M limits: - cpu: 100m - memory: 700M + cpu: 450m + memory: 1200M webservice: resources: requests: @@ -137,10 +137,10 @@ postgresql: enabled: false resources: requests: - cpu: 347m + cpu: 550m memory: 250M limits: - cpu: 520m + cpu: 825m memory: 375M prometheus: install: false diff --git a/scripts/review_apps/gcp_cleanup.sh b/scripts/review_apps/gcp_cleanup.sh index f289a50f629..3225631e8c7 100755 --- a/scripts/review_apps/gcp_cleanup.sh +++ b/scripts/review_apps/gcp_cleanup.sh @@ -11,7 +11,7 @@ function setup_gcp_dependencies() { # These scripts require the following environment variables: # - REVIEW_APPS_GCP_REGION - e.g `us-central1` -# - KUBE_NAMESPACE - e.g `review-apps-ee` +# - KUBE_NAMESPACE - e.g `review-apps` function delete_firewall_rules() { if [[ ${#@} -eq 0 ]]; then diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index 1214ee5f462..74291f6aef4 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -66,7 +66,7 @@ function kubectl_cleanup_release() { local release="${2}" echoinfo "Deleting all K8s resources matching '${release}'..." true - kubectl --namespace "${namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd 2>&1 \ + kubectl --namespace "${namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,clusterrole,clusterrolebinding,role,rolebinding,sa,crd 2>&1 \ | grep "${release}" \ | awk '{print $1}' \ | xargs kubectl --namespace "${namespace}" delete \ @@ -126,6 +126,38 @@ function get_pod() { echo "${pod_name}" } +function run_task() { + local namespace="${KUBE_NAMESPACE}" + local ruby_cmd="${1}" + local task_runner_pod=$(get_pod "task-runner") + + kubectl exec -it --namespace "${namespace}" "${task_runner_pod}" -- gitlab-rails runner "${ruby_cmd}" +} + +function disable_sign_ups() { + if [ -z ${REVIEW_APPS_ROOT_TOKEN+x} ]; then + echoerr "In order to protect Review Apps, REVIEW_APPS_ROOT_TOKEN variable must be set" + false + else + true + fi + + # Create the root token + local ruby_cmd="token = User.find_by_username('root').personal_access_tokens.create(scopes: [:api], name: 'Token to disable sign-ups'); token.set_token('${REVIEW_APPS_ROOT_TOKEN}'); begin; token.save!; rescue(ActiveRecord::RecordNotUnique); end" + run_task "${ruby_cmd}" + + # Disable sign-ups + curl --silent --show-error --request PUT --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings?signup_enabled=false" + + local signup_enabled=$(curl --silent --show-error --request GET --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings" | jq ".signup_enabled") + if [[ "${signup_enabled}" == "false" ]]; then + echoinfo "Sign-ups have been disabled successfully." + else + echoerr "Sign-ups should be disabled but are still enabled!" + false + fi +} + function check_kube_domain() { echoinfo "Checking that Kube domain exists..." true @@ -181,6 +213,32 @@ function install_external_dns() { fi } +# This script is used to install cert-manager in the cluster +# The installation steps are documented in +# https://gitlab.com/gitlab-org/quality/team-tasks/snippets/1990286 +function install_certmanager() { + local namespace="${KUBE_NAMESPACE}" + local release="cert-manager-review-app-helm3" + + echoinfo "Installing cert-manager..." true + + if ! deploy_exists "${namespace}" "${release}" || previous_deploy_failed "${namespace}" "${release}" ; then + kubectl apply \ + -f https://raw.githubusercontent.com/jetstack/cert-manager/release-0.10/deploy/manifests/00-crds.yaml + + echoinfo "Installing cert-manager Helm chart" + helm repo add jetstack https://charts.jetstack.io + helm repo update + + helm install "${release}" jetstack/cert-manager \ + --namespace "${namespace}" \ + --version v0.15.1 \ + --set installCRDS=true + else + echoinfo "The cert-manager Helm chart is already successfully deployed." + fi +} + function create_application_secret() { local namespace="${KUBE_NAMESPACE}" local release="${CI_ENVIRONMENT_SLUG}" @@ -233,6 +291,17 @@ function base_config_changed() { curl "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/changes" | jq '.changes | any(.old_path == "scripts/review_apps/base-config.yaml")' } +function parse_gitaly_image_tag() { + local gitaly_version="${GITALY_VERSION}" + + # prepend semver version with `v` + if [[ $gitaly_version =~ ^[0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?(-ee)?$ ]]; then + echo "v${gitaly_version}" + else + echo "${gitaly_version}" + fi +} + function deploy() { local namespace="${KUBE_NAMESPACE}" local release="${CI_ENVIRONMENT_SLUG}" @@ -248,6 +317,7 @@ function deploy() { gitlab_webservice_image_repository="${IMAGE_REPOSITORY}/gitlab-webservice-ee" gitlab_task_runner_image_repository="${IMAGE_REPOSITORY}/gitlab-task-runner-ee" gitlab_gitaly_image_repository="${IMAGE_REPOSITORY}/gitaly" + gitaly_image_tag=$(parse_gitaly_image_tag) gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell" gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-ee" @@ -269,7 +339,7 @@ HELM_CMD=$(cat << EOF --set gitlab.migrations.image.repository="${gitlab_migrations_image_repository}" \ --set gitlab.migrations.image.tag="${CI_COMMIT_REF_SLUG}" \ --set gitlab.gitaly.image.repository="${gitlab_gitaly_image_repository}" \ - --set gitlab.gitaly.image.tag="v${GITALY_VERSION}" \ + --set gitlab.gitaly.image.tag="${gitaly_image_tag}" \ --set gitlab.gitlab-shell.image.repository="${gitlab_shell_image_repository}" \ --set gitlab.gitlab-shell.image.tag="v${GITLAB_SHELL_VERSION}" \ --set gitlab.sidekiq.annotations.commit="${CI_COMMIT_SHORT_SHA}" \ diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index 110567a15be..1ecf9a566d7 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -119,12 +119,12 @@ function rspec_matched_tests() { local test_file_count=$(wc -w "${matching_tests_file}" | awk {'print $1'}) if [[ "${test_file_count}" -gt "${test_file_count_threshold}" ]]; then - echo "There are more than ${test_file_count_threshold} FOSS test files matched," + echo "This job is intentionally failed because there are more than ${test_file_count_threshold} FOSS test files matched," echo "which would take too long to run in this job." echo "To reduce the likelihood of breaking FOSS pipelines," echo "please add [RUN AS-IF-FOSS] to the MR title and restart the pipeline." echo "This would run all as-if-foss jobs in this merge request" - echo "and remove this job from the pipeline." + echo "and remove this failing job from the pipeline." exit 1 fi diff --git a/scripts/trigger-build b/scripts/trigger-build index b8bea95a069..9f0df21e7f1 100755 --- a/scripts/trigger-build +++ b/scripts/trigger-build @@ -184,6 +184,7 @@ module Trigger edition = Trigger.ee? ? 'EE' : 'CE' { + "ee" => Trigger.ee? ? "true" : "false", "GITLAB_VERSION" => ENV['CI_COMMIT_SHA'], "GITLAB_TAG" => ENV['CI_COMMIT_TAG'], "GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_SHA'], |