diff options
Diffstat (limited to 'scripts')
-rw-r--r-- | scripts/allowed_warnings.txt | 15 | ||||
-rwxr-xr-x | scripts/build_qa_image | 4 | ||||
-rwxr-xr-x | scripts/decomposition/generate-loose-foreign-key | 2 | ||||
-rw-r--r-- | scripts/frontend/extract_gettext_all.js | 2 | ||||
-rw-r--r-- | scripts/gitlab_component_helpers.sh | 4 | ||||
-rwxr-xr-x | scripts/glfm/run-spec-tests.sh | 23 | ||||
-rw-r--r-- | scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb | 2 | ||||
-rwxr-xr-x | scripts/lint-doc-quality.sh | 24 | ||||
-rwxr-xr-x | scripts/review_apps/automated_cleanup.rb | 2 | ||||
-rw-r--r-- | scripts/review_apps/base-config.yaml | 2 | ||||
-rwxr-xr-x | scripts/review_apps/review-apps.sh | 44 | ||||
-rwxr-xr-x | scripts/rspec_check_order_dependence | 2 | ||||
-rwxr-xr-x | scripts/setup/as-if-jh.sh | 5 | ||||
-rwxr-xr-x | scripts/setup/find-jh-branch.rb | 2 | ||||
-rwxr-xr-x | scripts/static-analysis | 28 | ||||
-rwxr-xr-x | scripts/trigger-build.rb | 14 | ||||
-rw-r--r-- | scripts/utils.sh | 31 | ||||
-rwxr-xr-x | scripts/verify-tff-mapping | 14 |
18 files changed, 146 insertions, 74 deletions
diff --git a/scripts/allowed_warnings.txt b/scripts/allowed_warnings.txt new file mode 100644 index 00000000000..19bd5d51a20 --- /dev/null +++ b/scripts/allowed_warnings.txt @@ -0,0 +1,15 @@ +# List of ignored warnings used by `fail_on_warnings` in `scripts/utils.sh`. +# Each line is a regexp and represents a match used by +# `grep --invert-match --extended-regexp --file`. +# Comments and empty lines are ignored. + +# https://github.com/browserslist/browserslist/blob/d0ec62eb48c41c218478cd3ac28684df051cc865/node.js#L329 +# warns if caniuse-lite package is older than 6 months. Ignore this +# warning message so that GitLab backports don't fail. +Browserslist: caniuse-lite is outdated\. Please run next command `yarn upgrade` + +# https://github.com/mime-types/mime-types-data/pull/50#issuecomment-1060908930 +Type application/netcdf is already registered as a variant of application/netcdf\. + +# This warning is emitted by scripts/static-analysis. +\*\*\*\* .+ had the following warning\(s\): diff --git a/scripts/build_qa_image b/scripts/build_qa_image index 3728608e32c..c0001181a51 100755 --- a/scripts/build_qa_image +++ b/scripts/build_qa_image @@ -3,7 +3,7 @@ QA_IMAGE_NAME="gitlab-ee-qa" QA_BUILD_TARGET="ee" -if [[ "${CI_PROJECT_NAME}" == "gitlabhq" || "${CI_PROJECT_NAME}" == "gitlab-foss" ]]; then +if [[ "${CI_PROJECT_NAME}" == "gitlabhq" || "${CI_PROJECT_NAME}" == "gitlab-foss" || "${FOSS_ONLY}" == "1" ]]; then QA_IMAGE_NAME="gitlab-ce-qa" QA_BUILD_TARGET="foss" fi @@ -28,7 +28,7 @@ if [ "${QA_IMAGE_NAME}" == "gitlab-ee-qa" ]; then DESTINATIONS="${DESTINATIONS} --tag $QA_IMAGE_FOR_AUTO_DEPLOY" fi -echo "Building QA image for destinations: ${DESTINATIONS}" +echo "Building QA image for '${QA_BUILD_TARGET}' for destinations: ${DESTINATIONS}" docker buildx build \ --cache-to=type=inline \ diff --git a/scripts/decomposition/generate-loose-foreign-key b/scripts/decomposition/generate-loose-foreign-key index 3f4c510020a..2e91aa662de 100755 --- a/scripts/decomposition/generate-loose-foreign-key +++ b/scripts/decomposition/generate-loose-foreign-key @@ -253,7 +253,7 @@ end # Show only cross-schema foreign keys if $options[:cross_schema] all_foreign_keys.select! do |definition| - Gitlab::Database::GitlabSchema.table_schema(definition.from_table) != Gitlab::Database::GitlabSchema.table_schema(definition.to_table) + Gitlab::Database::GitlabSchema.table_schema!(definition.from_table) != Gitlab::Database::GitlabSchema.table_schema!(definition.to_table) end end diff --git a/scripts/frontend/extract_gettext_all.js b/scripts/frontend/extract_gettext_all.js index 0a5e2b06971..922aa85241f 100644 --- a/scripts/frontend/extract_gettext_all.js +++ b/scripts/frontend/extract_gettext_all.js @@ -6,7 +6,7 @@ const { decorateExtractorWithHelpers, } = require('gettext-extractor-vue'); const vue2TemplateCompiler = require('vue-template-compiler'); -const ensureSingleLine = require('../../app/assets/javascripts/locale/ensure_single_line'); +const ensureSingleLine = require('../../app/assets/javascripts/locale/ensure_single_line.cjs'); const args = argumentsParser .option('-f, --file <file>', 'Extract message from one single file') diff --git a/scripts/gitlab_component_helpers.sh b/scripts/gitlab_component_helpers.sh index 0d72f940036..c46dbb57a58 100644 --- a/scripts/gitlab_component_helpers.sh +++ b/scripts/gitlab_component_helpers.sh @@ -87,12 +87,10 @@ function upload_package() { function read_curl_package() { local package_url="${1}" - local token_header="${CURL_TOKEN_HEADER}" - local token="${CI_JOB_TOKEN}" echoinfo "Downloading from ${package_url} ..." - curl --fail --silent --retry 3 --header "${token_header}: ${token}" "${package_url}" + curl --fail --silent --retry 3 "${package_url}" } function extract_package() { diff --git a/scripts/glfm/run-spec-tests.sh b/scripts/glfm/run-spec-tests.sh new file mode 100755 index 00000000000..b60f6b05051 --- /dev/null +++ b/scripts/glfm/run-spec-tests.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2059 + +set -o errexit # AKA -e - exit immediately on errors (http://mywiki.wooledge.org/BashFAQ/105) + +# https://stackoverflow.com/a/28938235 +BCyan='\033[1;36m' # Bold Cyan +BRed='\033[1;31m' # Bold Red +Color_Off='\033[0m' # Text Reset + +function onexit_err() { + local exit_status=${1:-$?} + printf "\n❌❌❌ ${BRed}GLFM spec tests failed!${Color_Off} ❌❌❌\n" + exit "${exit_status}" +} +trap onexit_err ERR +set -o errexit + +printf "${BCyan}" +printf "\nThis script is not yet implemented!\n" +printf "\nSee https://docs.gitlab.com/ee/development/gitlab_flavored_markdown/specification_guide/#run-spec-testssh-script for more details.\n\n" +printf "${Color_Off}" diff --git a/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb b/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb index 3d4570f74e5..ef8872ecbb0 100644 --- a/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb +++ b/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb @@ -54,7 +54,7 @@ module Glfm # Ensure that the diff output is flushed and output before we raise and exit. $stderr.flush - raise('ERROR: The generated files are not up to date.') + raise('ERROR: The generated files are not up to date. The specification or examples may need to be updated. See https://docs.gitlab.com/ee/development/gitlab_flavored_markdown/specification_guide/#workflows') end end end diff --git a/scripts/lint-doc-quality.sh b/scripts/lint-doc-quality.sh deleted file mode 100755 index 9d8409a7c80..00000000000 --- a/scripts/lint-doc-quality.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -echo '=> Generating code quality artifact...' -echo - -# Generate code quality artifact for Vale warnings only on changed files. -# Only works on merged results pipelines, so first checks if a merged results CI variable is present. -# If not present, runs on all files. - -if [ -z "${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}" ] -then - MD_DOC_PATH=${MD_DOC_PATH:-doc} - echo "Merge request pipeline (detached) detected. Testing all files." -else - MERGE_BASE=$(git merge-base "${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}" "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}") - MD_DOC_PATH=$(git diff --diff-filter=d --name-only "${MERGE_BASE}..${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}" -- 'doc/*.md') - if [ -n "${MD_DOC_PATH}" ] - then - echo -e "Merged results pipeline detected. Testing only the following files: ${MD_DOC_PATH}" - fi -fi - -echo "vale --output=doc/.vale/vale-json.tmpl --minAlertLevel warning ${MD_DOC_PATH} > gl-code-quality-report-docs.json" -vale --output=doc/.vale/vale-json.tmpl --minAlertLevel warning ${MD_DOC_PATH} > gl-code-quality-report-docs.json diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb index 2d87c18d7d2..f459cb9c0ea 100755 --- a/scripts/review_apps/automated_cleanup.rb +++ b/scripts/review_apps/automated_cleanup.rb @@ -213,7 +213,7 @@ module ReviewApps gitlab.environments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc', states: 'stopped', search: ENVIRONMENT_PREFIX[environment_type]).auto_paginate do |environment| next if skip_environment?(environment: environment, checked_environments: checked_environments, last_updated_threshold: last_updated_threshold, environment_type: environment_type) - yield environment if delete_environment(environment) + yield environment if delete_environment(environment) && block_given? checked_environments << environment.slug end diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml index 0981aafec22..4d0f0ffec42 100644 --- a/scripts/review_apps/base-config.yaml +++ b/scripts/review_apps/base-config.yaml @@ -99,7 +99,7 @@ gitlab: cpu: 746m memory: 2809Mi limits: - cpu: 1300m + cpu: 1400m memory: 4214Mi minReplicas: 1 maxReplicas: 1 diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index e185ed43e38..6a72b19c631 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -299,21 +299,21 @@ HELM_CMD=$(cat << EOF --set global.appConfig.sentry.dsn="${REVIEW_APPS_SENTRY_DSN}" \ --set global.appConfig.sentry.environment="review" \ --set gitlab.migrations.image.repository="${gitlab_toolbox_image_repository}" \ - --set gitlab.migrations.image.tag="${CI_COMMIT_REF_SLUG}" \ + --set gitlab.migrations.image.tag="${CI_COMMIT_SHA}" \ --set gitlab.gitaly.image.repository="${gitlab_gitaly_image_repository}" \ --set gitlab.gitaly.image.tag="${gitaly_image_tag}" \ --set gitlab.gitlab-shell.image.repository="${gitlab_shell_image_repository}" \ --set gitlab.gitlab-shell.image.tag="v${GITLAB_SHELL_VERSION}" \ --set gitlab.sidekiq.annotations.commit="${CI_COMMIT_SHORT_SHA}" \ --set gitlab.sidekiq.image.repository="${gitlab_sidekiq_image_repository}" \ - --set gitlab.sidekiq.image.tag="${CI_COMMIT_REF_SLUG}" \ + --set gitlab.sidekiq.image.tag="${CI_COMMIT_SHA}" \ --set gitlab.webservice.annotations.commit="${CI_COMMIT_SHORT_SHA}" \ --set gitlab.webservice.image.repository="${gitlab_webservice_image_repository}" \ - --set gitlab.webservice.image.tag="${CI_COMMIT_REF_SLUG}" \ + --set gitlab.webservice.image.tag="${CI_COMMIT_SHA}" \ --set gitlab.webservice.workhorse.image="${gitlab_workhorse_image_repository}" \ - --set gitlab.webservice.workhorse.tag="${CI_COMMIT_REF_SLUG}" \ + --set gitlab.webservice.workhorse.tag="${CI_COMMIT_SHA}" \ --set gitlab.toolbox.image.repository="${gitlab_toolbox_image_repository}" \ - --set gitlab.toolbox.image.tag="${CI_COMMIT_REF_SLUG}" + --set gitlab.toolbox.image.tag="${CI_COMMIT_SHA}" EOF ) @@ -362,20 +362,46 @@ function verify_deploy() { if [[ "${deployed}" == "true" ]]; then echoinfo "[$(date '+%H:%M:%S')] Review app is deployed to ${CI_ENVIRONMENT_URL}" - return 0 else echoerr "[$(date '+%H:%M:%S')] Review app is not available at ${CI_ENVIRONMENT_URL}: see the logs from cURL above for more details" return 1 fi } +# We need to be able to access the GitLab API to run this method. +# Since we are creating a personal access token in `disable_sign_ups`, +# This method should be executed after it. +function verify_commit_sha() { + local verify_success="false" + + for i in {1..60}; do # try for 2 minutes in case review-apps containers are restarting + echoinfo "[$(date '+%H:%M:%S')] Checking the correct commit is deployed in the review-app:" + echo "Expected commit sha: ${CI_COMMIT_SHA}" + + review_app_revision=$(curl --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/metadata" | jq -r .revision) + echo "review-app revision: ${review_app_revision}" + + if [[ "${CI_COMMIT_SHA}" == "${review_app_revision}"* ]]; then + verify_success="true" + break + fi + + sleep 2 + done + + if [[ "${verify_success}" != "true" ]]; then + echoerr "[$(date '+%H:%M:%S')] Review app revision is not the same as the current commit!" + return 1 + fi + + return 0 +} + function display_deployment_debug() { local namespace="${CI_ENVIRONMENT_SLUG}" # Install dig to inspect DNS entries - # - # Silent install: see https://stackoverflow.com/a/52642167/1620195 - apt-get -qq update && apt-get -qq install -y dnsutils < /dev/null > /dev/null + apk add -q bind-tools echoinfo "[debugging data] Check review-app webservice DNS entry:" dig +short $(echo "${CI_ENVIRONMENT_URL}" | sed 's~http[s]*://~~g') diff --git a/scripts/rspec_check_order_dependence b/scripts/rspec_check_order_dependence index 91d4c5938a7..00f1176e251 100755 --- a/scripts/rspec_check_order_dependence +++ b/scripts/rspec_check_order_dependence @@ -40,6 +40,8 @@ done set -xe +export RSPEC_WARN_MISSING_FEATURE_CATEGORY=0 + bin/rspec --order defined "${RSPEC_ARGS[@]}" "$@" RSPEC_ORDER=reverse bin/rspec "${RSPEC_ARGS[@]}" "$@" bin/rspec --order random "${RSPEC_ARGS[@]}" "$@" diff --git a/scripts/setup/as-if-jh.sh b/scripts/setup/as-if-jh.sh index 38c3ac9b913..445a988b996 100755 --- a/scripts/setup/as-if-jh.sh +++ b/scripts/setup/as-if-jh.sh @@ -19,10 +19,9 @@ download_jh_path() { echoinfo "Downloading ${path}" - curl --location -o "${output}" -H "Private-Token: ${ADD_JH_FILES_TOKEN}" "https://gitlab.com/api/v4/projects/${GITLAB_JH_MIRROR_PROJECT}/repository/archive?sha=${JH_BRANCH}&path=${path}" + curl --location --output "${output}" --header "Private-Token: ${ADD_JH_FILES_TOKEN}" --get --data-urlencode "sha=${JH_BRANCH}" --data-urlencode "path=${path}" "https://gitlab.com/api/v4/projects/${GITLAB_JH_MIRROR_PROJECT}/repository/archive" - tar -zxf "${output}" + tar -zxf "${output}" --strip-component 1 rm "${output}" - mv gitlab-"${JH_BRANCH}"-*/"${path}" ./ done } diff --git a/scripts/setup/find-jh-branch.rb b/scripts/setup/find-jh-branch.rb index 5b36aa7a1f4..67e83423d9c 100755 --- a/scripts/setup/find-jh-branch.rb +++ b/scripts/setup/find-jh-branch.rb @@ -92,7 +92,7 @@ class FindJhBranch def gitlab @gitlab ||= Gitlab.client( endpoint: API::DEFAULT_OPTIONS[:endpoint], - private_token: API::DEFAULT_OPTIONS[:api_token] || '' + private_token: ENV['ADD_JH_FILES_TOKEN'] || '' ) end end diff --git a/scripts/static-analysis b/scripts/static-analysis index 9a0057d8f4d..0d03dd42c73 100755 --- a/scripts/static-analysis +++ b/scripts/static-analysis @@ -7,14 +7,7 @@ require_relative '../lib/gitlab/popen' require_relative '../lib/gitlab/popen/runner' class StaticAnalysis - ALLOWED_WARNINGS = [ - # https://github.com/browserslist/browserslist/blob/d0ec62eb48c41c218478cd3ac28684df051cc865/node.js#L329 - # warns if caniuse-lite package is older than 6 months. Ignore this - # warning message so that GitLab backports don't fail. - "Browserslist: caniuse-lite is outdated. Please run next command `yarn upgrade`", - # https://github.com/mime-types/mime-types-data/pull/50#issuecomment-1060908930 - "Type application/netcdf is already registered as a variant of application/netcdf." - ].freeze + # `ALLOWED_WARNINGS` moved to scripts/allowed_warnings.txt Task = Struct.new(:command, :duration) do def cmd @@ -94,12 +87,12 @@ class StaticAnalysis if static_analysis.all_success_and_clean? puts 'All static analyses passed successfully.' elsif static_analysis.all_success? - puts 'All static analyses passed successfully, but we have warnings:' + puts 'All static analyses passed successfully with warnings.' puts emit_warnings(static_analysis) - exit 2 if warning_count(static_analysis).nonzero? + # We used to exit 2 on warnings but `fail_on_warnings` takes care of it now. else puts 'Some static analyses failed:' @@ -112,11 +105,11 @@ class StaticAnalysis def emit_warnings(static_analysis) static_analysis.warned_results.each do |result| - puts - puts "**** #{result.cmd.join(' ')} had the following warning(s):" - puts - puts result.stderr - puts + warn + warn "**** #{result.cmd.join(' ')} had the following warning(s):" + warn + warn result.stderr + warn end end @@ -131,11 +124,6 @@ class StaticAnalysis end end - def warning_count(static_analysis) - static_analysis.warned_results - .count { |result| !ALLOWED_WARNINGS.include?(result.stderr.strip) } # rubocop:disable Rails/NegateInclude - end - def tasks_to_run(node_total) total_time = TASKS_WITH_DURATIONS_SECONDS.sum(&:duration).to_f ideal_time_per_node = total_time / node_total diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb index 411e5ed13c6..033c2e55329 100755 --- a/scripts/trigger-build.rb +++ b/scripts/trigger-build.rb @@ -184,6 +184,20 @@ module Trigger true end + def gitlab_ref_slug + if ENV['CI_COMMIT_TAG'] + ENV['CI_COMMIT_REF_NAME'] + else + ENV['CI_COMMIT_SHA'] + end + end + + def base_variables + super.merge( + 'GITLAB_REF_SLUG' => gitlab_ref_slug + ) + end + def extra_variables { "TRIGGER_BRANCH" => ref, diff --git a/scripts/utils.sh b/scripts/utils.sh index 92f647958fe..c71de666ac6 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -109,21 +109,38 @@ function install_junit_merge_gem() { function fail_on_warnings() { local cmd="$*" - local warnings - warnings="$(mktemp)" + local warning_file + warning_file="$(mktemp)" + + local allowed_warning_file + allowed_warning_file="$(mktemp)" - eval "$cmd 2>$warnings" + eval "$cmd 2>$warning_file" local ret=$? - if test -s "$warnings"; + # Filter out comments and empty lines from allowed warnings file. + grep --invert-match --extended-regexp "^#|^$" scripts/allowed_warnings.txt > "$allowed_warning_file" + + local warnings + # Filter out allowed warnings from stderr. + # Turn grep errors into warnings so we fail later. + warnings=$(grep --invert-match --extended-regexp --file "$allowed_warning_file" "$warning_file" 2>&1 || true) + + rm -f "$allowed_warning_file" + + if [ "$warnings" != "" ] then echoerr "There were warnings:" - cat "$warnings" - rm "$warnings" + echoerr "======================== Filtered warnings =====================================" + echo "$warnings" >&2 + echoerr "======================= Unfiltered warnings ====================================" + cat "$warning_file" >&2 + echoerr "================================================================================" + rm -f "$warning_file" return 1 fi - rm "$warnings" + rm -f "$warning_file" return $ret } diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping index 302e50bf34f..08d9d7a33fd 100755 --- a/scripts/verify-tff-mapping +++ b/scripts/verify-tff-mapping @@ -188,6 +188,20 @@ tests = [ explanation: 'GLFM spec and config files for CE and EE should map to respective markdown snapshot specs', source: 'glfm_specification/foo', expected: ['spec/requests/api/markdown_snapshot_spec.rb', 'ee/spec/requests/api/markdown_snapshot_spec.rb'] + }, + + { + explanation: 'https://gitlab.com/gitlab-org/quality/engineering-productivity/master-broken-incidents/-/issues/287#note_1192008962', + # Note: The metrics seem to be changed every year or so, so this test will fail once a year or so. + # You will need to change the metric below for another metric present in the project. + source: 'ee/config/metrics/counts_all/20221114065035_delete_merge_request.yml', + expected: ['ee/spec/config/metrics/every_metric_definition_spec.rb'] + }, + + { + explanation: 'https://gitlab.com/gitlab-org/quality/engineering-productivity/master-broken-incidents/-/issues/287#note_1192008962', + source: 'ee/lib/ee/gitlab/usage_data_counters/known_events/common.yml', + expected: ['ee/spec/config/metrics/every_metric_definition_spec.rb'] } ] |