diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-19 09:35:24 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-19 09:35:24 +0000 |
commit | 1d20d43661f3e98bde2bb5dc163a45fd90f8ac88 (patch) | |
tree | 53feb6983348a364ed4f52a4108866bf4ff0a599 /scripts | |
parent | 2774ddc308f96f49a0f26871ff544681229f4eee (diff) | |
download | gitlab-ce-1d20d43661f3e98bde2bb5dc163a45fd90f8ac88.tar.gz |
Add latest changes from gitlab-org/gitlab@12-9-stable-ee
Diffstat (limited to 'scripts')
-rwxr-xr-x | scripts/clean-old-cached-assets | 2 | ||||
-rw-r--r-- | scripts/frontend/block_dependencies.js | 21 | ||||
-rw-r--r-- | scripts/frontend/merge_coverage_frontend.js | 31 | ||||
-rw-r--r-- | scripts/frontend/parallel_ci_sequencer.js | 41 | ||||
-rwxr-xr-x | scripts/gemfile_lock_changed.sh | 26 | ||||
-rwxr-xr-x | scripts/lint-doc.sh | 59 | ||||
-rwxr-xr-x | scripts/lint-rugged | 6 | ||||
-rwxr-xr-x | scripts/review_apps/automated_cleanup.rb | 34 | ||||
-rw-r--r-- | scripts/review_apps/base-config.yaml | 24 | ||||
-rwxr-xr-x | scripts/security-harness | 6 | ||||
-rwxr-xr-x | scripts/static-analysis | 4 | ||||
-rwxr-xr-x | scripts/trigger-build | 24 | ||||
-rw-r--r-- | scripts/utils.sh | 94 |
13 files changed, 236 insertions, 136 deletions
diff --git a/scripts/clean-old-cached-assets b/scripts/clean-old-cached-assets index 8bdd3a9cdb6..9a373439e5e 100755 --- a/scripts/clean-old-cached-assets +++ b/scripts/clean-old-cached-assets @@ -2,5 +2,7 @@ # Clean up cached files that are older than 4 days find tmp/cache/assets/sprockets/ -type f -mtime +4 -execdir rm -- "{}" \; +find tmp/cache/webpack-dlls/ -maxdepth 1 -type d -mtime +4 -exec rm -rf -- "{}" \; du -d 0 -h tmp/cache/assets/sprockets | cut -f1 | xargs -I % echo "tmp/cache/assets/sprockets/ is currently %" +du -d 0 -h tmp/cache/webpack-dlls | cut -f1 | xargs -I % echo "tmp/cache/webpack-dlls is currently %" diff --git a/scripts/frontend/block_dependencies.js b/scripts/frontend/block_dependencies.js new file mode 100644 index 00000000000..c9257c9f72b --- /dev/null +++ b/scripts/frontend/block_dependencies.js @@ -0,0 +1,21 @@ +const path = require('path'); +const packageJson = require(path.join(process.cwd(), 'package.json')); +const blockedDependencies = packageJson.blockedDependencies || {}; +const dependencies = packageJson.dependencies; +const devDependencies = packageJson.devDependencies; +const blockedDependenciesNames = Object.keys(blockedDependencies); +const blockedDependenciesFound = blockedDependenciesNames.filter( + blockedDependency => dependencies[blockedDependency] || devDependencies[blockedDependency], +); + +if (blockedDependenciesFound.length) { + console.log('The following package.json dependencies are not allowed:'); + + blockedDependenciesFound.forEach(blockedDependency => { + const infoLink = blockedDependencies[blockedDependency]; + + console.log(`- ${blockedDependency}: See ${infoLink} for more information.`); + }); + + process.exit(-1); +} diff --git a/scripts/frontend/merge_coverage_frontend.js b/scripts/frontend/merge_coverage_frontend.js new file mode 100644 index 00000000000..507695b45e5 --- /dev/null +++ b/scripts/frontend/merge_coverage_frontend.js @@ -0,0 +1,31 @@ +const { create } = require('istanbul-reports'); +const { createCoverageMap } = require('istanbul-lib-coverage'); +const { createContext } = require('istanbul-lib-report'); +const { resolve } = require('path'); +const { sync } = require('glob'); + +const coverageMap = createCoverageMap(); + +const coverageDir = resolve(__dirname, '../../coverage-frontend'); +const reportFiles = sync(`${coverageDir}/*/coverage-final.json`); + +// Normalize coverage report generated by jest that has additional "data" key +// https://github.com/facebook/jest/issues/2418#issuecomment-423806659 +const normalizeReport = report => { + const normalizedReport = Object.assign({}, report); + Object.entries(normalizedReport).forEach(([k, v]) => { + if (v.data) normalizedReport[k] = v.data; + }); + return normalizedReport; +}; + +reportFiles + .map(reportFile => require(reportFile)) + .map(normalizeReport) + .forEach(report => coverageMap.merge(report)); + +const context = createContext({ coverageMap: coverageMap, dir: 'coverage-frontend' }); + +['json', 'lcov', 'text-summary', 'clover'].forEach(reporter => { + create(reporter, {}).execute(context); +}); diff --git a/scripts/frontend/parallel_ci_sequencer.js b/scripts/frontend/parallel_ci_sequencer.js new file mode 100644 index 00000000000..d7a674535a6 --- /dev/null +++ b/scripts/frontend/parallel_ci_sequencer.js @@ -0,0 +1,41 @@ +const Sequencer = require('@jest/test-sequencer').default; + +class ParallelCISequencer extends Sequencer { + constructor() { + super(); + this.ciNodeIndex = Number(process.env.CI_NODE_INDEX || '1'); + this.ciNodeTotal = Number(process.env.CI_NODE_TOTAL || '1'); + } + + sort(tests) { + const sortedTests = this.sortByPath(tests); + const testsForThisRunner = this.distributeAcrossCINodes(sortedTests); + + console.log(`CI_NODE_INDEX: ${this.ciNodeIndex}`); + console.log(`CI_NODE_TOTAL: ${this.ciNodeTotal}`); + console.log(`Total number of tests: ${tests.length}`); + console.log(`Total number of tests for this runner: ${testsForThisRunner.length}`); + + return testsForThisRunner; + } + + sortByPath(tests) { + return tests.sort((test1, test2) => { + if (test1.path < test2.path) { + return -1; + } + if (test1.path > test2.path) { + return 1; + } + return 0; + }); + } + + distributeAcrossCINodes(tests) { + return tests.filter((test, index) => { + return index % this.ciNodeTotal === this.ciNodeIndex - 1; + }); + } +} + +module.exports = ParallelCISequencer; diff --git a/scripts/gemfile_lock_changed.sh b/scripts/gemfile_lock_changed.sh new file mode 100755 index 00000000000..24e2c685f11 --- /dev/null +++ b/scripts/gemfile_lock_changed.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +gemfile_lock_changed() { + if [ -n "$(git diff --name-only -- Gemfile.lock)" ]; then + cat << EOF + Gemfile was updated but Gemfile.lock was not updated. + + Usually, when Gemfile is updated, you should run + \`\`\` + bundle install + \`\`\` + + or + + \`\`\` + bundle update <the-added-or-updated-gem> + \`\`\` + + and commit the Gemfile.lock changes. +EOF + + exit 1 + fi +} + +gemfile_lock_changed diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh index 69c7a56c34f..d9de48fd0c0 100755 --- a/scripts/lint-doc.sh +++ b/scripts/lint-doc.sh @@ -2,36 +2,41 @@ cd "$(dirname "$0")/.." echo "=> Linting documents at path $(pwd) as $(whoami)..." +echo +ERRORCODE=0 # Use long options (e.g. --header instead of -H) for curl examples in documentation. echo '=> Checking for cURL short options...' +echo grep --extended-regexp --recursive --color=auto 'curl (.+ )?-[^- ].*' doc/ >/dev/null 2>&1 if [ $? -eq 0 ] then echo '✖ ERROR: Short options for curl should not be used in documentation! Use long options (e.g., --header instead of -H):' >&2 grep --extended-regexp --recursive --color=auto 'curl (.+ )?-[^- ].*' doc/ - exit 1 + ((ERRORCODE++)) fi # Ensure that the CHANGELOG.md does not contain duplicate versions DUPLICATE_CHANGELOG_VERSIONS=$(grep --extended-regexp '^## .+' CHANGELOG.md | sed -E 's| \(.+\)||' | sort -r | uniq -d) echo '=> Checking for CHANGELOG.md duplicate entries...' +echo if [ "${DUPLICATE_CHANGELOG_VERSIONS}" != "" ] then echo '✖ ERROR: Duplicate versions in CHANGELOG.md:' >&2 echo "${DUPLICATE_CHANGELOG_VERSIONS}" >&2 - exit 1 + ((ERRORCODE++)) fi # Make sure no files in doc/ are executable EXEC_PERM_COUNT=$(find doc/ -type f -perm 755 | wc -l) echo "=> Checking $(pwd)/doc for executable permissions..." +echo if [ "${EXEC_PERM_COUNT}" -ne 0 ] then echo '✖ ERROR: Executable permissions should not be used in documentation! Use `chmod 644` to the files in question:' >&2 find doc/ -type f -perm 755 - exit 1 + ((ERRORCODE++)) fi # Do not use 'README.md', instead use 'index.md' @@ -39,14 +44,56 @@ fi NUMBER_READMES=46 FIND_READMES=$(find doc/ -name "README.md" | wc -l) echo '=> Checking for new README.md files...' +echo if [ ${FIND_READMES} -ne $NUMBER_READMES ] then echo echo ' ✖ ERROR: New README.md file(s) detected, prefer index.md over README.md.' >&2 echo ' https://docs.gitlab.com/ee/development/documentation/styleguide.html#work-with-directories-and-files' echo - exit 1 + ((ERRORCODE++)) fi -echo "✔ Linting passed" -exit 0 +MD_DOC_PATH=${MD_DOC_PATH:-doc} + +function run_locally_or_in_docker() { + local cmd=$1 + local args=$2 + + if hash ${cmd} 2>/dev/null + then + $cmd $args + elif hash docker 2>/dev/null + then + docker run -t -v ${PWD}:/gitlab -w /gitlab --rm registry.gitlab.com/gitlab-org/gitlab-docs:lint ${cmd} ${args} + else + echo + echo " ✖ ERROR: '${cmd}' not found. Install '${cmd}' or Docker to proceed." >&2 + echo + ((ERRORCODE++)) + fi + + if [ $? -ne 0 ] + then + echo + echo " ✖ ERROR: '${cmd}' failed with errors." >&2 + echo + ((ERRORCODE++)) + fi +} + +echo '=> Linting markdown style...' +echo +run_locally_or_in_docker 'markdownlint' "--config .markdownlint.json ${MD_DOC_PATH}" + +echo '=> Linting prose...' +run_locally_or_in_docker 'vale' "--minAlertLevel error ${MD_DOC_PATH}" + +if [ $ERRORCODE -ne 0 ] +then + echo "✖ ${ERRORCODE} lint test(s) failed. Review the log carefully to see full listing." + exit 1 +else + echo "✔ Linting passed" + exit 0 +fi diff --git a/scripts/lint-rugged b/scripts/lint-rugged index 1b3fb54f70b..dfa4df8333f 100755 --- a/scripts/lint-rugged +++ b/scripts/lint-rugged @@ -15,6 +15,9 @@ ALLOWED = [ 'lib/gitlab/git/rugged_impl/', 'lib/gitlab/gitaly_client/storage_settings.rb', + # Needed to detect Rugged enabled: https://gitlab.com/gitlab-org/gitlab/issues/35371 + 'lib/gitlab/config_checker/puma_rugged_checker.rb', + # Needed for logging 'config/initializers/peek.rb', 'config/initializers/lograge.rb', @@ -27,7 +30,8 @@ ALLOWED = [ rugged_lines = IO.popen(%w[git grep -i -n rugged -- app config lib], &:read).lines rugged_lines = rugged_lines.select { |l| /^[^:]*\.rb:/ =~ l } rugged_lines = rugged_lines.reject { |l| l.start_with?(*ALLOWED) } -rugged_lines = rugged_lines.reject { |l| /(include|prepend) Gitlab::Git::RuggedImpl/ =~ l} +rugged_lines = rugged_lines.reject { |l| /(include|prepend) Gitlab::Git::RuggedImpl/ =~ l } +rugged_lines = rugged_lines.reject { |l| l.include?('Gitlab::ConfigChecker::PumaRuggedChecker.check') } rugged_lines = rugged_lines.reject do |line| code, _comment = line.split('# ', 2) code !~ /rugged/i diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb index 8a04d8e00bc..0a073a28bf3 100755 --- a/scripts/review_apps/automated_cleanup.rb +++ b/scripts/review_apps/automated_cleanup.rb @@ -54,7 +54,7 @@ class AutomatedCleanup end def perform_gitlab_environment_cleanup!(days_for_stop:, days_for_delete:) - puts "Checking for review apps not updated in the last #{days_for_stop} days..." + puts "Checking for Review Apps not updated in the last #{days_for_stop} days..." checked_environments = [] delete_threshold = threshold_time(days: days_for_delete) @@ -81,10 +81,13 @@ class AutomatedCleanup release = Quality::HelmClient::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace) releases_to_delete << release end - elsif deployed_at < stop_threshold - stop_environment(environment, deployment) else - print_release_state(subject: 'Review app', release_name: environment.slug, release_date: last_deploy, action: 'leaving') + if deployed_at >= stop_threshold + print_release_state(subject: 'Review App', release_name: environment.slug, release_date: last_deploy, action: 'leaving') + else + environment_state = fetch_environment(environment)&.state + stop_environment(environment, deployment) if environment_state && environment_state != 'stopped' + end end checked_environments << environment.slug @@ -94,9 +97,9 @@ class AutomatedCleanup end def perform_helm_releases_cleanup!(days:) - puts "Checking for Helm releases not updated in the last #{days} days..." + puts "Checking for Helm releases that are FAILED or not updated in the last #{days} days..." - threshold_day = threshold_time(days: days) + threshold = threshold_time(days: days) releases_to_delete = [] @@ -104,7 +107,7 @@ class AutomatedCleanup # Prevents deleting `dns-gitlab-review-app` releases or other unrelated releases next unless release.name.start_with?('review-') - if release.status == 'FAILED' || release.last_update < threshold_day + if release.status == 'FAILED' || release.last_update < threshold releases_to_delete << release else print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'leaving') @@ -116,12 +119,19 @@ class AutomatedCleanup private + def fetch_environment(environment) + gitlab.environment(project_path, environment.id) + rescue Errno::ETIMEDOUT => ex + puts "Failed to fetch '#{environment.name}' / '#{environment.slug}' (##{environment.id}):\n#{ex.message}" + nil + end + def delete_environment(environment, deployment) print_release_state(subject: 'Review app', release_name: environment.slug, release_date: deployment.created_at, action: 'deleting') gitlab.delete_environment(project_path, environment.id) rescue Gitlab::Error::Forbidden - puts "Review app '#{environment.slug}' is forbidden: skipping it" + puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it" end def stop_environment(environment, deployment) @@ -129,7 +139,7 @@ class AutomatedCleanup gitlab.stop_environment(project_path, environment.id) rescue Gitlab::Error::Forbidden - puts "Review app '#{environment.slug}' is forbidden: skipping it" + puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it" end def helm_releases @@ -180,14 +190,14 @@ end automated_cleanup = AutomatedCleanup.new -timed('Review apps cleanup') do - automated_cleanup.perform_gitlab_environment_cleanup!(days_for_stop: 2, days_for_delete: 3) +timed('Review Apps cleanup') do + automated_cleanup.perform_gitlab_environment_cleanup!(days_for_stop: 5, days_for_delete: 6) end puts timed('Helm releases cleanup') do - automated_cleanup.perform_helm_releases_cleanup!(days: 3) + automated_cleanup.perform_helm_releases_cleanup!(days: 7) end exit(0) diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml index 95b0295622a..a54dadb8d85 100644 --- a/scripts/review_apps/base-config.yaml +++ b/scripts/review_apps/base-config.yaml @@ -17,10 +17,10 @@ gitlab: resources: requests: cpu: 1200m - memory: 240M + memory: 245M limits: cpu: 1800m - memory: 360M + memory: 367M persistence: size: 10G gitlab-exporter: @@ -51,11 +51,11 @@ gitlab: sidekiq: resources: requests: - cpu: 650m - memory: 1018M + cpu: 855m + memory: 1071M limits: - cpu: 975m - memory: 1527M + cpu: 1282m + memory: 1606M hpa: targetAverageValue: 650m task-runner: @@ -69,11 +69,11 @@ gitlab: unicorn: resources: requests: - cpu: 525m - memory: 1711M + cpu: 746m + memory: 1873M limits: - cpu: 787m - memory: 2566M + cpu: 1119m + memory: 2809M deployment: readinessProbe: initialDelaySeconds: 5 # Default is 0 @@ -140,10 +140,10 @@ postgresql: enabled: false resources: requests: - cpu: 300m + cpu: 347m memory: 250M limits: - cpu: 450m + cpu: 520m memory: 375M prometheus: install: false diff --git a/scripts/security-harness b/scripts/security-harness index a1642489fe2..c101cd03454 100755 --- a/scripts/security-harness +++ b/scripts/security-harness @@ -28,9 +28,9 @@ HOOK_DATA = <<~HOOK if [ -e "$harness" ] then - if [[ ("$url" != *"dev.gitlab.org"*) && ("$url" != *"gitlab-org/security/"*) ]] + if [[ "$url" != *"gitlab-org/security/"* ]] then - echo "Pushing to remotes other than dev.gitlab.org and gitlab.com/gitlab-org/security has been disabled!" + echo "Pushing to remotes other than gitlab.com/gitlab-org/security has been disabled!" echo "Run scripts/security-harness to disable this check." echo @@ -58,7 +58,7 @@ def toggle else FileUtils.touch(harness_path) - puts "#{SHELL_GREEN}Security harness installed -- you will only be able to push to dev.gitlab.org or gitlab.com/gitlab-org/security!#{SHELL_CLEAR}" + puts "#{SHELL_GREEN}Security harness installed -- you will only be able to push to gitlab.com/gitlab-org/security!#{SHELL_CLEAR}" end end diff --git a/scripts/static-analysis b/scripts/static-analysis index 1f55c035ed1..ede29b85b8d 100755 --- a/scripts/static-analysis +++ b/scripts/static-analysis @@ -45,11 +45,13 @@ def jobs_to_run(node_index, node_total) %w[yarn run eslint], %w[yarn run stylelint], %w[yarn run prettier-all], + %w[yarn run block-dependencies], %w[bundle exec rubocop --parallel], %w[scripts/lint-conflicts.sh], %w[scripts/lint-rugged], %w[scripts/frontend/check_no_partial_karma_jest.sh], - %w[scripts/lint-changelog-filenames] + %w[scripts/lint-changelog-filenames], + %w[scripts/gemfile_lock_changed.sh] ] case node_total diff --git a/scripts/trigger-build b/scripts/trigger-build index 889dcc01043..c7b45480bf3 100755 --- a/scripts/trigger-build +++ b/scripts/trigger-build @@ -16,6 +16,14 @@ module Trigger %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) end + def self.non_empty_variable_value(variable) + variable_value = ENV[variable] + + return if variable_value.nil? || variable_value.empty? + + variable_value + end + class Base def invoke!(post_comment: false, downstream_job_name: nil) pipeline_variables = variables @@ -84,13 +92,15 @@ module Trigger end def base_variables + # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results, + # and fallback to CI_COMMIT_SHA for the `detached` pipelines. { 'GITLAB_REF_SLUG' => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_REF_SLUG'], 'TRIGGERED_USER' => ENV['TRIGGERED_USER'] || ENV['GITLAB_USER_NAME'], 'TRIGGER_SOURCE' => ENV['CI_JOB_URL'], 'TOP_UPSTREAM_SOURCE_PROJECT' => ENV['CI_PROJECT_PATH'], 'TOP_UPSTREAM_SOURCE_JOB' => ENV['CI_JOB_URL'], - 'TOP_UPSTREAM_SOURCE_SHA' => ENV['CI_COMMIT_SHA'], + 'TOP_UPSTREAM_SOURCE_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'], 'TOP_UPSTREAM_SOURCE_REF' => ENV['CI_COMMIT_REF_NAME'], 'TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID' => ENV['CI_MERGE_REQUEST_PROJECT_ID'], 'TOP_UPSTREAM_MERGE_REQUEST_IID' => ENV['CI_MERGE_REQUEST_IID'] @@ -125,8 +135,10 @@ module Trigger end def extra_variables + # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results + # and fallback to CI_COMMIT_SHA for the `detached` pipelines. { - 'GITLAB_VERSION' => ENV['CI_COMMIT_SHA'], + 'GITLAB_VERSION' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'], 'ALTERNATIVE_SOURCES' => 'true', 'ee' => Trigger.ee? ? 'true' : 'false', 'QA_BRANCH' => ENV['QA_BRANCH'] || 'master' @@ -164,11 +176,9 @@ module Trigger edition = Trigger.ee? ? 'EE' : 'CE' { - # Back-compatibility until https://gitlab.com/gitlab-org/build/CNG/merge_requests/189 is merged - "GITLAB_#{edition}_VERSION" => ENV['CI_COMMIT_REF_NAME'], - "GITLAB_VERSION" => ENV['CI_COMMIT_REF_NAME'], + "GITLAB_VERSION" => ENV['CI_COMMIT_SHA'], "GITLAB_TAG" => ENV['CI_COMMIT_TAG'], - "GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_REF_SLUG'], + "GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_SHA'], "FORCE_RAILS_IMAGE_BUILDS" => 'true', "#{edition}_PIPELINE" => 'true' } @@ -192,7 +202,7 @@ module Trigger Gitlab.create_commit_comment( ENV['CI_PROJECT_PATH'], - ENV['CI_COMMIT_SHA'], + Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'], "The [`#{ENV['CI_JOB_NAME']}`](#{ENV['CI_JOB_URL']}) job from pipeline #{ENV['CI_PIPELINE_URL']} triggered #{downstream_pipeline.web_url} downstream.") rescue Gitlab::Error::Error => error diff --git a/scripts/utils.sh b/scripts/utils.sh index 7eae9531f74..5d52ca0b40a 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -57,97 +57,3 @@ function echoinfo() { printf "\033[0;33m%s\n\033[0m" "${1}" >&2; fi } - -function get_job_id() { - local job_name="${1}" - local query_string="${2:+&${2}}" - local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" - if [ -z "${api_token}" ]; then - echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." - return - fi - - local max_page=3 - local page=1 - - while true; do - local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}" - echoinfo "GET ${url}" - - local job_id - job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq "map(select(.name == \"${job_name}\")) | map(.id) | last") - [[ "${job_id}" == "null" && "${page}" -lt "$max_page" ]] || break - - let "page++" - done - - if [[ "${job_id}" == "" ]]; then - echoerr "The '${job_name}' job ID couldn't be retrieved!" - else - echoinfo "The '${job_name}' job ID is ${job_id}" - echo "${job_id}" - fi -} - -function play_job() { - local job_name="${1}" - local job_id - job_id=$(get_job_id "${job_name}" "scope=manual"); - if [ -z "${job_id}" ]; then return; fi - - local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" - if [ -z "${api_token}" ]; then - echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." - return - fi - - local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}/play" - echoinfo "POST ${url}" - - local job_url - job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".web_url") - echoinfo "Manual job '${job_name}' started at: ${job_url}" -} - -function wait_for_job_to_be_done() { - local job_name="${1}" - local query_string="${2}" - local job_id - job_id=$(get_job_id "${job_name}" "${query_string}") - if [ -z "${job_id}" ]; then return; fi - - local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" - if [ -z "${api_token}" ]; then - echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." - return - fi - - echoinfo "Waiting for the '${job_name}' job to finish..." - - local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}" - echoinfo "GET ${url}" - - # In case the job hasn't finished yet. Keep trying until the job times out. - local interval=30 - local elapsed_seconds=0 - while true; do - local job_status - job_status=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".status" | sed -e s/\"//g) - [[ "${job_status}" == "pending" || "${job_status}" == "running" ]] || break - - printf "." - let "elapsed_seconds+=interval" - sleep ${interval} - done - - local elapsed_minutes=$((elapsed_seconds / 60)) - echoinfo "Waited '${job_name}' for ${elapsed_minutes} minutes." - - if [[ "${job_status}" == "failed" ]]; then - echoerr "The '${job_name}' failed." - elif [[ "${job_status}" == "manual" ]]; then - echoinfo "The '${job_name}' is manual." - else - echoinfo "The '${job_name}' passed." - fi -} |