summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-11-19 08:27:35 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2020-11-19 08:27:35 +0000
commit7e9c479f7de77702622631cff2628a9c8dcbc627 (patch)
treec8f718a08e110ad7e1894510980d2155a6549197 /scripts
parente852b0ae16db4052c1c567d9efa4facc81146e88 (diff)
downloadgitlab-ce-7e9c479f7de77702622631cff2628a9c8dcbc627.tar.gz
Add latest changes from gitlab-org/gitlab@13-6-stable-eev13.6.0-rc42
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/build_assets_image10
-rwxr-xr-xscripts/generate-test-mapping19
-rwxr-xr-xscripts/lint-doc.sh21
-rwxr-xr-xscripts/lint-rugged5
-rwxr-xr-xscripts/pack-test-mapping19
-rw-r--r--scripts/review_apps/base-config.yaml4
-rwxr-xr-xscripts/review_apps/review-apps.sh2
-rw-r--r--scripts/rspec_helpers.sh41
-rwxr-xr-xscripts/unpack-test-mapping17
-rw-r--r--scripts/utils.sh4
-rwxr-xr-xscripts/verify-tff-mapping6
11 files changed, 128 insertions, 20 deletions
diff --git a/scripts/build_assets_image b/scripts/build_assets_image
index 12beddfa184..e6a5f036fe5 100755
--- a/scripts/build_assets_image
+++ b/scripts/build_assets_image
@@ -20,13 +20,21 @@ cp Dockerfile.assets assets_container.build/
COMMIT_REF_SLUG_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_SLUG}
COMMIT_SHA_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA}
-COMMIT_REF_NAME_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME}
DESTINATIONS="--destination=$COMMIT_REF_SLUG_DESTINATION --destination=$COMMIT_SHA_DESTINATION"
+# For EE branch builds, add a truncated SHA destination for later use by Omnibus
+# auto-deploy builds
+if [[ "${ASSETS_IMAGE_NAME}" == "gitlab-assets-ee" ]] && [ -n "$CI_COMMIT_BRANCH" ]
+then
+ COMMIT_SHORT_SHA_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA:0:11}
+ DESTINATIONS="$DESTINATIONS --destination=$COMMIT_SHORT_SHA_DESTINATION"
+fi
+
# Also tag the image with GitLab version, if running on a tag pipeline, so
# other projects can simply use that instead of computing the slug.
if [ -n "$CI_COMMIT_TAG" ]; then
+ COMMIT_REF_NAME_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME}
DESTINATIONS="$DESTINATIONS --destination=$COMMIT_REF_NAME_DESTINATION"
fi
diff --git a/scripts/generate-test-mapping b/scripts/generate-test-mapping
new file mode 100755
index 00000000000..eabe6a5b513
--- /dev/null
+++ b/scripts/generate-test-mapping
@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'json'
+require_relative '../tooling/lib/tooling/test_map_generator'
+
+test_mapping_json = ARGV.shift
+crystalball_yamls = ARGV
+
+unless test_mapping_json && !crystalball_yamls.empty?
+ puts "usage: #{__FILE__} <test_mapping_json> [crystalball_yamls...]"
+ exit 1
+end
+
+map_generator = Tooling::TestMapGenerator.new
+map_generator.parse(crystalball_yamls)
+mapping = map_generator.mapping
+
+File.write(test_mapping_json, JSON.pretty_generate(mapping))
+puts "Saved #{test_mapping_json}."
diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh
index 87256269de2..9ae6ce400da 100755
--- a/scripts/lint-doc.sh
+++ b/scripts/lint-doc.sh
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
+set -o pipefail
-cd "$(dirname "$0")/.."
+cd "$(dirname "$0")/.." || exit 1
echo "=> Linting documents at path $(pwd) as $(whoami)..."
echo
ERRORCODE=0
@@ -65,8 +66,11 @@ then
else
MERGE_BASE=$(git merge-base ${CI_MERGE_REQUEST_TARGET_BRANCH_SHA} ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA})
MD_DOC_PATH=$(git diff --name-only "${MERGE_BASE}..${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}" 'doc/*.md')
- echo -e "Merged results pipeline detected. Testing only the following files:\n${MD_DOC_PATH}"
- fi
+ if [ -n "${MD_DOC_PATH}" ]
+ then
+ echo -e "Merged results pipeline detected. Testing only the following files:\n${MD_DOC_PATH}"
+ fi
+fi
function run_locally_or_in_docker() {
local cmd=$1
@@ -77,7 +81,7 @@ function run_locally_or_in_docker() {
$cmd $args
elif hash docker 2>/dev/null
then
- docker run -t -v ${PWD}:/gitlab -w /gitlab --rm registry.gitlab.com/gitlab-org/gitlab-docs/lint:latest ${cmd} ${args}
+ docker run -t -v ${PWD}:/gitlab -w /gitlab --rm registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.12-vale-2.6.1-markdownlint-0.24.0 ${cmd} ${args}
else
echo
echo " ✖ ERROR: '${cmd}' not found. Install '${cmd}' or Docker to proceed." >&2
@@ -96,10 +100,15 @@ function run_locally_or_in_docker() {
echo '=> Linting markdown style...'
echo
-run_locally_or_in_docker 'markdownlint' "--config .markdownlint.json ${MD_DOC_PATH}"
+if [ -z "${MD_DOC_PATH}" ]
+then
+ echo "Merged results pipeline detected, but no markdown files found. Skipping."
+else
+ run_locally_or_in_docker 'markdownlint' "--config .markdownlint.json ${MD_DOC_PATH}"
+fi
echo '=> Linting prose...'
-run_locally_or_in_docker 'vale' "--minAlertLevel error ${MD_DOC_PATH}"
+run_locally_or_in_docker 'vale' "--minAlertLevel error --output=doc/.vale/vale.tmpl ${MD_DOC_PATH}"
if [ $ERRORCODE -ne 0 ]
then
diff --git a/scripts/lint-rugged b/scripts/lint-rugged
index dfa4df8333f..d7af5499e1c 100755
--- a/scripts/lint-rugged
+++ b/scripts/lint-rugged
@@ -18,6 +18,11 @@ ALLOWED = [
# Needed to detect Rugged enabled: https://gitlab.com/gitlab-org/gitlab/issues/35371
'lib/gitlab/config_checker/puma_rugged_checker.rb',
+ # Needed for GPG/X509 commit signature API
+ #
+ 'app/models/commit.rb',
+ 'lib/api/entities/commit_signature.rb',
+
# Needed for logging
'config/initializers/peek.rb',
'config/initializers/lograge.rb',
diff --git a/scripts/pack-test-mapping b/scripts/pack-test-mapping
new file mode 100755
index 00000000000..58ace3eca67
--- /dev/null
+++ b/scripts/pack-test-mapping
@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'json'
+require_relative '../tooling/lib/tooling/test_map_packer'
+
+unpacked_json_mapping, packed_json_mapping = ARGV.shift(2)
+unless packed_json_mapping && unpacked_json_mapping
+ puts "usage: #{__FILE__} <unpacked_json_mapping> <packed_json_mapping>"
+ exit 1
+end
+
+puts "Compressing #{unpacked_json_mapping}"
+
+mapping = JSON.parse(File.read(unpacked_json_mapping))
+packed_mapping = Tooling::TestMapPacker.new.pack(mapping)
+
+puts "Writing packed #{packed_json_mapping}"
+File.write(packed_json_mapping, JSON.generate(packed_mapping))
+puts "Saved #{packed_json_mapping}."
diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml
index 82be2d3a691..bfc35a6abde 100644
--- a/scripts/review_apps/base-config.yaml
+++ b/scripts/review_apps/base-config.yaml
@@ -52,10 +52,10 @@ gitlab:
resources:
requests:
cpu: 855m
- memory: 1071M
+ memory: 1285M
limits:
cpu: 1282m
- memory: 1606M
+ memory: 1927M
hpa:
targetAverageValue: 650m
task-runner:
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index b8cbe625e5b..59e7d183ae7 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -75,7 +75,7 @@ function kubectl_cleanup_release() {
kubectl --namespace "${namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,clusterrole,clusterrolebinding,role,rolebinding,sa,crd 2>&1 \
| grep "${release}" \
| awk '{print $1}' \
- | xargs kubectl --namespace "${namespace}" delete \
+ | xargs kubectl --namespace "${namespace}" delete --ignore-not-found \
|| true
}
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index 3812a8b8ef7..5f003d032b7 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -48,12 +48,49 @@ function update_tests_metadata() {
fi
}
+function retrieve_tests_mapping() {
+ mkdir -p crystalball/
+
+ if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
+ (wget -O "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+ fi
+
+ scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}"
+}
+
+function update_tests_mapping() {
+ if ! crystalball_rspec_data_exists; then
+ echo "No crystalball rspec data found."
+ return 0
+ fi
+
+ scripts/generate-test-mapping "${RSPEC_TESTS_MAPPING_PATH}" crystalball/rspec*.yml
+
+ scripts/pack-test-mapping "${RSPEC_TESTS_MAPPING_PATH}" "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+
+ gzip "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+
+ if [[ -n "${TESTS_METADATA_S3_BUCKET}" ]]; then
+ if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
+ scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz"
+ else
+ echo "Not uploading report to S3 as the pipeline is not a scheduled one."
+ fi
+ fi
+
+ rm -f crystalball/rspec*.yml
+}
+
+function crystalball_rspec_data_exists() {
+ compgen -G "crystalball/rspec*.yml" > /dev/null;
+}
+
function rspec_simple_job() {
local rspec_opts="${1}"
export NO_KNAPSACK="1"
- bin/rspec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}
+ bin/rspec -Ispec -rspec_helper --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}
}
function rspec_paralellized_job() {
@@ -106,7 +143,7 @@ function rspec_paralellized_job() {
export MEMORY_TEST_PATH="tmp/memory_test/${report_name}_memory.csv"
- knapsack rspec "-Ispec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}"
+ knapsack rspec "-Ispec -rspec_helper --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}"
date
}
diff --git a/scripts/unpack-test-mapping b/scripts/unpack-test-mapping
new file mode 100755
index 00000000000..c0f706c3f9f
--- /dev/null
+++ b/scripts/unpack-test-mapping
@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require 'json'
+require_relative '../tooling/lib/tooling/test_map_packer'
+
+packed_json_mapping, unpacked_json_mapping = ARGV.shift(2)
+unless packed_json_mapping && unpacked_json_mapping
+ puts "usage: #{__FILE__} <packed_json_mapping> <unpacked_json_mapping>"
+ exit 1
+end
+
+packed_mapping = JSON.parse(File.read(packed_json_mapping))
+mapping = Tooling::TestMapPacker.new.unpack(packed_mapping)
+
+puts "Writing unpacked #{unpacked_json_mapping}"
+File.write(unpacked_json_mapping, JSON.generate(mapping))
+puts "Saved #{unpacked_json_mapping}."
diff --git a/scripts/utils.sh b/scripts/utils.sh
index 9d188fc7b77..3829bcdf24e 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -31,8 +31,8 @@ function install_api_client_dependencies_with_apt() {
}
function install_gitlab_gem() {
- gem install httparty --no-document --version 0.17.3
- gem install gitlab --no-document --version 4.13.0
+ gem install httparty --no-document --version 0.18.1
+ gem install gitlab --no-document --version 4.14.1
}
function install_tff_gem() {
diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping
index 9541d2468b1..1f73753be82 100755
--- a/scripts/verify-tff-mapping
+++ b/scripts/verify-tff-mapping
@@ -40,12 +40,6 @@ tests = [
},
{
- explanation: 'EE lib should map to respective spec',
- source: 'ee/lib/flipper_session.rb',
- expected: ['ee/spec/lib/flipper_session_spec.rb']
- },
-
- {
explanation: 'Tooling should map to respective spec',
source: 'tooling/lib/tooling/test_file_finder.rb',
expected: ['spec/tooling/lib/tooling/test_file_finder_spec.rb']