summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2021-06-16 18:25:58 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2021-06-16 18:25:58 +0000
commita5f4bba440d7f9ea47046a0a561d49adf0a1e6d4 (patch)
treefb69158581673816a8cd895f9d352dcb3c678b1e /scripts
parentd16b2e8639e99961de6ddc93909f3bb5c1445ba1 (diff)
downloadgitlab-ce-a5f4bba440d7f9ea47046a0a561d49adf0a1e6d4.tar.gz
Add latest changes from gitlab-org/gitlab@14-0-stable-eev14.0.0-rc42
Diffstat (limited to 'scripts')
-rw-r--r--scripts/frontend/startup_css/clean_css.js83
-rw-r--r--scripts/frontend/startup_css/constants.js106
-rw-r--r--scripts/frontend/startup_css/get_css_path.js22
-rw-r--r--scripts/frontend/startup_css/get_startup_css.js69
-rw-r--r--scripts/frontend/startup_css/main.js60
-rwxr-xr-xscripts/frontend/startup_css/setup.sh76
-rwxr-xr-xscripts/frontend/startup_css/startup_css_changed.sh40
-rw-r--r--scripts/frontend/startup_css/utils.js8
-rw-r--r--scripts/frontend/startup_css/write_startup_scss.js28
-rwxr-xr-xscripts/gitaly-test-build9
-rwxr-xr-xscripts/lint-changelog-filenames12
-rw-r--r--scripts/prepare_build.sh3
-rw-r--r--scripts/review_apps/base-config.yaml4
-rwxr-xr-xscripts/review_apps/review-apps.sh89
-rw-r--r--scripts/rspec_helpers.sh2
-rwxr-xr-xscripts/security-harness3
-rwxr-xr-xscripts/static-analysis29
-rwxr-xr-xscripts/trigger-build41
-rw-r--r--scripts/utils.sh14
-rwxr-xr-xscripts/verify-tff-mapping4
20 files changed, 622 insertions, 80 deletions
diff --git a/scripts/frontend/startup_css/clean_css.js b/scripts/frontend/startup_css/clean_css.js
new file mode 100644
index 00000000000..67a0453e816
--- /dev/null
+++ b/scripts/frontend/startup_css/clean_css.js
@@ -0,0 +1,83 @@
+const { memoize, isString, isRegExp } = require('lodash');
+const { parse } = require('postcss');
+const { CSS_TO_REMOVE } = require('./constants');
+
+const getSelectorRemoveTesters = memoize(() =>
+ CSS_TO_REMOVE.map((x) => {
+ if (isString(x)) {
+ return (selector) => x === selector;
+ }
+ if (isRegExp(x)) {
+ return (selector) => x.test(selector);
+ }
+
+ throw new Error(`Unexpected type in CSS_TO_REMOVE content "${x}". Expected String or RegExp.`);
+ }),
+);
+
+const getRemoveTesters = memoize(() => {
+ const selectorTesters = getSelectorRemoveTesters();
+
+ // These are mostly carried over from the previous project
+ // https://gitlab.com/gitlab-org/frontend/gitlab-css-statistics/-/blob/2aa00af25dba08fc71081c77206f45efe817ea4b/lib/gl_startup_extract.js
+ return [
+ (node) => node.type === 'comment',
+ (node) =>
+ node.type === 'atrule' &&
+ (node.params === 'print' ||
+ node.params === 'prefers-reduced-motion: reduce' ||
+ node.name === 'keyframe' ||
+ node.name === 'charset'),
+ (node) => node.selector && node.selectors && !node.selectors.length,
+ (node) => node.selector && selectorTesters.some((fn) => fn(node.selector)),
+ (node) =>
+ node.type === 'decl' &&
+ (node.prop === 'transition' ||
+ node.prop.indexOf('-webkit-') > -1 ||
+ node.prop.indexOf('-ms-') > -1),
+ ];
+});
+
+const getNodesToRemove = (nodes) => {
+ const removeTesters = getRemoveTesters();
+ const remNodes = [];
+
+ nodes.forEach((node) => {
+ if (removeTesters.some((fn) => fn(node))) {
+ remNodes.push(node);
+ } else if (node.nodes?.length) {
+ remNodes.push(...getNodesToRemove(node.nodes));
+ }
+ });
+
+ return remNodes;
+};
+
+const getEmptyNodesToRemove = (nodes) =>
+ nodes
+ .filter((node) => node.nodes)
+ .reduce((acc, node) => {
+ if (node.nodes.length) {
+ acc.push(...getEmptyNodesToRemove(node.nodes));
+ } else {
+ acc.push(node);
+ }
+
+ return acc;
+ }, []);
+
+const cleanCSS = (css) => {
+ const cssRoot = parse(css);
+
+ getNodesToRemove(cssRoot.nodes).forEach((node) => {
+ node.remove();
+ });
+
+ getEmptyNodesToRemove(cssRoot.nodes).forEach((node) => {
+ node.remove();
+ });
+
+ return cssRoot.toResult().css;
+};
+
+module.exports = { cleanCSS };
diff --git a/scripts/frontend/startup_css/constants.js b/scripts/frontend/startup_css/constants.js
new file mode 100644
index 00000000000..5f6189d9e59
--- /dev/null
+++ b/scripts/frontend/startup_css/constants.js
@@ -0,0 +1,106 @@
+const path = require('path');
+const IS_EE = require('../../../config/helpers/is_ee_env');
+
+// controls --------------------------------------------------------------------
+const HTML_TO_REMOVE = [
+ 'style',
+ 'script',
+ 'link[rel="stylesheet"]',
+ '.content-wrapper',
+ '#js-peek',
+ '.modal',
+ '.feature-highlight',
+ // The user has to open up the responsive nav, so we don't need it on load
+ '.top-nav-responsive',
+ // We don't want to capture all the children of a dropdown-menu
+ '.dropdown-menu',
+];
+const CSS_TO_REMOVE = [
+ '.tooltip',
+ '.tooltip.show',
+ '.fa',
+ '.gl-accessibility:focus',
+ '.toasted-container',
+ 'body .toasted-container.bottom-left',
+ '.popover',
+ '.with-performance-bar .navbar-gitlab',
+ '.text-secondary',
+ /\.feature-highlight-popover-content/,
+ /\.commit/,
+ /\.md/,
+ /\.with-performance-bar/,
+];
+const APPLICATION_CSS_PREFIX = 'application';
+const APPLICATION_DARK_CSS_PREFIX = 'application_dark';
+const UTILITIES_CSS_PREFIX = 'application_utilities';
+const UTILITIES_DARK_CSS_PREFIX = 'application_utilities_dark';
+
+// paths -----------------------------------------------------------------------
+const ROOT = path.resolve(__dirname, '../../..');
+const ROOT_RAILS = IS_EE ? path.join(ROOT, 'ee') : ROOT;
+const FIXTURES_FOLDER_NAME = IS_EE ? 'fixtures-ee' : 'fixtures';
+const FIXTURES_ROOT = path.join(ROOT, 'tmp/tests/frontend', FIXTURES_FOLDER_NAME);
+const PATH_SIGNIN_HTML = path.join(FIXTURES_ROOT, 'startup_css/sign-in.html');
+const PATH_ASSETS = path.join(ROOT, 'tmp/startup_css_assets');
+const PATH_STARTUP_SCSS = path.join(ROOT_RAILS, 'app/assets/stylesheets/startup');
+
+// helpers ---------------------------------------------------------------------
+const createMainOutput = ({ outFile, cssKeys, type }) => ({
+ outFile,
+ htmlPaths: [
+ path.join(FIXTURES_ROOT, `startup_css/project-${type}.html`),
+ path.join(FIXTURES_ROOT, `startup_css/project-${type}-legacy-menu.html`),
+ path.join(FIXTURES_ROOT, `startup_css/project-${type}-legacy-sidebar.html`),
+ path.join(FIXTURES_ROOT, `startup_css/project-${type}-signed-out.html`),
+ ],
+ cssKeys,
+ purgeOptions: {
+ safelist: {
+ standard: [
+ 'page-with-icon-sidebar',
+ 'sidebar-collapsed-desktop',
+ // We want to include the root dropdown-menu style since it should be hidden by default
+ 'dropdown-menu',
+ ],
+ // We want to include the identicon backgrounds
+ greedy: [/^bg[0-9]$/],
+ },
+ },
+});
+
+const OUTPUTS = [
+ createMainOutput({
+ type: 'general',
+ outFile: 'startup-general',
+ cssKeys: [APPLICATION_CSS_PREFIX, UTILITIES_CSS_PREFIX],
+ }),
+ createMainOutput({
+ type: 'dark',
+ outFile: 'startup-dark',
+ cssKeys: [APPLICATION_DARK_CSS_PREFIX, UTILITIES_DARK_CSS_PREFIX],
+ }),
+ {
+ outFile: 'startup-signin',
+ htmlPaths: [PATH_SIGNIN_HTML],
+ cssKeys: [APPLICATION_CSS_PREFIX, UTILITIES_CSS_PREFIX],
+ purgeOptions: {
+ safelist: {
+ standard: ['fieldset', 'hidden'],
+ deep: [/login-page$/],
+ },
+ },
+ },
+];
+
+module.exports = {
+ HTML_TO_REMOVE,
+ CSS_TO_REMOVE,
+ APPLICATION_CSS_PREFIX,
+ APPLICATION_DARK_CSS_PREFIX,
+ UTILITIES_CSS_PREFIX,
+ UTILITIES_DARK_CSS_PREFIX,
+ ROOT,
+ PATH_ASSETS,
+ PATH_STARTUP_SCSS,
+ OUTPUTS,
+};
diff --git a/scripts/frontend/startup_css/get_css_path.js b/scripts/frontend/startup_css/get_css_path.js
new file mode 100644
index 00000000000..54078cf3149
--- /dev/null
+++ b/scripts/frontend/startup_css/get_css_path.js
@@ -0,0 +1,22 @@
+const fs = require('fs');
+const path = require('path');
+const { memoize } = require('lodash');
+const { PATH_ASSETS } = require('./constants');
+const { die } = require('./utils');
+
+const listAssetsDir = memoize(() => fs.readdirSync(PATH_ASSETS));
+
+const getCSSPath = (prefix) => {
+ const matcher = new RegExp(`^${prefix}-[^-]+\\.css$`);
+ const cssPath = listAssetsDir().find((x) => matcher.test(x));
+
+ if (!cssPath) {
+ die(
+ `Could not find the CSS asset matching "${prefix}". Have you run "scripts/frontend/startup_css/setup.sh"?`,
+ );
+ }
+
+ return path.join(PATH_ASSETS, cssPath);
+};
+
+module.exports = { getCSSPath };
diff --git a/scripts/frontend/startup_css/get_startup_css.js b/scripts/frontend/startup_css/get_startup_css.js
new file mode 100644
index 00000000000..10e8371df8c
--- /dev/null
+++ b/scripts/frontend/startup_css/get_startup_css.js
@@ -0,0 +1,69 @@
+const fs = require('fs');
+const cheerio = require('cheerio');
+const { mergeWith, isArray } = require('lodash');
+const { PurgeCSS } = require('purgecss');
+const purgeHtml = require('purgecss-from-html');
+const { cleanCSS } = require('./clean_css');
+const { HTML_TO_REMOVE } = require('./constants');
+const { die } = require('./utils');
+
+const cleanHtml = (html) => {
+ const $ = cheerio.load(html);
+
+ HTML_TO_REMOVE.forEach((selector) => {
+ $(selector).remove();
+ });
+
+ return $.html();
+};
+
+const mergePurgeCSSOptions = (...options) =>
+ mergeWith(...options, (objValue, srcValue) => {
+ if (isArray(objValue)) {
+ return objValue.concat(srcValue);
+ }
+
+ return undefined;
+ });
+
+const getStartupCSS = async ({ htmlPaths, cssPaths, purgeOptions }) => {
+ const content = htmlPaths.map((htmlPath) => {
+ if (!fs.existsSync(htmlPath)) {
+ die(`Could not find fixture "${htmlPath}". Have you run the fixtures?`);
+ }
+
+ const rawHtml = fs.readFileSync(htmlPath);
+ const html = cleanHtml(rawHtml);
+
+ return { raw: html, extension: 'html' };
+ });
+
+ const purgeCSSResult = await new PurgeCSS().purge({
+ content,
+ css: cssPaths,
+ ...mergePurgeCSSOptions(
+ {
+ fontFace: true,
+ variables: true,
+ keyframes: true,
+ blocklist: [/:hover/, /:focus/, /-webkit-/, /-moz-focusring-/, /-ms-expand/],
+ safelist: {
+ standard: ['brand-header-logo'],
+ },
+ // By default, PurgeCSS ignores special characters, but our utilities use "!"
+ defaultExtractor: (x) => x.match(/[\w-!]+/g),
+ extractors: [
+ {
+ extractor: purgeHtml,
+ extensions: ['html'],
+ },
+ ],
+ },
+ purgeOptions,
+ ),
+ });
+
+ return purgeCSSResult.map(({ css }) => cleanCSS(css)).join('\n');
+};
+
+module.exports = { getStartupCSS };
diff --git a/scripts/frontend/startup_css/main.js b/scripts/frontend/startup_css/main.js
new file mode 100644
index 00000000000..1e8dcbebae2
--- /dev/null
+++ b/scripts/frontend/startup_css/main.js
@@ -0,0 +1,60 @@
+const { memoize } = require('lodash');
+const { OUTPUTS } = require('./constants');
+const { getCSSPath } = require('./get_css_path');
+const { getStartupCSS } = require('./get_startup_css');
+const { log, die } = require('./utils');
+const { writeStartupSCSS } = require('./write_startup_scss');
+
+const memoizedCSSPath = memoize(getCSSPath);
+
+const runTask = async ({ outFile, htmlPaths, cssKeys, purgeOptions = {} }) => {
+ try {
+ log(`Generating startup CSS for HTML files: ${htmlPaths}`);
+ const generalCSS = await getStartupCSS({
+ htmlPaths,
+ cssPaths: cssKeys.map(memoizedCSSPath),
+ purgeOptions,
+ });
+
+ log(`Writing to startup CSS...`);
+ const startupCSSPath = writeStartupSCSS(outFile, generalCSS);
+ log(`Finished writing to ${startupCSSPath}`);
+
+ return {
+ success: true,
+ outFile,
+ };
+ } catch (e) {
+ log(`ERROR! Unexpected error occurred while generating startup CSS for: ${outFile}`);
+ log(e);
+
+ return {
+ success: false,
+ outFile,
+ };
+ }
+};
+
+const main = async () => {
+ const result = await Promise.all(OUTPUTS.map(runTask));
+ const fullSuccess = result.every((x) => x.success);
+
+ log('RESULTS:');
+ log('--------');
+
+ result.forEach(({ success, outFile }) => {
+ const status = success ? '✓' : 'ⅹ';
+
+ log(`${status}: ${outFile}`);
+ });
+
+ log('--------');
+
+ if (fullSuccess) {
+ log('Done!');
+ } else {
+ die('Some tasks have failed');
+ }
+};
+
+main();
diff --git a/scripts/frontend/startup_css/setup.sh b/scripts/frontend/startup_css/setup.sh
new file mode 100755
index 00000000000..795799bd9fd
--- /dev/null
+++ b/scripts/frontend/startup_css/setup.sh
@@ -0,0 +1,76 @@
+path_public_dir="public"
+path_tmp="tmp"
+path_dest="$path_tmp/startup_css_assets"
+glob_css_dest="$path_dest/application*.css"
+glob_css_src="$path_public_dir/assets/application*.css"
+should_clean=false
+
+should_force() {
+ $1=="force"
+}
+
+has_dest_already() {
+ find $glob_css_dest -quit
+}
+
+has_src_already() {
+ find $glob_css_src -quit
+}
+
+compile_assets() {
+ # We need to build the same test bundle that is built in CI
+ RAILS_ENV=test bundle exec rake rake:assets:precompile
+}
+
+clean_assets() {
+ bundle exec rake rake:assets:clobber
+}
+
+copy_assets() {
+ rm -rf $path_dest
+ mkdir $path_dest
+ cp $glob_css_src $path_dest
+}
+
+echo "-----------------------------------------------------------"
+echo "If you are run into any issues with Startup CSS generation,"
+echo "please check out the feedback issue:"
+echo ""
+echo "https://gitlab.com/gitlab-org/gitlab/-/issues/331812"
+echo "-----------------------------------------------------------"
+
+if [ ! -e $path_public_dir ]; then
+ echo "Could not find '$path_public_dir/'. This script must be run in the root directory of the gitlab project."
+ exit 1
+fi
+
+if [ ! -e $path_tmp ]; then
+ echo "Could not find '$path_tmp/'. This script must be run in the root directory of the gitlab project."
+ exit 1
+fi
+
+if [ "$1" != "force" ] && has_dest_already; then
+ echo "Already found assets for '$glob_css_dest'. Did you want to run this script with 'force' argument?"
+ exit 0
+fi
+
+# If we are in CI, don't recompile things...
+if [ -n "$CI" ]; then
+ if ! has_src_already; then
+ echo "Could not find '$glob_css_src'. Expected these artifacts to be generated by CI pipeline."
+ exit 1
+ fi
+elif has_src_already; then
+ echo "Found '$glob_css_src'. Skipping compile assets..."
+else
+ echo "Starting compile assets process..."
+ compile_assets
+ should_clean=true
+fi
+
+copy_assets
+
+if $should_clean; then
+ echo "Starting cleanup..."
+ clean_assets
+fi
diff --git a/scripts/frontend/startup_css/startup_css_changed.sh b/scripts/frontend/startup_css/startup_css_changed.sh
new file mode 100755
index 00000000000..f214e61cdfb
--- /dev/null
+++ b/scripts/frontend/startup_css/startup_css_changed.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+echo "-----------------------------------------------------------"
+echo "If you run into any issues with Startup CSS generation"
+echo "please check out the feedback issue:"
+echo ""
+echo "https://gitlab.com/gitlab-org/gitlab/-/issues/331812"
+echo "-----------------------------------------------------------"
+
+startup_glob="*stylesheets/startup*"
+
+echo "Staging changes to '${startup_glob}' so we can check for untracked files..."
+git add ${startup_glob}
+
+if [ -n "$(git diff HEAD --name-only -- ${startup_glob})" ]; then
+ diff=$(git diff HEAD -- ${startup_glob})
+ cat <<EOF
+
+Startup CSS changes detected!
+
+It looks like there have been recent changes which require
+regenerating the Startup CSS files.
+
+**What should I do now?**
+
+IMPORTANT: Please make sure to update your MR title with "[RUN AS-IF-FOSS]" and start a new MR pipeline
+
+To fix this job, consider one of the following options:
+
+ 1. Regenerating locally with "yarn run generate:startup_css".
+ 2. Copy and apply the following diff:
+
+----- start diff -----
+$diff
+
+----- end diff -------
+EOF
+
+ exit 1
+fi
diff --git a/scripts/frontend/startup_css/utils.js b/scripts/frontend/startup_css/utils.js
new file mode 100644
index 00000000000..49ad201fb6b
--- /dev/null
+++ b/scripts/frontend/startup_css/utils.js
@@ -0,0 +1,8 @@
+const die = (message) => {
+ console.log(message);
+ process.exit(1);
+};
+
+const log = (message) => console.error(`[gitlab.startup_css] ${message}`);
+
+module.exports = { die, log };
diff --git a/scripts/frontend/startup_css/write_startup_scss.js b/scripts/frontend/startup_css/write_startup_scss.js
new file mode 100644
index 00000000000..245681bada3
--- /dev/null
+++ b/scripts/frontend/startup_css/write_startup_scss.js
@@ -0,0 +1,28 @@
+const { writeFileSync } = require('fs');
+const path = require('path');
+const prettier = require('prettier');
+const { PATH_STARTUP_SCSS } = require('./constants');
+
+const buildFinalContent = (raw) => {
+ const content = `// DO NOT EDIT! This is auto-generated from "yarn run generate:startup_css"
+// Please see the feedback issue for more details and help:
+// https://gitlab.com/gitlab-org/gitlab/-/issues/331812
+@charset "UTF-8";
+${raw}
+@import 'startup/cloaking';
+@include cloak-startup-scss(none);
+`;
+
+ // We run prettier so that there is more determinism with the generated file.
+ return prettier.format(content, { parser: 'scss' });
+};
+
+const writeStartupSCSS = (name, raw) => {
+ const fullPath = path.join(PATH_STARTUP_SCSS, `${name}.scss`);
+
+ writeFileSync(fullPath, buildFinalContent(raw));
+
+ return fullPath;
+};
+
+module.exports = { writeStartupSCSS };
diff --git a/scripts/gitaly-test-build b/scripts/gitaly-test-build
index 4890e6912cd..e6afadccc7e 100755
--- a/scripts/gitaly-test-build
+++ b/scripts/gitaly-test-build
@@ -15,7 +15,14 @@ class GitalyTestBuild
def run
set_bundler_config
- abort 'gitaly build failed' unless build_gitaly
+ # If we have the binaries from the cache, we can skip building them again
+ if File.exist?(tmp_tests_gitaly_bin_dir)
+ GitalySetup::LOGGER.debug "Gitaly binary already built. Skip building...\n"
+ # We still need to install the gems in that case
+ install_gitaly_gems
+ else
+ abort 'gitaly build failed' unless build_gitaly
+ end
ensure_gitlab_shell_secret!
check_gitaly_config!
diff --git a/scripts/lint-changelog-filenames b/scripts/lint-changelog-filenames
deleted file mode 100755
index fc07b7153aa..00000000000
--- a/scripts/lint-changelog-filenames
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/sh
-
-lint_paths="changelogs/unreleased"
-[ -d "ee/" ] && lint_paths="$lint_paths ee/changelogs/unreleased"
-
-invalid_files=$(find $lint_paths -type f -not -name "*.yml" -not -name ".gitkeep")
-
-if [ -n "$invalid_files" ]; then
- echo "Changelog files must end in .yml, but these did not:"
- echo "$invalid_files" | sed -e "s/^/* /"
- exit 1
-fi
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index 5753a0af4f8..e0be80d429f 100644
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -42,6 +42,9 @@ sed -i 's|url:.*$|url: redis://redis:6379/11|g' config/redis.queues.yml
cp config/redis.shared_state.yml.example config/redis.shared_state.yml
sed -i 's|url:.*$|url: redis://redis:6379/12|g' config/redis.shared_state.yml
+cp config/redis.trace_chunks.yml.example config/redis.trace_chunks.yml
+sed -i 's|url:.*$|url: redis://redis:6379/13|g' config/redis.trace_chunks.yml
+
if [ "$SETUP_DB" != "false" ]; then
setup_db
elif getent hosts postgres; then
diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml
index 7daf3f80efc..bb4d5392b3b 100644
--- a/scripts/review_apps/base-config.yaml
+++ b/scripts/review_apps/base-config.yaml
@@ -113,10 +113,10 @@ nginx-ingress:
ssl-ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4
resources:
requests:
- cpu: 100m
+ cpu: 300m
memory: 450M
limits:
- cpu: 200m
+ cpu: 600m
memory: 675M
service:
enableHttp: false
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index 5b52797d285..6fb83e79f7f 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -40,7 +40,7 @@ function previous_deploy_failed() {
}
function delete_release() {
- local namespace="${KUBE_NAMESPACE}"
+ local namespace="${CI_ENVIRONMENT_SLUG}"
local release="${CI_ENVIRONMENT_SLUG}"
if [ -z "${release}" ]; then
@@ -48,39 +48,11 @@ function delete_release() {
return
fi
- # Check if helm release exists before attempting to delete
- # There may be situation where k8s resources exist, but helm release does not,
- # for example, following a failed helm install.
- # In such cases, we still want to continue to clean up k8s resources.
- if deploy_exists "${namespace}" "${release}"; then
- helm_delete_release "${namespace}" "${release}"
- fi
- kubectl_cleanup_release "${namespace}" "${release}"
-}
-
-function helm_delete_release() {
- local namespace="${1}"
- local release="${2}"
-
- echoinfo "Deleting Helm release '${release}'..." true
-
- helm uninstall --namespace "${namespace}" "${release}"
-}
-
-function kubectl_cleanup_release() {
- local namespace="${1}"
- local release="${2}"
-
- echoinfo "Deleting all K8s resources matching '${release}'..." true
- kubectl --namespace "${namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,clusterrole,clusterrolebinding,role,rolebinding,sa,crd 2>&1 \
- | grep "${release}" \
- | awk '{print $1}' \
- | xargs kubectl --namespace "${namespace}" delete --ignore-not-found \
- || true
+ delete_k8s_release_namespace
}
function delete_failed_release() {
- local namespace="${KUBE_NAMESPACE}"
+ local namespace="${CI_ENVIRONMENT_SLUG}"
local release="${CI_ENVIRONMENT_SLUG}"
if [ -z "${release}" ]; then
@@ -93,7 +65,7 @@ function delete_failed_release() {
else
# Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
if previous_deploy_failed "${namespace}" "${release}" ; then
- echoinfo "Review App deployment in bad state, cleaning up ${release}"
+ echoinfo "Review App deployment in bad state, cleaning up namespace ${release}"
delete_release
else
echoinfo "Review App deployment in good state"
@@ -101,8 +73,14 @@ function delete_failed_release() {
fi
}
+function delete_k8s_release_namespace() {
+ local namespace="${CI_ENVIRONMENT_SLUG}"
+
+ kubectl delete namespace "${namespace}" --wait
+}
+
function get_pod() {
- local namespace="${KUBE_NAMESPACE}"
+ local namespace="${CI_ENVIRONMENT_SLUG}"
local release="${CI_ENVIRONMENT_SLUG}"
local app_name="${1}"
local status="${2-Running}"
@@ -133,7 +111,7 @@ function get_pod() {
}
function run_task() {
- local namespace="${KUBE_NAMESPACE}"
+ local namespace="${CI_ENVIRONMENT_SLUG}"
local ruby_cmd="${1}"
local task_runner_pod=$(get_pod "task-runner")
@@ -149,13 +127,12 @@ function disable_sign_ups() {
fi
# Create the root token
- local ruby_cmd="token = User.find_by_username('root').personal_access_tokens.create(scopes: [:api], name: 'Token to disable sign-ups'); token.set_token('${REVIEW_APPS_ROOT_TOKEN}'); begin; token.save!; rescue(ActiveRecord::RecordNotUnique); end"
- retry "run_task \"${ruby_cmd}\""
+ local set_token_rb="token = User.find_by_username('root').personal_access_tokens.create(scopes: [:api], name: 'Token to disable sign-ups'); token.set_token('${REVIEW_APPS_ROOT_TOKEN}'); begin; token.save!; rescue(ActiveRecord::RecordNotUnique); end"
+ retry "run_task \"${set_token_rb}\""
# Disable sign-ups
- local signup_enabled=$(retry 'curl --silent --show-error --request PUT --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings?signup_enabled=false" | jq ".signup_enabled"')
-
- if [[ "${signup_enabled}" == "false" ]]; then
+ local disable_signup_rb="Gitlab::CurrentSettings.current_application_settings.update!(signup_enabled: false)"
+ if (retry "run_task \"${disable_signup_rb}\""); then
echoinfo "Sign-ups have been disabled successfully."
else
echoerr "Sign-ups are still enabled!"
@@ -177,13 +154,22 @@ function check_kube_domain() {
}
function ensure_namespace() {
- local namespace="${KUBE_NAMESPACE}"
+ local namespace="${1}"
echoinfo "Ensuring the ${namespace} namespace exists..." true
kubectl describe namespace "${namespace}" || kubectl create namespace "${namespace}"
}
+function label_namespace() {
+ local namespace="${1}"
+ local label="${2}"
+
+ echoinfo "Labeling the ${namespace} namespace with ${label}" true
+
+ kubectl label namespace "${namespace}" "${label}"
+}
+
function install_external_dns() {
local namespace="${KUBE_NAMESPACE}"
local release="dns-gitlab-review-app-helm3"
@@ -245,7 +231,7 @@ function install_certmanager() {
}
function create_application_secret() {
- local namespace="${KUBE_NAMESPACE}"
+ local namespace="${CI_ENVIRONMENT_SLUG}"
local release="${CI_ENVIRONMENT_SLUG}"
local initial_root_password_shared_secret
local gitlab_license_shared_secret
@@ -306,7 +292,7 @@ function parse_gitaly_image_tag() {
}
function deploy() {
- local namespace="${KUBE_NAMESPACE}"
+ local namespace="${CI_ENVIRONMENT_SLUG}"
local release="${CI_ENVIRONMENT_SLUG}"
local base_config_file_ref="${CI_DEFAULT_BRANCH}"
if [[ "$(base_config_changed)" == "true" ]]; then base_config_file_ref="${CI_COMMIT_SHA}"; fi
@@ -324,11 +310,15 @@ function deploy() {
gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell"
gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-ee"
+ ensure_namespace "${namespace}"
+ label_namespace "${namespace}" "tls=review-apps-tls" # label namespace for kubed to sync tls
+
create_application_secret
HELM_CMD=$(cat << EOF
helm upgrade \
--namespace="${namespace}" \
+ --create-namespace \
--install \
--wait \
--timeout "${HELM_INSTALL_TIMEOUT:-20m}" \
@@ -370,6 +360,7 @@ HELM_CMD=$(cat << EOF
${HELM_CMD} \
--version="${CI_PIPELINE_ID}-${CI_JOB_ID}" \
-f "${base_config_file}" \
+ -v "${HELM_LOG_VERBOSITY:-1}" \
"${release}" "gitlab-${GITLAB_HELM_CHART_REF}"
EOF
)
@@ -380,8 +371,20 @@ EOF
eval "${HELM_CMD}"
}
+function verify_deploy() {
+ echoinfo "Verifying deployment at ${CI_ENVIRONMENT_URL}"
+
+ if retry "test_url \"${CI_ENVIRONMENT_URL}\" curl_output.txt"; then
+ echoinfo "Review app is deployed to ${CI_ENVIRONMENT_URL}"
+ return 0
+ else
+ echoerr "Review app is not available at ${CI_ENVIRONMENT_URL}. See curl_output.txt artifact for detail."
+ return 1
+ fi
+}
+
function display_deployment_debug() {
- local namespace="${KUBE_NAMESPACE}"
+ local namespace="${CI_ENVIRONMENT_SLUG}"
local release="${CI_ENVIRONMENT_SLUG}"
# Get all pods for this release
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index b1a618270b0..0484cabca82 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -85,7 +85,7 @@ function rspec_db_library_code() {
local db_files="spec/lib/gitlab/database/ spec/support/helpers/database/"
if [[ -d "ee/" ]]; then
- db_files="${db_files} ee/spec/lib/gitlab/database/ ee/spec/lib/ee/gitlab/database_spec.rb"
+ db_files="${db_files} ee/spec/lib/ee/gitlab/database_spec.rb"
fi
rspec_simple_job "-- ${db_files}"
diff --git a/scripts/security-harness b/scripts/security-harness
index 0c1ade06587..ec062fc17cc 100755
--- a/scripts/security-harness
+++ b/scripts/security-harness
@@ -4,6 +4,7 @@
require 'digest'
require 'fileutils'
+require 'open3'
if ENV['NO_COLOR']
SHELL_RED = ''
@@ -18,7 +19,7 @@ else
end
LEFTHOOK_GLOBAL_CONFIG_PATH = File.expand_path("../lefthook.yml", __dir__)
-HOOK_PATH = File.expand_path("../.git/hooks/pre-push", __dir__)
+HOOK_PATH = Open3.capture3("git rev-parse --path-format=absolute --git-path hooks/pre-push")[0].strip
HOOK_DATA = <<~HOOK
#!/usr/bin/env bash
diff --git a/scripts/static-analysis b/scripts/static-analysis
index 136b2966244..7aa2fbf1594 100755
--- a/scripts/static-analysis
+++ b/scripts/static-analysis
@@ -20,26 +20,25 @@ class StaticAnalysis
# contain values that a FOSS installation won't find. To work
# around this we will only enable this task on EE installations.
TASKS_BY_DURATIONS_SECONDS_DESC = {
- %w[bin/rake lint:haml] => 338,
- (Gitlab.ee? ? %w[bin/rake gettext:updated_check] : nil) => 308,
+ %w[bin/rake lint:haml] => 488,
+ (Gitlab.ee? ? %w[bin/rake gettext:updated_check] : nil) => 410,
# Most of the time, RuboCop finishes in 30 seconds, but sometimes it can take around 1200 seconds so we set a
# duration of 300 to lower the likelihood that it will run in the same job as another long task...
%w[bundle exec rubocop --parallel] => 300,
- %w[yarn run lint:eslint:all] => 197,
- %w[yarn run lint:prettier] => 124,
- %w[bin/rake gettext:lint] => 96,
+ %w[yarn run lint:eslint:all] => 264,
+ %w[yarn run lint:prettier] => 134,
+ %w[bin/rake gettext:lint] => 81,
%w[bundle exec license_finder] => 49,
- %w[bin/rake lint:static_verification] => 22,
- %w[bin/rake gitlab:sidekiq:all_queues_yml:check] => 13,
- (Gitlab.ee? ? %w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check] : nil) => 13,
+ %w[bin/rake lint:static_verification] => 24,
+ %w[bin/rake gitlab:sidekiq:all_queues_yml:check] => 12,
+ (Gitlab.ee? ? %w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check] : nil) => 11,
%w[bin/rake config_lint] => 11,
- %w[yarn run internal:stylelint] => 9,
- %w[scripts/lint-conflicts.sh] => 0.59,
- %w[yarn run block-dependencies] => 0.35,
- %w[scripts/lint-rugged] => 0.23,
- %w[scripts/gemfile_lock_changed.sh] => 0.02,
- %w[scripts/frontend/check_no_partial_karma_jest.sh] => 0.01,
- %w[scripts/lint-changelog-filenames] => 0.01
+ %w[yarn run internal:stylelint] => 8,
+ %w[scripts/lint-conflicts.sh] => 1,
+ %w[yarn run block-dependencies] => 1,
+ %w[scripts/lint-rugged] => 1,
+ %w[scripts/gemfile_lock_changed.sh] => 1,
+ %w[scripts/frontend/check_no_partial_karma_jest.sh] => 1
}.reject { |k| k.nil? }.sort_by { |a| -a[1] }.to_h.keys.freeze
def run_tasks!
diff --git a/scripts/trigger-build b/scripts/trigger-build
index 0d8a46bdd2d..23c9ebbe294 100755
--- a/scripts/trigger-build
+++ b/scripts/trigger-build
@@ -255,7 +255,7 @@ module Trigger
end
def ref
- ENV['DOCS_BRANCH'] || 'master'
+ ENV['DOCS_BRANCH'] || 'main'
end
# `gitlab-org/gitlab-docs` pipeline trigger "Triggered from gitlab-org/gitlab 'review-docs-deploy' job"
@@ -297,10 +297,43 @@ module Trigger
end
class DatabaseTesting < Base
+ IDENTIFIABLE_NOTE_TAG = 'gitlab-org/database-team/gitlab-com-database-testing:identifiable-note'
+
def self.access_token
ENV['GITLABCOM_DATABASE_TESTING_ACCESS_TOKEN']
end
+ def invoke!(post_comment: false, downstream_job_name: nil)
+ pipeline = super
+ gitlab = gitlab_client(:upstream)
+ project_path = base_variables['TOP_UPSTREAM_SOURCE_PROJECT']
+ merge_request_id = base_variables['TOP_UPSTREAM_MERGE_REQUEST_IID']
+ comment = "<!-- #{IDENTIFIABLE_NOTE_TAG} --> \nStarted database testing [pipeline](https://ops.gitlab.net/#{downstream_project_path}/-/pipelines/#{pipeline.id}) " \
+ "(limited access). This comment will be updated once the pipeline has finished running."
+
+ # Look for a note to update
+ db_testing_notes = gitlab.merge_request_notes(project_path, merge_request_id).auto_paginate.select do |note|
+ note.body.include?(IDENTIFIABLE_NOTE_TAG)
+ end
+
+ note = db_testing_notes.max_by { |note| Time.parse(note.created_at) }
+
+ if note && note.type != 'DiscussionNote'
+ # The latest note has not led to a discussion. Update it.
+ gitlab.edit_merge_request_note(project_path, merge_request_id, note.id, comment)
+
+ puts "Updated comment:\n"
+ else
+ # This is the first note or the latest note has been discussed on the MR.
+ # Don't update, create new note instead.
+ note = gitlab.create_merge_request_note(project_path, merge_request_id, comment)
+
+ puts "Posted comment to:\n"
+ end
+
+ puts "https://gitlab.com/#{project_path}/-/merge_requests/#{merge_request_id}#note_#{note.id}"
+ end
+
private
def gitlab_client(type)
@@ -356,6 +389,8 @@ module Trigger
INTERVAL = 60 # seconds
MAX_DURATION = 3600 * 3 # 3 hours
+ attr_reader :id
+
def self.unscoped_class_name
name.split('::').last
end
@@ -384,7 +419,7 @@ module Trigger
raise "#{self.class.unscoped_class_name} did not succeed!"
end
- STDOUT.flush
+ $stdout.flush
end
raise "#{self.class.unscoped_class_name} timed out after waiting for #{duration} minutes!"
@@ -405,7 +440,7 @@ module Trigger
private
- attr_reader :project, :id, :gitlab_client, :start_time
+ attr_reader :project, :gitlab_client, :start_time
end
Job = Class.new(Pipeline)
diff --git a/scripts/utils.sh b/scripts/utils.sh
index d4436e1171d..529491c3a0d 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -13,6 +13,20 @@ function retry() {
return 1
}
+function test_url() {
+ local url="${1}"
+ local curl_output="${2}"
+ local status
+
+ status=$(curl -s -o "${curl_output}" -L -w ''%{http_code}'' "${url}")
+
+ if [[ $status == "200" ]]; then
+ return 0
+ fi
+
+ return 1
+}
+
function bundle_install_script() {
local extra_install_args="${1}"
diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping
index e18a14e17d6..4555a9854dd 100755
--- a/scripts/verify-tff-mapping
+++ b/scripts/verify-tff-mapping
@@ -84,8 +84,8 @@ tests = [
{
explanation: 'FOSS views should map to respective spec',
- source: 'app/views/admin/users/_user.html.haml',
- expected: ['spec/views/admin/users/_user.html.haml_spec.rb']
+ source: 'app/views/admin/dashboard/index.html.haml',
+ expected: ['spec/views/admin/dashboard/index.html.haml_spec.rb']
},
{