summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorRobert Speicher <rspeicher@gmail.com>2021-01-20 13:34:23 -0600
committerRobert Speicher <rspeicher@gmail.com>2021-01-20 13:34:23 -0600
commit6438df3a1e0fb944485cebf07976160184697d72 (patch)
tree00b09bfd170e77ae9391b1a2f5a93ef6839f2597 /scripts
parent42bcd54d971da7ef2854b896a7b34f4ef8601067 (diff)
downloadgitlab-ce-6438df3a1e0fb944485cebf07976160184697d72.tar.gz
Add latest changes from gitlab-org/gitlab@13-8-stable-eev13.8.0-rc42
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/api/cancel_pipeline.rb (renamed from scripts/api/cancel_pipeline)0
-rwxr-xr-xscripts/api/download_job_artifact.rb (renamed from scripts/api/download_job_artifact)0
-rwxr-xr-xscripts/api/get_job_id.rb (renamed from scripts/api/get_job_id)0
-rwxr-xr-xscripts/api/play_job.rb (renamed from scripts/api/play_job)9
-rw-r--r--scripts/frontend/block_dependencies.js4
-rwxr-xr-xscripts/frontend/check_page_bundle_mixins_css_for_sideeffects.js2
-rw-r--r--scripts/frontend/extract_gettext_all.js2
-rwxr-xr-xscripts/frontend/file_test_coverage.js16
-rw-r--r--scripts/frontend/frontend_script_utils.js9
-rw-r--r--scripts/frontend/merge_coverage_frontend.js8
-rw-r--r--scripts/frontend/prettier.js16
-rw-r--r--scripts/frontend/stylelint/stylelint-duplicate-selectors.js4
-rw-r--r--scripts/frontend/stylelint/stylelint-utility-classes.js4
-rw-r--r--scripts/frontend/stylelint/stylelint-utility-map.js4
-rw-r--r--scripts/frontend/stylelint/stylelint-utils.js4
-rwxr-xr-xscripts/frontend/webpack_dev_server.js6
-rwxr-xr-xscripts/gitaly-test-build2
-rwxr-xr-xscripts/gitaly-test-spawn1
-rw-r--r--scripts/gitaly_test.rb10
-rwxr-xr-xscripts/perf/gc/collect_gc_stats.rb97
-rwxr-xr-xscripts/perf/gc/print_gc_stats.rb69
-rwxr-xr-xscripts/review_apps/automated_cleanup.rb10
-rw-r--r--scripts/rspec_helpers.sh10
-rw-r--r--scripts/utils.sh4
-rwxr-xr-xscripts/validate_migration_schema119
-rwxr-xr-xscripts/verify-tff-mapping4
26 files changed, 357 insertions, 57 deletions
diff --git a/scripts/api/cancel_pipeline b/scripts/api/cancel_pipeline.rb
index 0965877a69a..0965877a69a 100755
--- a/scripts/api/cancel_pipeline
+++ b/scripts/api/cancel_pipeline.rb
diff --git a/scripts/api/download_job_artifact b/scripts/api/download_job_artifact.rb
index 8e2207c6fa7..8e2207c6fa7 100755
--- a/scripts/api/download_job_artifact
+++ b/scripts/api/download_job_artifact.rb
diff --git a/scripts/api/get_job_id b/scripts/api/get_job_id.rb
index c7fe859db91..c7fe859db91 100755
--- a/scripts/api/get_job_id
+++ b/scripts/api/get_job_id.rb
diff --git a/scripts/api/play_job b/scripts/api/play_job.rb
index 199f7e65633..408dfdf1ef0 100755
--- a/scripts/api/play_job
+++ b/scripts/api/play_job.rb
@@ -14,7 +14,6 @@ class PlayJob
}.freeze
def initialize(options)
- @project = options.delete(:project)
@options = options
Gitlab.configure do |config|
@@ -24,14 +23,18 @@ class PlayJob
end
def execute
- job = JobFinder.new(project, options.slice(:api_token, :pipeline_id, :job_name).merge(scope: 'manual')).execute
+ job = JobFinder.new(options.slice(:project, :api_token, :pipeline_id, :job_name).merge(scope: 'manual')).execute
Gitlab.job_play(project, job.id)
end
private
- attr_reader :project, :options
+ attr_reader :options
+
+ def project
+ options[:project]
+ end
end
if $0 == __FILE__
diff --git a/scripts/frontend/block_dependencies.js b/scripts/frontend/block_dependencies.js
index c9257c9f72b..a1ff8d5ee36 100644
--- a/scripts/frontend/block_dependencies.js
+++ b/scripts/frontend/block_dependencies.js
@@ -5,13 +5,13 @@ const dependencies = packageJson.dependencies;
const devDependencies = packageJson.devDependencies;
const blockedDependenciesNames = Object.keys(blockedDependencies);
const blockedDependenciesFound = blockedDependenciesNames.filter(
- blockedDependency => dependencies[blockedDependency] || devDependencies[blockedDependency],
+ (blockedDependency) => dependencies[blockedDependency] || devDependencies[blockedDependency],
);
if (blockedDependenciesFound.length) {
console.log('The following package.json dependencies are not allowed:');
- blockedDependenciesFound.forEach(blockedDependency => {
+ blockedDependenciesFound.forEach((blockedDependency) => {
const infoLink = blockedDependencies[blockedDependency];
console.log(`- ${blockedDependency}: See ${infoLink} for more information.`);
diff --git a/scripts/frontend/check_page_bundle_mixins_css_for_sideeffects.js b/scripts/frontend/check_page_bundle_mixins_css_for_sideeffects.js
index a2bb9f56d84..34e939e3ceb 100755
--- a/scripts/frontend/check_page_bundle_mixins_css_for_sideeffects.js
+++ b/scripts/frontend/check_page_bundle_mixins_css_for_sideeffects.js
@@ -27,7 +27,7 @@ const file = fs.readFileSync(paths[0], 'utf-8');
const parsed = pjs.parse(file);
-if (parsed.nodes.every(node => ['comment', 'atrule'].includes(node.type))) {
+if (parsed.nodes.every((node) => ['comment', 'atrule'].includes(node.type))) {
console.log('The file does not introduce any side effects, we are all good.');
process.exit(0);
}
diff --git a/scripts/frontend/extract_gettext_all.js b/scripts/frontend/extract_gettext_all.js
index 725522a3540..c34c9a0233d 100644
--- a/scripts/frontend/extract_gettext_all.js
+++ b/scripts/frontend/extract_gettext_all.js
@@ -45,7 +45,7 @@ function printJson() {
text += `\u0000${message.textPlural}`;
}
- message.references.forEach(reference => {
+ message.references.forEach((reference) => {
const filename = reference.replace(/:\d+$/, '');
if (!Array.isArray(result[filename])) {
diff --git a/scripts/frontend/file_test_coverage.js b/scripts/frontend/file_test_coverage.js
index 7d1eb45d4bc..ec6ec4a1e9d 100755
--- a/scripts/frontend/file_test_coverage.js
+++ b/scripts/frontend/file_test_coverage.js
@@ -17,11 +17,11 @@ const sourceDirectories = ['app/assets/javascripts'];
const testDirectories = ['spec/javascripts', 'spec/frontend'];
if (fs.existsSync('ee')) {
- sourceDirectories.forEach(dir => {
+ sourceDirectories.forEach((dir) => {
sourceDirectories.push(`ee/${dir}`);
});
- testDirectories.forEach(dir => {
+ testDirectories.forEach((dir) => {
testDirectories.push(`ee/${dir}`);
});
}
@@ -29,10 +29,10 @@ if (fs.existsSync('ee')) {
let numSourceFiles = 0;
let numTestFiles = 0;
-const isVerbose = process.argv.some(arg => arg === '-v');
+const isVerbose = process.argv.some((arg) => arg === '-v');
-const countSourceFiles = path =>
- forEachFileIn(path, fileName => {
+const countSourceFiles = (path) =>
+ forEachFileIn(path, (fileName) => {
if (fileName.endsWith('.vue') || fileName.endsWith('.js')) {
if (isVerbose) {
console.log(`source file: ${fileName}`);
@@ -42,8 +42,8 @@ const countSourceFiles = path =>
}
});
-const countTestFiles = path =>
- forEachFileIn(path, fileName => {
+const countTestFiles = (path) =>
+ forEachFileIn(path, (fileName) => {
if (fileName.endsWith('_spec.js')) {
if (isVerbose) {
console.log(`test file: ${fileName}`);
@@ -63,7 +63,7 @@ function forEachFileIn(dirPath, callback) {
return;
}
- files.forEach(fileName => {
+ files.forEach((fileName) => {
const absolutePath = path.join(dirPath, fileName);
const stats = fs.statSync(absolutePath);
if (stats.isFile()) {
diff --git a/scripts/frontend/frontend_script_utils.js b/scripts/frontend/frontend_script_utils.js
index e3d357b4a40..43016dce6a4 100644
--- a/scripts/frontend/frontend_script_utils.js
+++ b/scripts/frontend/frontend_script_utils.js
@@ -9,15 +9,10 @@ const exec = (command, args) => {
return execFileSync(command, args, options);
};
-const execGitCmd = args =>
- exec('git', args)
- .trim()
- .toString()
- .split('\n')
- .filter(Boolean);
+const execGitCmd = (args) => exec('git', args).trim().toString().split('\n').filter(Boolean);
module.exports = {
- getStagedFiles: fileExtensionFilter => {
+ getStagedFiles: (fileExtensionFilter) => {
const gitOptions = ['diff', '--name-only', '--cached', '--diff-filter=ACMRTUB'];
if (fileExtensionFilter) gitOptions.push(...fileExtensionFilter);
return execGitCmd(gitOptions);
diff --git a/scripts/frontend/merge_coverage_frontend.js b/scripts/frontend/merge_coverage_frontend.js
index 99034176b29..0c45a38b9b5 100644
--- a/scripts/frontend/merge_coverage_frontend.js
+++ b/scripts/frontend/merge_coverage_frontend.js
@@ -11,7 +11,7 @@ const reportFiles = sync(`${coverageDir}/*/coverage-final.json`);
// Normalize coverage report generated by jest that has additional "data" key
// https://github.com/facebook/jest/issues/2418#issuecomment-423806659
-const normalizeReport = report => {
+const normalizeReport = (report) => {
const normalizedReport = Object.assign({}, report);
Object.entries(normalizedReport).forEach(([k, v]) => {
if (v.data) normalizedReport[k] = v.data;
@@ -20,12 +20,12 @@ const normalizeReport = report => {
};
reportFiles
- .map(reportFile => require(reportFile))
+ .map((reportFile) => require(reportFile))
.map(normalizeReport)
- .forEach(report => coverageMap.merge(report));
+ .forEach((report) => coverageMap.merge(report));
const context = createContext({ coverageMap: coverageMap, dir: 'coverage-frontend' });
-['json', 'lcov', 'text-summary', 'clover', 'cobertura'].forEach(reporter => {
+['json', 'lcov', 'text-summary', 'clover', 'cobertura'].forEach((reporter) => {
create(reporter, {}).execute(context);
});
diff --git a/scripts/frontend/prettier.js b/scripts/frontend/prettier.js
index 7772f80c233..f721e46f36b 100644
--- a/scripts/frontend/prettier.js
+++ b/scripts/frontend/prettier.js
@@ -7,11 +7,11 @@ const matchExtensions = ['js', 'vue', 'graphql'];
// This will improve glob performance by excluding certain directories.
// The .prettierignore file will also be respected, but after the glob has executed.
-const globIgnore = ['**/node_modules/**', 'vendor/**', 'public/**'];
+const globIgnore = ['**/node_modules/**', 'vendor/**', 'public/**', 'fixtures/**'];
const readFileAsync = (file, options) =>
new Promise((resolve, reject) => {
- fs.readFile(file, options, function(err, data) {
+ fs.readFile(file, options, function (err, data) {
if (err) reject(err);
else resolve(data);
});
@@ -19,7 +19,7 @@ const readFileAsync = (file, options) =>
const writeFileAsync = (file, data, options) =>
new Promise((resolve, reject) => {
- fs.writeFile(file, data, options, function(err) {
+ fs.writeFile(file, data, options, function (err) {
if (err) reject(err);
else resolve();
});
@@ -35,7 +35,7 @@ console.log(
`Loading all ${allFiles ? '' : 'staged '}files ${globDir ? `within ${globDir} ` : ''}...`,
);
-const globPatterns = matchExtensions.map(ext => `${globDir}**/*.${ext}`);
+const globPatterns = matchExtensions.map((ext) => `${globDir}**/*.${ext}`);
const matchedFiles = allFiles
? glob.sync(`{${globPatterns.join(',')}}`, { ignore: globIgnore })
: getStagedFiles(globPatterns);
@@ -62,7 +62,7 @@ Please format each file listed below or run "${fixCommand}"
`;
const checkFileWithOptions = (filePath, options) =>
- readFileAsync(filePath, 'utf8').then(input => {
+ readFileAsync(filePath, 'utf8').then((input) => {
if (shouldSave) {
const output = prettier.format(input, options);
if (input === output) {
@@ -92,7 +92,7 @@ const checkFileWithOptions = (filePath, options) =>
}
});
-const checkFileWithPrettierConfig = filePath =>
+const checkFileWithPrettierConfig = (filePath) =>
prettier
.getFileInfo(filePath, { ignorePath: '.prettierignore' })
.then(({ ignored, inferredParser }) => {
@@ -100,7 +100,7 @@ const checkFileWithPrettierConfig = filePath =>
ignoredCount += 1;
return;
}
- return prettier.resolveConfig(filePath).then(fileOptions => {
+ return prettier.resolveConfig(filePath).then((fileOptions) => {
const options = { ...fileOptions, parser: inferredParser };
return checkFileWithOptions(filePath, options);
});
@@ -115,7 +115,7 @@ Promise.all(matchedFiles.map(checkFileWithPrettierConfig))
if (didWarn) process.exit(1);
})
- .catch(e => {
+ .catch((e) => {
console.log(`\nAn error occurred while processing files with prettier: ${e.message}\n`);
process.exit(1);
});
diff --git a/scripts/frontend/stylelint/stylelint-duplicate-selectors.js b/scripts/frontend/stylelint/stylelint-duplicate-selectors.js
index 4b46ee21a7a..89242158157 100644
--- a/scripts/frontend/stylelint/stylelint-duplicate-selectors.js
+++ b/scripts/frontend/stylelint/stylelint-duplicate-selectors.js
@@ -8,12 +8,12 @@ const messages = stylelint.utils.ruleMessages(ruleName, {
},
});
-module.exports = stylelint.createPlugin(ruleName, function(enabled) {
+module.exports = stylelint.createPlugin(ruleName, function (enabled) {
if (!enabled) {
return;
}
- return function(root, result) {
+ return function (root, result) {
const selectorGroups = {};
utils.createPropertiesHashmap(root, result, ruleName, messages, selectorGroups, true);
};
diff --git a/scripts/frontend/stylelint/stylelint-utility-classes.js b/scripts/frontend/stylelint/stylelint-utility-classes.js
index 8a1cfdbf302..1b266fc31c9 100644
--- a/scripts/frontend/stylelint/stylelint-utility-classes.js
+++ b/scripts/frontend/stylelint/stylelint-utility-classes.js
@@ -10,12 +10,12 @@ const messages = stylelint.utils.ruleMessages(ruleName, {
},
});
-module.exports = stylelint.createPlugin(ruleName, function(enabled) {
+module.exports = stylelint.createPlugin(ruleName, function (enabled) {
if (!enabled) {
return;
}
- return function(root, result) {
+ return function (root, result) {
utils.createPropertiesHashmap(root, result, ruleName, messages, utilityClasses, false);
};
});
diff --git a/scripts/frontend/stylelint/stylelint-utility-map.js b/scripts/frontend/stylelint/stylelint-utility-map.js
index 941198e82a4..bf8ee362740 100644
--- a/scripts/frontend/stylelint/stylelint-utility-map.js
+++ b/scripts/frontend/stylelint/stylelint-utility-map.js
@@ -32,7 +32,7 @@ sass.render(
// This suppresses a postcss warning
from: undefined,
})
- .then(result => {
+ .then((result) => {
const selectorGroups = {};
utils.createPropertiesHashmap(result.root, result, null, null, selectorGroups, true);
@@ -42,7 +42,7 @@ sass.render(
prettierOptions,
);
- fs.writeFile(hashMapPath, prettyHashmap, function(err) {
+ fs.writeFile(hashMapPath, prettyHashmap, function (err) {
if (err) {
return console.log(err);
}
diff --git a/scripts/frontend/stylelint/stylelint-utils.js b/scripts/frontend/stylelint/stylelint-utils.js
index 2d931c1c4c2..e7452b0cdb2 100644
--- a/scripts/frontend/stylelint/stylelint-utils.js
+++ b/scripts/frontend/stylelint/stylelint-utils.js
@@ -9,7 +9,7 @@ module.exports.createPropertiesHashmap = (
selectorGroups,
addSelectors,
) => {
- ruleRoot.walkRules(rule => {
+ ruleRoot.walkRules((rule) => {
const selector = rule.selector.replace(/(?:\r\n|\r|\n)/g, ' ');
if (
@@ -25,7 +25,7 @@ module.exports.createPropertiesHashmap = (
)
) {
let cssArray = [];
- rule.nodes.forEach(function(property) {
+ rule.nodes.forEach(function (property) {
const { prop, value } = property;
if (property && value) {
const propval = `${prop}${value}${property.important ? '!important' : ''}`;
diff --git a/scripts/frontend/webpack_dev_server.js b/scripts/frontend/webpack_dev_server.js
index 8026a8d47e2..fbb80c9617d 100755
--- a/scripts/frontend/webpack_dev_server.js
+++ b/scripts/frontend/webpack_dev_server.js
@@ -51,7 +51,7 @@ else {
// print useful messages for nodemon events
nodemon
- .on('start', function() {
+ .on('start', function () {
console.log(`Starting webpack webserver on http://${DEV_SERVER_HOST}:${DEV_SERVER_PORT}`);
if (STATIC_MODE) {
console.log('You are starting webpack in compile-once mode');
@@ -59,10 +59,10 @@ nodemon
console.log('If you change them often, you might want to unset DEV_SERVER_STATIC');
}
})
- .on('quit', function() {
+ .on('quit', function () {
console.log('Shutting down webpack process');
process.exit();
})
- .on('restart', function(files) {
+ .on('restart', function (files) {
console.log('Restarting webpack process due to: ', files);
});
diff --git a/scripts/gitaly-test-build b/scripts/gitaly-test-build
index 00927646046..62d3dbda911 100755
--- a/scripts/gitaly-test-build
+++ b/scripts/gitaly-test-build
@@ -12,7 +12,7 @@ class GitalyTestBuild
include GitalyTest
def run
- abort 'gitaly build failed' unless system(env, 'make', chdir: tmp_tests_gitaly_dir)
+ abort 'gitaly build failed' unless build_gitaly
ensure_gitlab_shell_secret!
check_gitaly_config!
diff --git a/scripts/gitaly-test-spawn b/scripts/gitaly-test-spawn
index c2ff9cd08aa..caa41a9a0c3 100755
--- a/scripts/gitaly-test-spawn
+++ b/scripts/gitaly-test-spawn
@@ -8,6 +8,7 @@ class GitalyTestSpawn
include GitalyTest
def run
+ install_gitaly_gems if ENV['CI']
check_gitaly_config!
# # Uncomment line below to see all gitaly logs merged into CI trace
diff --git a/scripts/gitaly_test.rb b/scripts/gitaly_test.rb
index 559ad8f4345..c7b3f72d590 100644
--- a/scripts/gitaly_test.rb
+++ b/scripts/gitaly_test.rb
@@ -41,7 +41,7 @@ module GitalyTest
'HOME' => File.expand_path('tmp/tests'),
'GEM_PATH' => Gem.path.join(':'),
'BUNDLE_APP_CONFIG' => File.join(File.dirname(gemfile), '.bundle/config'),
- 'BUNDLE_FLAGS' => "--jobs=4 --retry=3 --quiet",
+ 'BUNDLE_FLAGS' => "--jobs=4 --retry=3",
'BUNDLE_INSTALL_FLAGS' => nil,
'BUNDLE_GEMFILE' => gemfile,
'RUBYOPT' => nil,
@@ -78,6 +78,14 @@ module GitalyTest
end
end
+ def install_gitaly_gems
+ system(env, "make #{tmp_tests_gitaly_dir}/.ruby-bundle", chdir: tmp_tests_gitaly_dir) # rubocop:disable GitlabSecurity/SystemCommandInjection
+ end
+
+ def build_gitaly
+ system(env, 'make', chdir: tmp_tests_gitaly_dir) # rubocop:disable GitlabSecurity/SystemCommandInjection
+ end
+
def start_gitaly
start(:gitaly)
end
diff --git a/scripts/perf/gc/collect_gc_stats.rb b/scripts/perf/gc/collect_gc_stats.rb
new file mode 100755
index 00000000000..050ad66a926
--- /dev/null
+++ b/scripts/perf/gc/collect_gc_stats.rb
@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+####
+# Loads GitLab application classes with a variety of GC settings and prints
+# GC stats and timing data to standard out as CSV.
+#
+# The degree of parallelism can be increased by setting the PAR environment
+# variable (default: 2).
+
+require 'benchmark'
+
+SETTINGS = {
+ 'DEFAULTS' => [''],
+ # Default: 10_000
+ 'RUBY_GC_HEAP_INIT_SLOTS' => %w[100000 1000000 5000000],
+ # Default: 1.8
+ 'RUBY_GC_HEAP_GROWTH_FACTOR' => %w[1.2 1 0.8],
+ # Default: 0 (disabled)
+ 'RUBY_GC_HEAP_GROWTH_MAX_SLOTS' => %w[10000 100000 1000000],
+ # Default: 2.0
+ 'RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR' => %w[2.5 1.5 1],
+ # Default: 4096 (= 2^12)
+ 'RUBY_GC_HEAP_FREE_SLOTS' => %w[16384 2048 0],
+ # Default: 0.20 (20%)
+ 'RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO' => %w[0.1 0.01 0.001],
+ # Default: 0.40 (40%)
+ 'RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO' => %w[0.2 0.01 0.001],
+ # Default: 0.65 (65%)
+ 'RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO' => %w[0.2 0.02 0.002],
+ # Default: 16MB
+ 'RUBY_GC_MALLOC_LIMIT' => %w[8388608 4194304 1048576],
+ # Default: 32MB
+ 'RUBY_GC_MALLOC_LIMIT_MAX' => %w[16777216 8388608 1048576],
+ # Default: 1.4
+ 'RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR' => %w[1.6 1 0.8],
+ # Default: 16MB
+ 'RUBY_GC_OLDMALLOC_LIMIT' => %w[8388608 4194304 1048576],
+ # Default: 128MB
+ 'RUBY_GC_OLDMALLOC_LIMIT_MAX' => %w[33554432 16777216 1048576],
+ # Default: 1.2
+ 'RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR' => %w[1.4 1 0.8]
+}.freeze
+
+USED_GCSTAT_KEYS = [
+ :minor_gc_count,
+ :major_gc_count,
+ :heap_live_slots,
+ :heap_free_slots,
+ :total_allocated_pages,
+ :total_freed_pages,
+ :malloc_increase_bytes,
+ :malloc_increase_bytes_limit,
+ :oldmalloc_increase_bytes,
+ :oldmalloc_increase_bytes_limit
+].freeze
+
+CSV_USED_GCSTAT_KEYS = USED_GCSTAT_KEYS.join(',')
+CSV_HEADER = "setting,value,#{CSV_USED_GCSTAT_KEYS},RSS,gc_time_s,cpu_utime_s,cpu_stime_s,real_time_s\n"
+
+SCRIPT_PATH = __dir__
+RAILS_ROOT = "#{SCRIPT_PATH}/../../../"
+
+def collect_stats(setting, value)
+ warn "Testing #{setting} = #{value} ..."
+ env = {
+ setting => value,
+ 'RAILS_ROOT' => RAILS_ROOT,
+ 'SETTING_CSV' => "#{setting},#{value}",
+ 'GC_STAT_KEYS' => CSV_USED_GCSTAT_KEYS
+ }
+ system(env, 'ruby', "#{SCRIPT_PATH}/print_gc_stats.rb")
+end
+
+par = ENV['PAR']&.to_i || 2
+batch_size = (SETTINGS.size.to_f / par).ceil
+batches = SETTINGS.each_slice(batch_size)
+
+warn "Requested parallelism: #{par} (batches: #{batches.size}, batch size: #{batch_size})"
+
+puts CSV_HEADER
+
+elapsed = Benchmark.realtime do
+ threads = batches.each_with_index.map do |settings_batch, n|
+ Thread.new do
+ settings_batch.each do |setting, values|
+ values.each do |v|
+ collect_stats(setting, v)
+ end
+ end
+ end
+ end
+
+ threads.each(&:join)
+end
+
+warn "All done in #{elapsed} sec"
diff --git a/scripts/perf/gc/print_gc_stats.rb b/scripts/perf/gc/print_gc_stats.rb
new file mode 100755
index 00000000000..4aeb2f1ef07
--- /dev/null
+++ b/scripts/perf/gc/print_gc_stats.rb
@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# Promotes survivors from eden to old gen and runs a compaction.
+#
+# aka "Nakayoshi GC"
+#
+# https://github.com/puma/puma/blob/de632261ac45d7dd85230c83f6af6dd720f1cbd9/lib/puma/util.rb#L26-L35
+def nakayoshi_gc
+ 4.times { GC.start(full_mark: false) }
+ GC.compact
+end
+
+# GC::Profiler is used elsewhere in the code base, so we provide a way for it
+# to be used exclusively by this script, or otherwise results will be tainted.
+module GC::Profiler
+ class << self
+ attr_accessor :use_exclusive
+
+ %i[enable disable clear].each do |method|
+ alias_method "#{method}_orig", "#{method}"
+
+ define_method(method) do
+ if use_exclusive
+ warn "GC::Profiler: ignoring call to #{method}"
+ return
+ end
+
+ send("#{method}_orig") # rubocop: disable GitlabSecurity/PublicSend
+ end
+ end
+ end
+end
+
+GC::Profiler.enable
+GC::Profiler.use_exclusive = true
+
+require 'benchmark'
+
+RAILS_ROOT = ENV['RAILS_ROOT']
+
+tms = Benchmark.measure do
+ require RAILS_ROOT + 'config/boot'
+ require RAILS_ROOT + 'config/environment'
+end
+
+GC::Profiler.use_exclusive = false
+
+nakayoshi_gc
+
+gc_stats = GC.stat
+warn gc_stats.inspect
+
+gc_total_time = GC::Profiler.total_time
+
+GC::Profiler.report($stderr)
+GC::Profiler.disable
+
+gc_stat_keys = ENV['GC_STAT_KEYS'].to_s.split(',').map(&:to_sym)
+
+values = []
+values << ENV['SETTING_CSV']
+values += gc_stat_keys.map { |k| gc_stats[k] }
+values << ::Gitlab::Metrics::System.memory_usage_rss
+values << gc_total_time
+values << tms.utime + tms.cutime
+values << tms.stime + tms.cstime
+values << tms.real
+
+puts values.join(',')
diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb
index e40c6cd7276..bef5b7ad5ee 100755
--- a/scripts/review_apps/automated_cleanup.rb
+++ b/scripts/review_apps/automated_cleanup.rb
@@ -115,6 +115,10 @@ class AutomatedCleanup
delete_helm_releases(releases_to_delete)
end
+ def perform_stale_pvc_cleanup!(days:)
+ kubernetes.cleanup_by_created_at(resource_type: 'pvc', created_before: threshold_time(days: days), wait: false)
+ end
+
private
def fetch_environment(environment)
@@ -155,7 +159,7 @@ class AutomatedCleanup
releases_names = releases.map(&:name)
helm.delete(release_name: releases_names)
- kubernetes.cleanup(release_name: releases_names, wait: false)
+ kubernetes.cleanup_by_release(release_name: releases_names, wait: false)
rescue Tooling::Helm3Client::CommandFailedError => ex
raise ex unless ignore_exception?(ex.message, IGNORED_HELM_ERRORS)
@@ -198,4 +202,8 @@ timed('Helm releases cleanup') do
automated_cleanup.perform_helm_releases_cleanup!(days: 7)
end
+timed('Stale PVC cleanup') do
+ automated_cleanup.perform_stale_pvc_cleanup!(days: 30)
+end
+
exit(0)
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index 5b724c9251b..85d7dcec476 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -7,14 +7,14 @@ function retrieve_tests_metadata() {
local test_metadata_job_id
# Ruby
- test_metadata_job_id=$(scripts/api/get_job_id --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata")
+ test_metadata_job_id=$(scripts/api/get_job_id.rb --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata")
if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
- scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+ scripts/api/download_job_artifact.rb --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
fi
if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
- scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+ scripts/api/download_job_artifact.rb --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
fi
}
@@ -42,10 +42,10 @@ function retrieve_tests_mapping() {
local project_path="gitlab-org/gitlab"
local test_metadata_with_mapping_job_id
- test_metadata_with_mapping_job_id=$(scripts/api/get_job_id --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz")
+ test_metadata_with_mapping_job_id=$(scripts/api/get_job_id.rb --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz")
if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
- (scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_with_mapping_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+ (scripts/api/download_job_artifact.rb --project "${project_path}" --job-id "${test_metadata_with_mapping_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
fi
scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}"
diff --git a/scripts/utils.sh b/scripts/utils.sh
index 6747efa73d7..c598afc4582 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -89,12 +89,12 @@ function echosuccess() {
function fail_pipeline_early() {
local dont_interrupt_me_job_id
- dont_interrupt_me_job_id=$(scripts/api/get_job_id --job-query "scope=success" --job-name "dont-interrupt-me")
+ dont_interrupt_me_job_id=$(scripts/api/get_job_id.rb --job-query "scope=success" --job-name "dont-interrupt-me")
if [[ -n "${dont_interrupt_me_job_id}" ]]; then
echoinfo "This pipeline cannot be interrupted due to \`dont-interrupt-me\` job ${dont_interrupt_me_job_id}"
else
echoinfo "Failing pipeline early for fast feedback due to test failures in rspec fail-fast."
- scripts/api/cancel_pipeline
+ scripts/api/cancel_pipeline.rb
fi
}
diff --git a/scripts/validate_migration_schema b/scripts/validate_migration_schema
new file mode 100755
index 00000000000..95a9ae9f93f
--- /dev/null
+++ b/scripts/validate_migration_schema
@@ -0,0 +1,119 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+require 'open3'
+
+class MigrationSchemaValidator
+ FILENAME = 'db/structure.sql'
+
+ MIGRATION_DIRS = %w[db/migrate db/post_migrate].freeze
+
+ SCHEMA_VERSION_DIR = 'db/schema_migrations'
+
+ VERSION_DIGITS = 14
+
+ def validate!
+ if committed_migrations.empty?
+ puts "\e[32m No migrations found, skipping schema validation\e[0m"
+ return
+ end
+
+ validate_schema_on_rollback!
+ validate_schema_on_migrate!
+ validate_schema_version_files!
+ end
+
+ private
+
+ def validate_schema_on_rollback!
+ committed_migrations.each do |filename|
+ version = find_migration_version(filename)
+
+ run("bin/rails db:migrate:down VERSION=#{version}")
+ end
+
+ git_command = "git diff #{diff_target} -- #{FILENAME}"
+ base_message = "rollback of added migrations does not revert #{FILENAME} to previous state"
+
+ validate_clean_output!(git_command, base_message)
+ end
+
+ def validate_schema_on_migrate!
+ run('bin/rails db:migrate')
+
+ git_command = "git diff -- #{FILENAME}"
+ base_message = "the committed #{FILENAME} does not match the one generated by running added migrations"
+
+ validate_clean_output!(git_command, base_message)
+ end
+
+ def validate_schema_version_files!
+ git_command = "git add -A -n #{SCHEMA_VERSION_DIR}"
+ base_message = "the committed files in #{SCHEMA_VERSION_DIR} do not match those expected by the added migrations"
+
+ validate_clean_output!(git_command, base_message)
+ end
+
+ def committed_migrations
+ @committed_migrations ||= begin
+ git_command = "git diff --name-only --diff-filter=A #{diff_target} -- #{MIGRATION_DIRS.join(' ')}"
+
+ run(git_command).split("\n")
+ end
+ end
+
+ def diff_target
+ @diff_target ||= pipeline_for_merged_results? ? target_branch : merge_base
+ end
+
+ def merge_base
+ run("git merge-base #{target_branch} #{source_ref}")
+ end
+
+ def target_branch
+ ENV['CI_MERGE_REQUEST_TARGET_BRANCH_NAME'] || ENV['TARGET'] || 'master'
+ end
+
+ def source_ref
+ ENV['CI_COMMIT_SHA'] || 'HEAD'
+ end
+
+ def pipeline_for_merged_results?
+ ENV.key?('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA')
+ end
+
+ def find_migration_version(filename)
+ file_basename = File.basename(filename)
+ version_match = /\A(?<version>\d{#{VERSION_DIGITS}})_/.match(file_basename)
+
+ die "#{filename} has an invalid migration version" if version_match.nil?
+
+ version_match[:version]
+ end
+
+ def validate_clean_output!(command, base_message)
+ command_output = run(command)
+
+ return if command_output.empty?
+
+ die "#{base_message}:\n#{command_output}"
+ end
+
+ def die(message, error_code: 1)
+ puts "\e[31mError: #{message}\e[0m"
+ exit error_code
+ end
+
+ def run(cmd)
+ puts "\e[32m$ #{cmd}\e[37m"
+ stdout_str, stderr_str, status = Open3.capture3(cmd)
+ puts "#{stdout_str}#{stderr_str}\e[0m"
+
+ die "command failed: #{stderr_str}" unless status.success?
+
+ stdout_str.chomp
+ end
+end
+
+MigrationSchemaValidator.new.validate!
diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping
index 1c66e19df50..8bf25ea3b5f 100755
--- a/scripts/verify-tff-mapping
+++ b/scripts/verify-tff-mapping
@@ -41,8 +41,8 @@ tests = [
{
explanation: 'Tooling should map to respective spec',
- source: 'tooling/lib/tooling/test_file_finder.rb',
- expected: ['spec/tooling/lib/tooling/test_file_finder_spec.rb']
+ source: 'tooling/lib/tooling/helm3_client.rb',
+ expected: ['spec/tooling/lib/tooling/helm3_client_spec.rb']
},
{