diff options
author | Robert Speicher <rspeicher@gmail.com> | 2021-01-20 13:34:23 -0600 |
---|---|---|
committer | Robert Speicher <rspeicher@gmail.com> | 2021-01-20 13:34:23 -0600 |
commit | 6438df3a1e0fb944485cebf07976160184697d72 (patch) | |
tree | 00b09bfd170e77ae9391b1a2f5a93ef6839f2597 /scripts | |
parent | 42bcd54d971da7ef2854b896a7b34f4ef8601067 (diff) | |
download | gitlab-ce-6438df3a1e0fb944485cebf07976160184697d72.tar.gz |
Add latest changes from gitlab-org/gitlab@13-8-stable-eev13.8.0-rc42
Diffstat (limited to 'scripts')
26 files changed, 357 insertions, 57 deletions
diff --git a/scripts/api/cancel_pipeline b/scripts/api/cancel_pipeline.rb index 0965877a69a..0965877a69a 100755 --- a/scripts/api/cancel_pipeline +++ b/scripts/api/cancel_pipeline.rb diff --git a/scripts/api/download_job_artifact b/scripts/api/download_job_artifact.rb index 8e2207c6fa7..8e2207c6fa7 100755 --- a/scripts/api/download_job_artifact +++ b/scripts/api/download_job_artifact.rb diff --git a/scripts/api/get_job_id b/scripts/api/get_job_id.rb index c7fe859db91..c7fe859db91 100755 --- a/scripts/api/get_job_id +++ b/scripts/api/get_job_id.rb diff --git a/scripts/api/play_job b/scripts/api/play_job.rb index 199f7e65633..408dfdf1ef0 100755 --- a/scripts/api/play_job +++ b/scripts/api/play_job.rb @@ -14,7 +14,6 @@ class PlayJob }.freeze def initialize(options) - @project = options.delete(:project) @options = options Gitlab.configure do |config| @@ -24,14 +23,18 @@ class PlayJob end def execute - job = JobFinder.new(project, options.slice(:api_token, :pipeline_id, :job_name).merge(scope: 'manual')).execute + job = JobFinder.new(options.slice(:project, :api_token, :pipeline_id, :job_name).merge(scope: 'manual')).execute Gitlab.job_play(project, job.id) end private - attr_reader :project, :options + attr_reader :options + + def project + options[:project] + end end if $0 == __FILE__ diff --git a/scripts/frontend/block_dependencies.js b/scripts/frontend/block_dependencies.js index c9257c9f72b..a1ff8d5ee36 100644 --- a/scripts/frontend/block_dependencies.js +++ b/scripts/frontend/block_dependencies.js @@ -5,13 +5,13 @@ const dependencies = packageJson.dependencies; const devDependencies = packageJson.devDependencies; const blockedDependenciesNames = Object.keys(blockedDependencies); const blockedDependenciesFound = blockedDependenciesNames.filter( - blockedDependency => dependencies[blockedDependency] || devDependencies[blockedDependency], + (blockedDependency) => dependencies[blockedDependency] || devDependencies[blockedDependency], ); if (blockedDependenciesFound.length) { console.log('The following package.json dependencies are not allowed:'); - blockedDependenciesFound.forEach(blockedDependency => { + blockedDependenciesFound.forEach((blockedDependency) => { const infoLink = blockedDependencies[blockedDependency]; console.log(`- ${blockedDependency}: See ${infoLink} for more information.`); diff --git a/scripts/frontend/check_page_bundle_mixins_css_for_sideeffects.js b/scripts/frontend/check_page_bundle_mixins_css_for_sideeffects.js index a2bb9f56d84..34e939e3ceb 100755 --- a/scripts/frontend/check_page_bundle_mixins_css_for_sideeffects.js +++ b/scripts/frontend/check_page_bundle_mixins_css_for_sideeffects.js @@ -27,7 +27,7 @@ const file = fs.readFileSync(paths[0], 'utf-8'); const parsed = pjs.parse(file); -if (parsed.nodes.every(node => ['comment', 'atrule'].includes(node.type))) { +if (parsed.nodes.every((node) => ['comment', 'atrule'].includes(node.type))) { console.log('The file does not introduce any side effects, we are all good.'); process.exit(0); } diff --git a/scripts/frontend/extract_gettext_all.js b/scripts/frontend/extract_gettext_all.js index 725522a3540..c34c9a0233d 100644 --- a/scripts/frontend/extract_gettext_all.js +++ b/scripts/frontend/extract_gettext_all.js @@ -45,7 +45,7 @@ function printJson() { text += `\u0000${message.textPlural}`; } - message.references.forEach(reference => { + message.references.forEach((reference) => { const filename = reference.replace(/:\d+$/, ''); if (!Array.isArray(result[filename])) { diff --git a/scripts/frontend/file_test_coverage.js b/scripts/frontend/file_test_coverage.js index 7d1eb45d4bc..ec6ec4a1e9d 100755 --- a/scripts/frontend/file_test_coverage.js +++ b/scripts/frontend/file_test_coverage.js @@ -17,11 +17,11 @@ const sourceDirectories = ['app/assets/javascripts']; const testDirectories = ['spec/javascripts', 'spec/frontend']; if (fs.existsSync('ee')) { - sourceDirectories.forEach(dir => { + sourceDirectories.forEach((dir) => { sourceDirectories.push(`ee/${dir}`); }); - testDirectories.forEach(dir => { + testDirectories.forEach((dir) => { testDirectories.push(`ee/${dir}`); }); } @@ -29,10 +29,10 @@ if (fs.existsSync('ee')) { let numSourceFiles = 0; let numTestFiles = 0; -const isVerbose = process.argv.some(arg => arg === '-v'); +const isVerbose = process.argv.some((arg) => arg === '-v'); -const countSourceFiles = path => - forEachFileIn(path, fileName => { +const countSourceFiles = (path) => + forEachFileIn(path, (fileName) => { if (fileName.endsWith('.vue') || fileName.endsWith('.js')) { if (isVerbose) { console.log(`source file: ${fileName}`); @@ -42,8 +42,8 @@ const countSourceFiles = path => } }); -const countTestFiles = path => - forEachFileIn(path, fileName => { +const countTestFiles = (path) => + forEachFileIn(path, (fileName) => { if (fileName.endsWith('_spec.js')) { if (isVerbose) { console.log(`test file: ${fileName}`); @@ -63,7 +63,7 @@ function forEachFileIn(dirPath, callback) { return; } - files.forEach(fileName => { + files.forEach((fileName) => { const absolutePath = path.join(dirPath, fileName); const stats = fs.statSync(absolutePath); if (stats.isFile()) { diff --git a/scripts/frontend/frontend_script_utils.js b/scripts/frontend/frontend_script_utils.js index e3d357b4a40..43016dce6a4 100644 --- a/scripts/frontend/frontend_script_utils.js +++ b/scripts/frontend/frontend_script_utils.js @@ -9,15 +9,10 @@ const exec = (command, args) => { return execFileSync(command, args, options); }; -const execGitCmd = args => - exec('git', args) - .trim() - .toString() - .split('\n') - .filter(Boolean); +const execGitCmd = (args) => exec('git', args).trim().toString().split('\n').filter(Boolean); module.exports = { - getStagedFiles: fileExtensionFilter => { + getStagedFiles: (fileExtensionFilter) => { const gitOptions = ['diff', '--name-only', '--cached', '--diff-filter=ACMRTUB']; if (fileExtensionFilter) gitOptions.push(...fileExtensionFilter); return execGitCmd(gitOptions); diff --git a/scripts/frontend/merge_coverage_frontend.js b/scripts/frontend/merge_coverage_frontend.js index 99034176b29..0c45a38b9b5 100644 --- a/scripts/frontend/merge_coverage_frontend.js +++ b/scripts/frontend/merge_coverage_frontend.js @@ -11,7 +11,7 @@ const reportFiles = sync(`${coverageDir}/*/coverage-final.json`); // Normalize coverage report generated by jest that has additional "data" key // https://github.com/facebook/jest/issues/2418#issuecomment-423806659 -const normalizeReport = report => { +const normalizeReport = (report) => { const normalizedReport = Object.assign({}, report); Object.entries(normalizedReport).forEach(([k, v]) => { if (v.data) normalizedReport[k] = v.data; @@ -20,12 +20,12 @@ const normalizeReport = report => { }; reportFiles - .map(reportFile => require(reportFile)) + .map((reportFile) => require(reportFile)) .map(normalizeReport) - .forEach(report => coverageMap.merge(report)); + .forEach((report) => coverageMap.merge(report)); const context = createContext({ coverageMap: coverageMap, dir: 'coverage-frontend' }); -['json', 'lcov', 'text-summary', 'clover', 'cobertura'].forEach(reporter => { +['json', 'lcov', 'text-summary', 'clover', 'cobertura'].forEach((reporter) => { create(reporter, {}).execute(context); }); diff --git a/scripts/frontend/prettier.js b/scripts/frontend/prettier.js index 7772f80c233..f721e46f36b 100644 --- a/scripts/frontend/prettier.js +++ b/scripts/frontend/prettier.js @@ -7,11 +7,11 @@ const matchExtensions = ['js', 'vue', 'graphql']; // This will improve glob performance by excluding certain directories. // The .prettierignore file will also be respected, but after the glob has executed. -const globIgnore = ['**/node_modules/**', 'vendor/**', 'public/**']; +const globIgnore = ['**/node_modules/**', 'vendor/**', 'public/**', 'fixtures/**']; const readFileAsync = (file, options) => new Promise((resolve, reject) => { - fs.readFile(file, options, function(err, data) { + fs.readFile(file, options, function (err, data) { if (err) reject(err); else resolve(data); }); @@ -19,7 +19,7 @@ const readFileAsync = (file, options) => const writeFileAsync = (file, data, options) => new Promise((resolve, reject) => { - fs.writeFile(file, data, options, function(err) { + fs.writeFile(file, data, options, function (err) { if (err) reject(err); else resolve(); }); @@ -35,7 +35,7 @@ console.log( `Loading all ${allFiles ? '' : 'staged '}files ${globDir ? `within ${globDir} ` : ''}...`, ); -const globPatterns = matchExtensions.map(ext => `${globDir}**/*.${ext}`); +const globPatterns = matchExtensions.map((ext) => `${globDir}**/*.${ext}`); const matchedFiles = allFiles ? glob.sync(`{${globPatterns.join(',')}}`, { ignore: globIgnore }) : getStagedFiles(globPatterns); @@ -62,7 +62,7 @@ Please format each file listed below or run "${fixCommand}" `; const checkFileWithOptions = (filePath, options) => - readFileAsync(filePath, 'utf8').then(input => { + readFileAsync(filePath, 'utf8').then((input) => { if (shouldSave) { const output = prettier.format(input, options); if (input === output) { @@ -92,7 +92,7 @@ const checkFileWithOptions = (filePath, options) => } }); -const checkFileWithPrettierConfig = filePath => +const checkFileWithPrettierConfig = (filePath) => prettier .getFileInfo(filePath, { ignorePath: '.prettierignore' }) .then(({ ignored, inferredParser }) => { @@ -100,7 +100,7 @@ const checkFileWithPrettierConfig = filePath => ignoredCount += 1; return; } - return prettier.resolveConfig(filePath).then(fileOptions => { + return prettier.resolveConfig(filePath).then((fileOptions) => { const options = { ...fileOptions, parser: inferredParser }; return checkFileWithOptions(filePath, options); }); @@ -115,7 +115,7 @@ Promise.all(matchedFiles.map(checkFileWithPrettierConfig)) if (didWarn) process.exit(1); }) - .catch(e => { + .catch((e) => { console.log(`\nAn error occurred while processing files with prettier: ${e.message}\n`); process.exit(1); }); diff --git a/scripts/frontend/stylelint/stylelint-duplicate-selectors.js b/scripts/frontend/stylelint/stylelint-duplicate-selectors.js index 4b46ee21a7a..89242158157 100644 --- a/scripts/frontend/stylelint/stylelint-duplicate-selectors.js +++ b/scripts/frontend/stylelint/stylelint-duplicate-selectors.js @@ -8,12 +8,12 @@ const messages = stylelint.utils.ruleMessages(ruleName, { }, }); -module.exports = stylelint.createPlugin(ruleName, function(enabled) { +module.exports = stylelint.createPlugin(ruleName, function (enabled) { if (!enabled) { return; } - return function(root, result) { + return function (root, result) { const selectorGroups = {}; utils.createPropertiesHashmap(root, result, ruleName, messages, selectorGroups, true); }; diff --git a/scripts/frontend/stylelint/stylelint-utility-classes.js b/scripts/frontend/stylelint/stylelint-utility-classes.js index 8a1cfdbf302..1b266fc31c9 100644 --- a/scripts/frontend/stylelint/stylelint-utility-classes.js +++ b/scripts/frontend/stylelint/stylelint-utility-classes.js @@ -10,12 +10,12 @@ const messages = stylelint.utils.ruleMessages(ruleName, { }, }); -module.exports = stylelint.createPlugin(ruleName, function(enabled) { +module.exports = stylelint.createPlugin(ruleName, function (enabled) { if (!enabled) { return; } - return function(root, result) { + return function (root, result) { utils.createPropertiesHashmap(root, result, ruleName, messages, utilityClasses, false); }; }); diff --git a/scripts/frontend/stylelint/stylelint-utility-map.js b/scripts/frontend/stylelint/stylelint-utility-map.js index 941198e82a4..bf8ee362740 100644 --- a/scripts/frontend/stylelint/stylelint-utility-map.js +++ b/scripts/frontend/stylelint/stylelint-utility-map.js @@ -32,7 +32,7 @@ sass.render( // This suppresses a postcss warning from: undefined, }) - .then(result => { + .then((result) => { const selectorGroups = {}; utils.createPropertiesHashmap(result.root, result, null, null, selectorGroups, true); @@ -42,7 +42,7 @@ sass.render( prettierOptions, ); - fs.writeFile(hashMapPath, prettyHashmap, function(err) { + fs.writeFile(hashMapPath, prettyHashmap, function (err) { if (err) { return console.log(err); } diff --git a/scripts/frontend/stylelint/stylelint-utils.js b/scripts/frontend/stylelint/stylelint-utils.js index 2d931c1c4c2..e7452b0cdb2 100644 --- a/scripts/frontend/stylelint/stylelint-utils.js +++ b/scripts/frontend/stylelint/stylelint-utils.js @@ -9,7 +9,7 @@ module.exports.createPropertiesHashmap = ( selectorGroups, addSelectors, ) => { - ruleRoot.walkRules(rule => { + ruleRoot.walkRules((rule) => { const selector = rule.selector.replace(/(?:\r\n|\r|\n)/g, ' '); if ( @@ -25,7 +25,7 @@ module.exports.createPropertiesHashmap = ( ) ) { let cssArray = []; - rule.nodes.forEach(function(property) { + rule.nodes.forEach(function (property) { const { prop, value } = property; if (property && value) { const propval = `${prop}${value}${property.important ? '!important' : ''}`; diff --git a/scripts/frontend/webpack_dev_server.js b/scripts/frontend/webpack_dev_server.js index 8026a8d47e2..fbb80c9617d 100755 --- a/scripts/frontend/webpack_dev_server.js +++ b/scripts/frontend/webpack_dev_server.js @@ -51,7 +51,7 @@ else { // print useful messages for nodemon events nodemon - .on('start', function() { + .on('start', function () { console.log(`Starting webpack webserver on http://${DEV_SERVER_HOST}:${DEV_SERVER_PORT}`); if (STATIC_MODE) { console.log('You are starting webpack in compile-once mode'); @@ -59,10 +59,10 @@ nodemon console.log('If you change them often, you might want to unset DEV_SERVER_STATIC'); } }) - .on('quit', function() { + .on('quit', function () { console.log('Shutting down webpack process'); process.exit(); }) - .on('restart', function(files) { + .on('restart', function (files) { console.log('Restarting webpack process due to: ', files); }); diff --git a/scripts/gitaly-test-build b/scripts/gitaly-test-build index 00927646046..62d3dbda911 100755 --- a/scripts/gitaly-test-build +++ b/scripts/gitaly-test-build @@ -12,7 +12,7 @@ class GitalyTestBuild include GitalyTest def run - abort 'gitaly build failed' unless system(env, 'make', chdir: tmp_tests_gitaly_dir) + abort 'gitaly build failed' unless build_gitaly ensure_gitlab_shell_secret! check_gitaly_config! diff --git a/scripts/gitaly-test-spawn b/scripts/gitaly-test-spawn index c2ff9cd08aa..caa41a9a0c3 100755 --- a/scripts/gitaly-test-spawn +++ b/scripts/gitaly-test-spawn @@ -8,6 +8,7 @@ class GitalyTestSpawn include GitalyTest def run + install_gitaly_gems if ENV['CI'] check_gitaly_config! # # Uncomment line below to see all gitaly logs merged into CI trace diff --git a/scripts/gitaly_test.rb b/scripts/gitaly_test.rb index 559ad8f4345..c7b3f72d590 100644 --- a/scripts/gitaly_test.rb +++ b/scripts/gitaly_test.rb @@ -41,7 +41,7 @@ module GitalyTest 'HOME' => File.expand_path('tmp/tests'), 'GEM_PATH' => Gem.path.join(':'), 'BUNDLE_APP_CONFIG' => File.join(File.dirname(gemfile), '.bundle/config'), - 'BUNDLE_FLAGS' => "--jobs=4 --retry=3 --quiet", + 'BUNDLE_FLAGS' => "--jobs=4 --retry=3", 'BUNDLE_INSTALL_FLAGS' => nil, 'BUNDLE_GEMFILE' => gemfile, 'RUBYOPT' => nil, @@ -78,6 +78,14 @@ module GitalyTest end end + def install_gitaly_gems + system(env, "make #{tmp_tests_gitaly_dir}/.ruby-bundle", chdir: tmp_tests_gitaly_dir) # rubocop:disable GitlabSecurity/SystemCommandInjection + end + + def build_gitaly + system(env, 'make', chdir: tmp_tests_gitaly_dir) # rubocop:disable GitlabSecurity/SystemCommandInjection + end + def start_gitaly start(:gitaly) end diff --git a/scripts/perf/gc/collect_gc_stats.rb b/scripts/perf/gc/collect_gc_stats.rb new file mode 100755 index 00000000000..050ad66a926 --- /dev/null +++ b/scripts/perf/gc/collect_gc_stats.rb @@ -0,0 +1,97 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +#### +# Loads GitLab application classes with a variety of GC settings and prints +# GC stats and timing data to standard out as CSV. +# +# The degree of parallelism can be increased by setting the PAR environment +# variable (default: 2). + +require 'benchmark' + +SETTINGS = { + 'DEFAULTS' => [''], + # Default: 10_000 + 'RUBY_GC_HEAP_INIT_SLOTS' => %w[100000 1000000 5000000], + # Default: 1.8 + 'RUBY_GC_HEAP_GROWTH_FACTOR' => %w[1.2 1 0.8], + # Default: 0 (disabled) + 'RUBY_GC_HEAP_GROWTH_MAX_SLOTS' => %w[10000 100000 1000000], + # Default: 2.0 + 'RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR' => %w[2.5 1.5 1], + # Default: 4096 (= 2^12) + 'RUBY_GC_HEAP_FREE_SLOTS' => %w[16384 2048 0], + # Default: 0.20 (20%) + 'RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO' => %w[0.1 0.01 0.001], + # Default: 0.40 (40%) + 'RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO' => %w[0.2 0.01 0.001], + # Default: 0.65 (65%) + 'RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO' => %w[0.2 0.02 0.002], + # Default: 16MB + 'RUBY_GC_MALLOC_LIMIT' => %w[8388608 4194304 1048576], + # Default: 32MB + 'RUBY_GC_MALLOC_LIMIT_MAX' => %w[16777216 8388608 1048576], + # Default: 1.4 + 'RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR' => %w[1.6 1 0.8], + # Default: 16MB + 'RUBY_GC_OLDMALLOC_LIMIT' => %w[8388608 4194304 1048576], + # Default: 128MB + 'RUBY_GC_OLDMALLOC_LIMIT_MAX' => %w[33554432 16777216 1048576], + # Default: 1.2 + 'RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR' => %w[1.4 1 0.8] +}.freeze + +USED_GCSTAT_KEYS = [ + :minor_gc_count, + :major_gc_count, + :heap_live_slots, + :heap_free_slots, + :total_allocated_pages, + :total_freed_pages, + :malloc_increase_bytes, + :malloc_increase_bytes_limit, + :oldmalloc_increase_bytes, + :oldmalloc_increase_bytes_limit +].freeze + +CSV_USED_GCSTAT_KEYS = USED_GCSTAT_KEYS.join(',') +CSV_HEADER = "setting,value,#{CSV_USED_GCSTAT_KEYS},RSS,gc_time_s,cpu_utime_s,cpu_stime_s,real_time_s\n" + +SCRIPT_PATH = __dir__ +RAILS_ROOT = "#{SCRIPT_PATH}/../../../" + +def collect_stats(setting, value) + warn "Testing #{setting} = #{value} ..." + env = { + setting => value, + 'RAILS_ROOT' => RAILS_ROOT, + 'SETTING_CSV' => "#{setting},#{value}", + 'GC_STAT_KEYS' => CSV_USED_GCSTAT_KEYS + } + system(env, 'ruby', "#{SCRIPT_PATH}/print_gc_stats.rb") +end + +par = ENV['PAR']&.to_i || 2 +batch_size = (SETTINGS.size.to_f / par).ceil +batches = SETTINGS.each_slice(batch_size) + +warn "Requested parallelism: #{par} (batches: #{batches.size}, batch size: #{batch_size})" + +puts CSV_HEADER + +elapsed = Benchmark.realtime do + threads = batches.each_with_index.map do |settings_batch, n| + Thread.new do + settings_batch.each do |setting, values| + values.each do |v| + collect_stats(setting, v) + end + end + end + end + + threads.each(&:join) +end + +warn "All done in #{elapsed} sec" diff --git a/scripts/perf/gc/print_gc_stats.rb b/scripts/perf/gc/print_gc_stats.rb new file mode 100755 index 00000000000..4aeb2f1ef07 --- /dev/null +++ b/scripts/perf/gc/print_gc_stats.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +# Promotes survivors from eden to old gen and runs a compaction. +# +# aka "Nakayoshi GC" +# +# https://github.com/puma/puma/blob/de632261ac45d7dd85230c83f6af6dd720f1cbd9/lib/puma/util.rb#L26-L35 +def nakayoshi_gc + 4.times { GC.start(full_mark: false) } + GC.compact +end + +# GC::Profiler is used elsewhere in the code base, so we provide a way for it +# to be used exclusively by this script, or otherwise results will be tainted. +module GC::Profiler + class << self + attr_accessor :use_exclusive + + %i[enable disable clear].each do |method| + alias_method "#{method}_orig", "#{method}" + + define_method(method) do + if use_exclusive + warn "GC::Profiler: ignoring call to #{method}" + return + end + + send("#{method}_orig") # rubocop: disable GitlabSecurity/PublicSend + end + end + end +end + +GC::Profiler.enable +GC::Profiler.use_exclusive = true + +require 'benchmark' + +RAILS_ROOT = ENV['RAILS_ROOT'] + +tms = Benchmark.measure do + require RAILS_ROOT + 'config/boot' + require RAILS_ROOT + 'config/environment' +end + +GC::Profiler.use_exclusive = false + +nakayoshi_gc + +gc_stats = GC.stat +warn gc_stats.inspect + +gc_total_time = GC::Profiler.total_time + +GC::Profiler.report($stderr) +GC::Profiler.disable + +gc_stat_keys = ENV['GC_STAT_KEYS'].to_s.split(',').map(&:to_sym) + +values = [] +values << ENV['SETTING_CSV'] +values += gc_stat_keys.map { |k| gc_stats[k] } +values << ::Gitlab::Metrics::System.memory_usage_rss +values << gc_total_time +values << tms.utime + tms.cutime +values << tms.stime + tms.cstime +values << tms.real + +puts values.join(',') diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb index e40c6cd7276..bef5b7ad5ee 100755 --- a/scripts/review_apps/automated_cleanup.rb +++ b/scripts/review_apps/automated_cleanup.rb @@ -115,6 +115,10 @@ class AutomatedCleanup delete_helm_releases(releases_to_delete) end + def perform_stale_pvc_cleanup!(days:) + kubernetes.cleanup_by_created_at(resource_type: 'pvc', created_before: threshold_time(days: days), wait: false) + end + private def fetch_environment(environment) @@ -155,7 +159,7 @@ class AutomatedCleanup releases_names = releases.map(&:name) helm.delete(release_name: releases_names) - kubernetes.cleanup(release_name: releases_names, wait: false) + kubernetes.cleanup_by_release(release_name: releases_names, wait: false) rescue Tooling::Helm3Client::CommandFailedError => ex raise ex unless ignore_exception?(ex.message, IGNORED_HELM_ERRORS) @@ -198,4 +202,8 @@ timed('Helm releases cleanup') do automated_cleanup.perform_helm_releases_cleanup!(days: 7) end +timed('Stale PVC cleanup') do + automated_cleanup.perform_stale_pvc_cleanup!(days: 30) +end + exit(0) diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index 5b724c9251b..85d7dcec476 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -7,14 +7,14 @@ function retrieve_tests_metadata() { local test_metadata_job_id # Ruby - test_metadata_job_id=$(scripts/api/get_job_id --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata") + test_metadata_job_id=$(scripts/api/get_job_id.rb --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata") if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then - scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" + scripts/api/download_job_artifact.rb --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" fi if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then - scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}" + scripts/api/download_job_artifact.rb --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}" fi } @@ -42,10 +42,10 @@ function retrieve_tests_mapping() { local project_path="gitlab-org/gitlab" local test_metadata_with_mapping_job_id - test_metadata_with_mapping_job_id=$(scripts/api/get_job_id --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") + test_metadata_with_mapping_job_id=$(scripts/api/get_job_id.rb --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then - (scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_with_mapping_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}" + (scripts/api/download_job_artifact.rb --project "${project_path}" --job-id "${test_metadata_with_mapping_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}" fi scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}" diff --git a/scripts/utils.sh b/scripts/utils.sh index 6747efa73d7..c598afc4582 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -89,12 +89,12 @@ function echosuccess() { function fail_pipeline_early() { local dont_interrupt_me_job_id - dont_interrupt_me_job_id=$(scripts/api/get_job_id --job-query "scope=success" --job-name "dont-interrupt-me") + dont_interrupt_me_job_id=$(scripts/api/get_job_id.rb --job-query "scope=success" --job-name "dont-interrupt-me") if [[ -n "${dont_interrupt_me_job_id}" ]]; then echoinfo "This pipeline cannot be interrupted due to \`dont-interrupt-me\` job ${dont_interrupt_me_job_id}" else echoinfo "Failing pipeline early for fast feedback due to test failures in rspec fail-fast." - scripts/api/cancel_pipeline + scripts/api/cancel_pipeline.rb fi } diff --git a/scripts/validate_migration_schema b/scripts/validate_migration_schema new file mode 100755 index 00000000000..95a9ae9f93f --- /dev/null +++ b/scripts/validate_migration_schema @@ -0,0 +1,119 @@ +#!/usr/bin/env ruby + +# frozen_string_literal: true + +require 'open3' + +class MigrationSchemaValidator + FILENAME = 'db/structure.sql' + + MIGRATION_DIRS = %w[db/migrate db/post_migrate].freeze + + SCHEMA_VERSION_DIR = 'db/schema_migrations' + + VERSION_DIGITS = 14 + + def validate! + if committed_migrations.empty? + puts "\e[32m No migrations found, skipping schema validation\e[0m" + return + end + + validate_schema_on_rollback! + validate_schema_on_migrate! + validate_schema_version_files! + end + + private + + def validate_schema_on_rollback! + committed_migrations.each do |filename| + version = find_migration_version(filename) + + run("bin/rails db:migrate:down VERSION=#{version}") + end + + git_command = "git diff #{diff_target} -- #{FILENAME}" + base_message = "rollback of added migrations does not revert #{FILENAME} to previous state" + + validate_clean_output!(git_command, base_message) + end + + def validate_schema_on_migrate! + run('bin/rails db:migrate') + + git_command = "git diff -- #{FILENAME}" + base_message = "the committed #{FILENAME} does not match the one generated by running added migrations" + + validate_clean_output!(git_command, base_message) + end + + def validate_schema_version_files! + git_command = "git add -A -n #{SCHEMA_VERSION_DIR}" + base_message = "the committed files in #{SCHEMA_VERSION_DIR} do not match those expected by the added migrations" + + validate_clean_output!(git_command, base_message) + end + + def committed_migrations + @committed_migrations ||= begin + git_command = "git diff --name-only --diff-filter=A #{diff_target} -- #{MIGRATION_DIRS.join(' ')}" + + run(git_command).split("\n") + end + end + + def diff_target + @diff_target ||= pipeline_for_merged_results? ? target_branch : merge_base + end + + def merge_base + run("git merge-base #{target_branch} #{source_ref}") + end + + def target_branch + ENV['CI_MERGE_REQUEST_TARGET_BRANCH_NAME'] || ENV['TARGET'] || 'master' + end + + def source_ref + ENV['CI_COMMIT_SHA'] || 'HEAD' + end + + def pipeline_for_merged_results? + ENV.key?('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') + end + + def find_migration_version(filename) + file_basename = File.basename(filename) + version_match = /\A(?<version>\d{#{VERSION_DIGITS}})_/.match(file_basename) + + die "#{filename} has an invalid migration version" if version_match.nil? + + version_match[:version] + end + + def validate_clean_output!(command, base_message) + command_output = run(command) + + return if command_output.empty? + + die "#{base_message}:\n#{command_output}" + end + + def die(message, error_code: 1) + puts "\e[31mError: #{message}\e[0m" + exit error_code + end + + def run(cmd) + puts "\e[32m$ #{cmd}\e[37m" + stdout_str, stderr_str, status = Open3.capture3(cmd) + puts "#{stdout_str}#{stderr_str}\e[0m" + + die "command failed: #{stderr_str}" unless status.success? + + stdout_str.chomp + end +end + +MigrationSchemaValidator.new.validate! diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping index 1c66e19df50..8bf25ea3b5f 100755 --- a/scripts/verify-tff-mapping +++ b/scripts/verify-tff-mapping @@ -41,8 +41,8 @@ tests = [ { explanation: 'Tooling should map to respective spec', - source: 'tooling/lib/tooling/test_file_finder.rb', - expected: ['spec/tooling/lib/tooling/test_file_finder_spec.rb'] + source: 'tooling/lib/tooling/helm3_client.rb', + expected: ['spec/tooling/lib/tooling/helm3_client_spec.rb'] }, { |