summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/frontend/check_no_partial_karma_jest.sh44
-rwxr-xr-xscripts/frontend/file_test_coverage.js2
-rw-r--r--scripts/frontend/startup_css/constants.js1
-rw-r--r--scripts/prepare_build.sh7
-rwxr-xr-xscripts/review_apps/automated_cleanup.rb40
-rw-r--r--scripts/review_apps/base-config.yaml3
-rwxr-xr-xscripts/review_apps/review-apps.sh7
-rw-r--r--scripts/rspec_helpers.sh62
-rwxr-xr-xscripts/setup-test-env2
-rwxr-xr-xscripts/static-analysis145
-rwxr-xr-xscripts/used-feature-flags1
-rw-r--r--scripts/utils.sh3
12 files changed, 208 insertions, 109 deletions
diff --git a/scripts/frontend/check_no_partial_karma_jest.sh b/scripts/frontend/check_no_partial_karma_jest.sh
deleted file mode 100755
index c5fffa5900b..00000000000
--- a/scripts/frontend/check_no_partial_karma_jest.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-
-karma_directory=spec/javascripts
-
-if [ -d ee ]; then
- karma_directory="$karma_directory ee/$karma_directory"
-fi
-
-karma_files=$(find $karma_directory -type f -name '*_spec.js' -not -path '*/helpers/*')
-violations=""
-
-for karma_file in $karma_files; do
- jest_file=${karma_file/spec\/javascripts/"spec/frontend"}
-
- if [ -f $jest_file ]; then
- violations="$violations $jest_file"
- fi
-done
-
-if [[ -z "$violations" ]]; then
- echo "All good!"
- exit 0
-else
- echo "Danger! The following Jest specs have corresponding files in the Karma spec directory (i.e. spec/javascripts):"
- echo ""
- echo "------------------------------"
- for file in $violations; do
- echo $file
- done
- echo "------------------------------"
- echo ""
- echo "For each of these files, please either:"
- echo ""
- echo "1. Fully migrate the file to Jest and remove the corresponding Karma file."
- echo "2. Remove the Jest file for now, make any relevant changes in the corresponding Karma file, and handle the migration to Jest in a separate MR."
- echo ""
- echo "Why is this a problem?"
- echo ""
- echo "- It's nice to have a single source of truth for the unit tests of a subject."
- echo "- This will cause conflicts if the remaining Karma spec is migrated using our automated tool."
- echo " https://gitlab.com/gitlab-org/frontend/playground/migrate-karma-to-jest"
- echo ""
- exit 1
-fi
diff --git a/scripts/frontend/file_test_coverage.js b/scripts/frontend/file_test_coverage.js
index 04a9035fce2..3ad92a5abbe 100755
--- a/scripts/frontend/file_test_coverage.js
+++ b/scripts/frontend/file_test_coverage.js
@@ -14,7 +14,7 @@ const fs = require('fs');
const path = require('path');
const sourceDirectories = ['app/assets/javascripts'];
-const testDirectories = ['spec/javascripts', 'spec/frontend'];
+const testDirectories = ['spec/frontend'];
if (fs.existsSync('ee')) {
sourceDirectories.forEach((dir) => {
diff --git a/scripts/frontend/startup_css/constants.js b/scripts/frontend/startup_css/constants.js
index 8f183e63659..83f43143e1b 100644
--- a/scripts/frontend/startup_css/constants.js
+++ b/scripts/frontend/startup_css/constants.js
@@ -50,6 +50,7 @@ const createMainOutput = ({ outFile, cssKeys, type }) => ({
htmlPaths: [
path.join(FIXTURES_ROOT, `startup_css/project-${type}.html`),
path.join(FIXTURES_ROOT, `startup_css/project-${type}-signed-out.html`),
+ path.join(FIXTURES_ROOT, `startup_css/project-${type}-search-ff-on.html`),
],
cssKeys,
purgeOptions: {
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index e0be80d429f..f3b9ac56082 100644
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -10,7 +10,12 @@ fi
cp config/gitlab.yml.example config/gitlab.yml
sed -i 's/bin_path: \/usr\/bin\/git/bin_path: \/usr\/local\/bin\/git/' config/gitlab.yml
-cp config/database.yml.postgresql config/database.yml
+if [ "$DECOMPOSED_DB" == "true" ]; then
+ echo "Using decomposed database config (config/database.yml.decomposed-postgresql)"
+ cp config/database.yml.decomposed-postgresql config/database.yml
+else
+ cp config/database.yml.postgresql config/database.yml
+fi
if [ -f config/database_geo.yml.postgresql ]; then
cp config/database_geo.yml.postgresql config/database_geo.yml
diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb
index 5707f02d3f0..90dc0fd418e 100755
--- a/scripts/review_apps/automated_cleanup.rb
+++ b/scripts/review_apps/automated_cleanup.rb
@@ -95,6 +95,42 @@ class AutomatedCleanup
delete_helm_releases(releases_to_delete)
end
+ def perform_gitlab_docs_environment_cleanup!(days_for_stop:, days_for_delete:)
+ puts "Checking for Docs Review Apps not updated in the last #{days_for_stop} days..."
+
+ checked_environments = []
+ stop_threshold = threshold_time(days: days_for_stop)
+ delete_threshold = threshold_time(days: days_for_delete)
+
+ max_delete_count = 1000
+ delete_count = 0
+
+ gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment|
+ environment = deployment.environment
+
+ next unless environment
+ next unless environment.name.start_with?('review-docs/')
+ next if checked_environments.include?(environment.slug)
+
+ last_deploy = deployment.created_at
+ deployed_at = Time.parse(last_deploy)
+
+ if deployed_at < stop_threshold
+ environment_state = fetch_environment(environment)&.state
+ stop_environment(environment, deployment) if environment_state && environment_state != 'stopped'
+ end
+
+ if deployed_at < delete_threshold
+ delete_environment(environment, deployment)
+ delete_count += 1
+
+ break if delete_count > max_delete_count
+ end
+
+ checked_environments << environment.slug
+ end
+ end
+
def perform_helm_releases_cleanup!(days:)
puts "Checking for Helm releases that are failed or not updated in the last #{days} days..."
@@ -203,6 +239,10 @@ timed('Review Apps cleanup') do
automated_cleanup.perform_gitlab_environment_cleanup!(days_for_stop: 5, days_for_delete: 6)
end
+timed('Docs Review Apps cleanup') do
+ automated_cleanup.perform_gitlab_docs_environment_cleanup!(days_for_stop: 20, days_for_delete: 30)
+end
+
puts
timed('Helm releases cleanup') do
diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml
index 3480b7e8bec..7bb9c010016 100644
--- a/scripts/review_apps/base-config.yaml
+++ b/scripts/review_apps/base-config.yaml
@@ -1,7 +1,8 @@
global:
appConfig:
enableUsagePing: false
- imagePullPolicy: Always
+ image:
+ pullPolicy: Always
ingress:
annotations:
external-dns.alpha.kubernetes.io/ttl: 10
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index 641bf6a5d10..8ec26e7ba89 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -244,10 +244,9 @@ function deploy() {
echoinfo "Deploying ${release} to ${CI_ENVIRONMENT_URL} ..." true
IMAGE_REPOSITORY="registry.gitlab.com/gitlab-org/build/cng-mirror"
- gitlab_migrations_image_repository="${IMAGE_REPOSITORY}/gitlab-rails-ee"
+ gitlab_toolbox_image_repository="${IMAGE_REPOSITORY}/gitlab-toolbox-ee"
gitlab_sidekiq_image_repository="${IMAGE_REPOSITORY}/gitlab-sidekiq-ee"
gitlab_webservice_image_repository="${IMAGE_REPOSITORY}/gitlab-webservice-ee"
- gitlab_task_runner_image_repository="${IMAGE_REPOSITORY}/gitlab-toolbox-ee"
gitlab_gitaly_image_repository="${IMAGE_REPOSITORY}/gitaly"
gitaly_image_tag=$(parse_gitaly_image_tag)
gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell"
@@ -272,7 +271,7 @@ HELM_CMD=$(cat << EOF
--set releaseOverride="${release}" \
--set global.hosts.hostSuffix="${HOST_SUFFIX}" \
--set global.hosts.domain="${REVIEW_APPS_DOMAIN}" \
- --set gitlab.migrations.image.repository="${gitlab_migrations_image_repository}" \
+ --set gitlab.migrations.image.repository="${gitlab_toolbox_image_repository}" \
--set gitlab.migrations.image.tag="${CI_COMMIT_REF_SLUG}" \
--set gitlab.gitaly.image.repository="${gitlab_gitaly_image_repository}" \
--set gitlab.gitaly.image.tag="${gitaly_image_tag}" \
@@ -286,7 +285,7 @@ HELM_CMD=$(cat << EOF
--set gitlab.webservice.image.tag="${CI_COMMIT_REF_SLUG}" \
--set gitlab.webservice.workhorse.image="${gitlab_workhorse_image_repository}" \
--set gitlab.webservice.workhorse.tag="${CI_COMMIT_REF_SLUG}" \
- --set gitlab.task-runner.image.repository="${gitlab_task_runner_image_repository}" \
+ --set gitlab.task-runner.image.repository="${gitlab_toolbox_image_repository}" \
--set gitlab.task-runner.image.tag="${CI_COMMIT_REF_SLUG}"
EOF
)
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index 0714ecfce80..797d9188f81 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -1,23 +1,33 @@
#!/usr/bin/env bash
function retrieve_tests_metadata() {
- mkdir -p knapsack/ rspec_flaky/ rspec_profiling/
+ mkdir -p $(dirname "$KNAPSACK_RSPEC_SUITE_REPORT_PATH") $(dirname "$FLAKY_RSPEC_SUITE_REPORT_PATH") rspec_profiling/
- # ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
- # always target the canonical project here, so the branch must be hardcoded
- local project_path="gitlab-org/gitlab"
- local artifact_branch="master"
- local test_metadata_job_id
+ if [[ -n "${RETRIEVE_TESTS_METADATA_FROM_PAGES}" ]]; then
+ if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
+ curl --location -o "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+ fi
- # Ruby
- test_metadata_job_id=$(scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" -q "status=success" -q "ref=${artifact_branch}" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata")
+ if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
+ curl --location -o "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+ fi
+ else
+ # ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
+ # always target the canonical project here, so the branch must be hardcoded
+ local project_path="gitlab-org/gitlab"
+ local artifact_branch="master"
+ local test_metadata_job_id
- if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
- scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
- fi
+ # Ruby
+ test_metadata_job_id=$(scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" -q "status=success" -q "ref=${artifact_branch}" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata")
- if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
- scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+ if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
+ scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+ fi
+
+ if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
+ scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+ fi
fi
}
@@ -40,18 +50,24 @@ function update_tests_metadata() {
}
function retrieve_tests_mapping() {
- mkdir -p crystalball/
+ mkdir -p $(dirname "$RSPEC_PACKED_TESTS_MAPPING_PATH")
- # ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
- # always target the canonical project here, so the branch must be hardcoded
- local project_path="gitlab-org/gitlab"
- local artifact_branch="master"
- local test_metadata_with_mapping_job_id
+ if [[ -n "${RETRIEVE_TESTS_METADATA_FROM_PAGES}" ]]; then
+ if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
+ (curl --location -o "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" "https://gitlab-org.gitlab.io/gitlab/${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+ fi
+ else
+ # ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
+ # always target the canonical project here, so the branch must be hardcoded
+ local project_path="gitlab-org/gitlab"
+ local artifact_branch="master"
+ local test_metadata_with_mapping_job_id
- test_metadata_with_mapping_job_id=$(scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" -q "status=success" -q "ref=${artifact_branch}" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz")
+ test_metadata_with_mapping_job_id=$(scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" -q "status=success" -q "ref=${artifact_branch}" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz")
- if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
- (scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_with_mapping_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+ if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
+ (scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_with_mapping_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+ fi
fi
scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}"
@@ -184,7 +200,7 @@ function rspec_matched_foss_tests() {
echo "This job is intentionally failed because there are more than ${test_file_count_threshold} FOSS test files matched,"
echo "which would take too long to run in this job."
echo "To reduce the likelihood of breaking FOSS pipelines,"
- echo "please add [RUN AS-IF-FOSS] to the MR title and restart the pipeline."
+ echo "please add ~\"pipeline:run-as-if-foss\" label to the merge request and trigger a new pipeline."
echo "This would run all as-if-foss jobs in this merge request"
echo "and remove this failing job from the pipeline."
exit 1
diff --git a/scripts/setup-test-env b/scripts/setup-test-env
index ebd3a48ae15..a81aaa5cda3 100755
--- a/scripts/setup-test-env
+++ b/scripts/setup-test-env
@@ -2,7 +2,7 @@
# frozen_string_literal: true
-require 'bundler/setup'
+require_relative '../config/bundler_setup'
require 'request_store'
require 'rake'
diff --git a/scripts/static-analysis b/scripts/static-analysis
index a1859254459..de5a1b407f9 100755
--- a/scripts/static-analysis
+++ b/scripts/static-analysis
@@ -14,51 +14,80 @@ class StaticAnalysis
"Browserslist: caniuse-lite is outdated. Please run next command `yarn upgrade`"
].freeze
+ Task = Struct.new(:command, :duration) do
+ def cmd
+ command.join(' ')
+ end
+ end
+ NodeAssignment = Struct.new(:index, :tasks) do
+ def total_duration
+ return 0 if tasks.empty?
+
+ tasks.sum(&:duration)
+ end
+ end
+
+ def self.project_path
+ project_root = File.expand_path('..', __dir__)
+
+ if Gitlab.jh?
+ "#{project_root}/jh"
+ else
+ project_root
+ end
+ end
+
# `gettext:updated_check` and `gitlab:sidekiq:sidekiq_queues_yml:check` will fail on FOSS installations
# (e.g. gitlab-org/gitlab-foss) since they test against a single
# file that is generated by an EE installation, which can
# contain values that a FOSS installation won't find. To work
# around this we will only enable this task on EE installations.
- TASKS_BY_DURATIONS_SECONDS_DESC = {
- %w[bin/rake lint:haml] => 800,
+ TASKS_WITH_DURATIONS_SECONDS = [
+ Task.new(%w[bin/rake lint:haml], 562),
# We need to disable the cache for this cop since it creates files under tmp/feature_flags/*.used,
# the cache would prevent these files from being created.
- %w[bundle exec rubocop --only Gitlab/MarkUsedFeatureFlags --cache false] => 600,
- (Gitlab.ee? ? %w[bin/rake gettext:updated_check] : nil) => 360,
- %w[yarn run lint:eslint:all] => 312,
- %w[yarn run lint:prettier] => 162,
- %w[bin/rake gettext:lint] => 65,
- %w[bundle exec license_finder] => 61,
- %w[bin/rake lint:static_verification] => 45,
- %w[bundle exec rubocop --parallel] => 40,
- %w[bin/rake config_lint] => 26,
- %w[bin/rake gitlab:sidekiq:all_queues_yml:check] => 15,
- (Gitlab.ee? ? %w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check] : nil) => 11,
- %w[yarn run internal:stylelint] => 8,
- %w[scripts/lint-conflicts.sh] => 1,
- %w[yarn run block-dependencies] => 1,
- %w[scripts/lint-rugged] => 1,
- %w[scripts/gemfile_lock_changed.sh] => 1,
- %w[scripts/frontend/check_no_partial_karma_jest.sh] => 1
- }.reject { |k| k.nil? }.sort_by { |a| -a[1] }.to_h.keys.freeze
-
- def run_tasks!
- tasks = tasks_to_run((ENV['CI_NODE_INDEX'] || 1).to_i, (ENV['CI_NODE_TOTAL'] || 1).to_i)
+ Task.new(%w[bundle exec rubocop --only Gitlab/MarkUsedFeatureFlags --cache false], 400),
+ (Gitlab.ee? ? Task.new(%w[bin/rake gettext:updated_check], 360) : nil),
+ Task.new(%w[yarn run lint:eslint:all], 312),
+ Task.new(%w[bundle exec rubocop --parallel], 60),
+ Task.new(%w[yarn run lint:prettier], 160),
+ Task.new(%w[bin/rake gettext:lint], 85),
+ Task.new(%W[bundle exec license_finder --decisions-file config/dependency_decisions.yml --project-path #{project_path}], 20),
+ Task.new(%w[bin/rake lint:static_verification], 35),
+ Task.new(%w[bin/rake config_lint], 10),
+ Task.new(%w[bin/rake gitlab:sidekiq:all_queues_yml:check], 15),
+ (Gitlab.ee? ? Task.new(%w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check], 11) : nil),
+ Task.new(%w[yarn run internal:stylelint], 8),
+ Task.new(%w[scripts/lint-conflicts.sh], 1),
+ Task.new(%w[yarn run block-dependencies], 1),
+ Task.new(%w[scripts/lint-rugged], 1),
+ Task.new(%w[scripts/gemfile_lock_changed.sh], 1)
+ ].compact.freeze
+
+ def run_tasks!(options = {})
+ node_assignment = tasks_to_run((ENV['CI_NODE_TOTAL'] || 1).to_i)[(ENV['CI_NODE_INDEX'] || 1).to_i - 1]
+
+ if options[:dry_run]
+ puts "Dry-run mode!"
+ return
+ end
static_analysis = Gitlab::Popen::Runner.new
-
- static_analysis.run(tasks) do |cmd, &run|
+ start_time = Time.now
+ static_analysis.run(node_assignment.tasks.map(&:command)) do |command, &run|
+ task = node_assignment.tasks.find { |task| task.command == command }
puts
- puts "$ #{cmd.join(' ')}"
+ puts "$ #{task.cmd}"
result = run.call
- puts "==> Finished in #{result.duration} seconds"
+ puts "==> Finished in #{result.duration} seconds (expected #{task.duration} seconds)"
puts
end
puts
puts '==================================================='
+ puts "Node finished running all tasks in #{Time.now - start_time} seconds (expected #{node_assignment.total_duration})"
puts
puts
@@ -107,16 +136,66 @@ class StaticAnalysis
.count { |result| !ALLOWED_WARNINGS.include?(result.stderr.strip) }
end
- def tasks_to_run(node_index, node_total)
- tasks = []
- TASKS_BY_DURATIONS_SECONDS_DESC.each_with_index do |task, i|
- tasks << task if i % node_total == (node_index - 1)
+ def tasks_to_run(node_total)
+ total_time = TASKS_WITH_DURATIONS_SECONDS.sum(&:duration).to_f
+ ideal_time_per_node = total_time / node_total
+ tasks_by_duration_desc = TASKS_WITH_DURATIONS_SECONDS.sort_by { |a| -a.duration }
+ nodes = Array.new(node_total) { |i| NodeAssignment.new(i + 1, []) }
+
+ puts "Total expected time: #{total_time}; ideal time per job: #{ideal_time_per_node}.\n\n"
+ puts "Tasks to distribute:"
+ tasks_by_duration_desc.each { |task| puts "* #{task.cmd} (#{task.duration}s)" }
+
+ # Distribute tasks optimally first
+ puts "\nAssigning tasks optimally."
+ distribute_tasks(tasks_by_duration_desc, nodes, ideal_time_per_node: ideal_time_per_node)
+
+ # Distribute remaining tasks, ordered by ascending duration
+ leftover_tasks = tasks_by_duration_desc - nodes.flat_map(&:tasks)
+
+ if leftover_tasks.any?
+ puts "\n\nAssigning remaining tasks: #{leftover_tasks.flat_map(&:cmd)}"
+ distribute_tasks(leftover_tasks, nodes.sort_by { |node| node.total_duration })
end
- tasks
+ nodes.each do |node|
+ puts "\nExpected duration for node #{node.index}: #{node.total_duration} seconds"
+ node.tasks.each { |task| puts "* #{task.cmd} (#{task.duration}s)" }
+ end
+
+ nodes
+ end
+
+ def distribute_tasks(tasks, nodes, ideal_time_per_node: nil)
+ condition =
+ if ideal_time_per_node
+ ->(task, node, ideal_time_per_node) { (task.duration + node.total_duration) <= ideal_time_per_node }
+ else
+ ->(*) { true }
+ end
+
+ tasks.each do |task|
+ nodes.each do |node|
+ if condition.call(task, node, ideal_time_per_node)
+ assign_task_to_node(tasks, node, task)
+ break
+ end
+ end
+ end
+ end
+
+ def assign_task_to_node(remaining_tasks, node, task)
+ node.tasks << task
+ puts "Assigning #{task.command} (#{task.duration}s) to node ##{node.index}. Node total duration: #{node.total_duration}s."
end
end
if $0 == __FILE__
- StaticAnalysis.new.run_tasks!
+ options = {}
+
+ if ARGV.include?('--dry-run')
+ options[:dry_run] = true
+ end
+
+ StaticAnalysis.new.run_tasks!(options)
end
diff --git a/scripts/used-feature-flags b/scripts/used-feature-flags
index 07c022a4c1a..e6a8149da71 100755
--- a/scripts/used-feature-flags
+++ b/scripts/used-feature-flags
@@ -2,6 +2,7 @@
# frozen_string_literal: true
require 'set'
+require 'fileutils'
class String
def red
diff --git a/scripts/utils.sh b/scripts/utils.sh
index 700dad58779..d2e8c151438 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -37,8 +37,9 @@ function bundle_install_script() {
fi;
bundle --version
- bundle config set path 'vendor'
+ bundle config set path "$(pwd)/vendor"
bundle config set clean 'true'
+ test -d jh && bundle config set gemfile 'jh/Gemfile'
echo "${BUNDLE_WITHOUT}"
bundle config