diff options
author | Mikhail Shchatko <mikhail.shchatko@mongodb.com> | 2023-02-20 10:06:36 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2023-02-22 11:40:03 +0000 |
commit | 65b9065fd72f65047e347f6bb3a56261f4d22810 (patch) | |
tree | a7aa94aab17fd055174e81566e570d51943cbe25 | |
parent | 72bf5680cbba28b5f2c909459093fba6c9f348d4 (diff) | |
download | mongo-65b9065fd72f65047e347f6bb3a56261f4d22810.tar.gz |
SERVER-73707 Evergreen timeout calculation fixes and cleanups
(cherry picked from commit 73d0f845e05e52c882bfc8e45d6e6758b7a6761d)
-rwxr-xr-x | buildscripts/evergreen_task_timeout.py | 22 | ||||
-rw-r--r-- | buildscripts/timeouts/timeout_service.py | 8 | ||||
-rw-r--r-- | etc/evergreen_timeouts.yml | 119 | ||||
-rw-r--r-- | evergreen/functions/task_timeout_determine.sh | 8 |
4 files changed, 30 insertions, 127 deletions
diff --git a/buildscripts/evergreen_task_timeout.py b/buildscripts/evergreen_task_timeout.py index 198288c4e44..f10b8aaba0c 100755 --- a/buildscripts/evergreen_task_timeout.py +++ b/buildscripts/evergreen_task_timeout.py @@ -28,20 +28,21 @@ DEFAULT_TIMEOUT_OVERRIDES = "etc/evergreen_timeouts.yml" DEFAULT_EVERGREEN_CONFIG = "etc/evergreen.yml" DEFAULT_EVERGREEN_AUTH_CONFIG = "~/.evergreen.yml" COMMIT_QUEUE_ALIAS = "__commit_queue" -UNITTEST_TASK = "run_unittests" IGNORED_SUITES = { - "integration_tests_replset", "integration_tests_replset_ssl_auth", "integration_tests_sharded", - "integration_tests_standalone", "integration_tests_standalone_audit", "mongos_test", - "server_selection_json_test" + "integration_tests_replset", + "integration_tests_replset_ssl_auth", + "integration_tests_sharded", + "integration_tests_standalone", + "integration_tests_standalone_audit", + "mongos_test", + "server_selection_json_test", + "sdam_json_test", } HISTORY_LOOKBACK = timedelta(weeks=2) -COMMIT_QUEUE_TIMEOUT = timedelta(minutes=40) +COMMIT_QUEUE_TIMEOUT = timedelta(minutes=20) DEFAULT_REQUIRED_BUILD_TIMEOUT = timedelta(hours=1, minutes=20) DEFAULT_NON_REQUIRED_BUILD_TIMEOUT = timedelta(hours=2) -# 2x the longest "run tests" phase for unittests as of c9bf1dbc9cc46e497b2f12b2d6685ef7348b0726, -# which is 5 mins 47 secs, excluding outliers below -UNITTESTS_TIMEOUT = timedelta(minutes=12) class TimeoutOverride(BaseModel): @@ -222,11 +223,6 @@ class TaskTimeoutOrchestrator: LOGGER.info("Overriding configured timeout", exec_timeout_secs=override.total_seconds()) determined_timeout = override - elif task_name == UNITTEST_TASK and override is None: - LOGGER.info("Overriding unittest timeout", - exec_timeout_secs=UNITTESTS_TIMEOUT.total_seconds()) - determined_timeout = UNITTESTS_TIMEOUT - elif _is_required_build_variant( variant) and determined_timeout > DEFAULT_REQUIRED_BUILD_TIMEOUT: LOGGER.info("Overriding required-builder timeout", diff --git a/buildscripts/timeouts/timeout_service.py b/buildscripts/timeouts/timeout_service.py index bc16e35f07e..3df4ecddb74 100644 --- a/buildscripts/timeouts/timeout_service.py +++ b/buildscripts/timeouts/timeout_service.py @@ -5,8 +5,7 @@ import inject import structlog from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService from buildscripts.timeouts.timeout import TimeoutEstimate -from buildscripts.util.teststats import HistoricTaskData -from evergreen import EvergreenApi +from buildscripts.util.teststats import HistoricTaskData, normalize_test_name LOGGER = structlog.get_logger(__name__) CLEAN_EVERY_N_HOOK = "CleanEveryN" @@ -53,7 +52,10 @@ class TimeoutService: if not historic_stats: return TimeoutEstimate.no_timeouts() - test_set = set(self.resmoke_proxy.list_tests(timeout_params.suite_name)) + test_set = { + normalize_test_name(test) + for test in self.resmoke_proxy.list_tests(timeout_params.suite_name) + } test_runtimes = [ stat for stat in historic_stats.get_tests_runtimes() if stat.test_name in test_set ] diff --git a/etc/evergreen_timeouts.yml b/etc/evergreen_timeouts.yml index 36c9811944c..453a4a5dffa 100644 --- a/etc/evergreen_timeouts.yml +++ b/etc/evergreen_timeouts.yml @@ -10,125 +10,28 @@ # in alphabetical order. overrides: - amazon-linux2-arm64-compile: - - task: server_discovery_and_monitoring_json_test - exec_timeout: 30 # 30 min. - - enterprise-amazon2-arm64: - - task: config_fuzzer_replica_sets_jscore_passthrough - exec_timeout: 150 # 2.5 hours. - - enterprise-amazon2022-arm64: - - task: config_fuzzer_replica_sets_jscore_passthrough - exec_timeout: 150 # 2.5 hours. - - enterprise-rhel-70-64-bit: - - task: config_fuzzer_replica_sets_jscore_passthrough - exec_timeout: 150 # 2.5 hours. enterprise-macos: - task: replica_sets_jscore_passthrough exec_timeout: 150 # 2.5 hours - - task: run_unittests - exec_timeout: 60 enterprise-macos-arm64: - task: replica_sets_jscore_passthrough exec_timeout: 150 # 2.5 hours - - task: run_unittests - exec_timeout: 60 - - enterprise-macos-cxx20: - - task: run_unittests - exec_timeout: 60 - - enterprise-macos-rosetta-2: - - task: run_unittests - exec_timeout: 60 enterprise-rhel-80-64-bit-coverage: - task: replica_sets_jscore_passthrough exec_timeout: 150 # 2.5 hours. - enterprise-rhel-80-64-bit-dynamic-required: - - task: replica_sets_large_txns_format - exec_timeout: 120 # 2 hours. - - task: config_fuzzer_replica_sets_jscore_passthrough - exec_timeout: 150 # 2.5 hours. - - enterprise-rhel80-debug-tsan: - - task: run_unittests - exec_timeout: 24 - - enterprise-windows: - - task: replica_sets_jscore_passthrough - exec_timeout: 180 # 3 hours. - - enterprise-windows-all-feature-flags-suggested: - - task: replica_sets_jscore_passthrough - exec_timeout: 180 # 3 hours. - - task: replica_sets_update_v1_oplog_jscore_passthrough - exec_timeout: 150 # 2.5 hours. - - enterprise-windows-inmem: - - task: replica_sets_jscore_passthrough - exec_timeout: 180 # 3 hours. - - enterprise-windows-required: - - task: replica_sets_jscore_passthrough - exec_timeout: 180 # 3 hours. - - task: replica_sets_update_v1_oplog_jscore_passthrough - exec_timeout: 150 # 2.5 hours. - - linux-64-debug: - - task: auth - exec_timeout: 60 # 1 hour. - - linux-x86-dynamic-compile-required: - - task: server_discovery_and_monitoring_json_test - exec_timeout: 30 # 30 min. - - linux-64-debug-repeated-execution: - - task: run_unittests - exec_timeout: 120 # 2 hours. - macos: - task: replica_sets_jscore_passthrough exec_timeout: 150 # 2.5 hours - - task: run_unittests - exec_timeout: 60 - - task: sharded_collections_jscore_passthrough - exec_timeout: 300 # 5 hours - - macos-debug-suggested: - - task: run_unittests - exec_timeout: 60 macos-arm64: - - task: run_unittests - exec_timeout: 60 - - task: sharding_update_v1_oplog_jscore_passthrough - exec_timeout: 150 # 2.5 hours - - task: sharded_collections_jscore_passthrough - exec_timeout: 300 # 5 hours - task: replica_sets_jscore_passthrough exec_timeout: 150 # 2.5 hours - rhel80-asan: - - task: run_unittests - exec_timeout: 24 - - rhel80-debug-asan: - - task: run_unittests - exec_timeout: 24 - - rhel80-debug-aubsan-lite: - - task: run_unittests - exec_timeout: 24 - rhel80-debug-ubsan: - - task: run_unittests - exec_timeout: 24 - task: update_timeseries_fuzzer exec_timeout: 150 # 2.5 hours @@ -136,18 +39,14 @@ overrides: - task: replica_sets_jscore_passthrough exec_timeout: 180 # 3 hours. - rhel80-ubsan: - - task: run_unittests - exec_timeout: 24 + rhel80-debug-ubsan-all-feature-flags: + - task: update_timeseries_fuzzer + exec_timeout: 150 # 2.5 hours - windows: - - task: replica_sets - exec_timeout: 180 # 3 hours. + rhel80-debug-ubsan-classic-engine: + - task: update_timeseries_fuzzer + exec_timeout: 150 # 2.5 hours - windows-debug-suggested: - - task: replica_sets_initsync_jscore_passthrough - exec_timeout: 150 # 2.5 hours. - - task: replica_sets_jscore_passthrough - exec_timeout: 180 # 3 hours. - - task: replica_sets_update_v1_oplog_jscore_passthrough - exec_timeout: 150 # 2.5 hours. + rhel80-debug-v4ubsan-all-feature-flags-experimental: + - task: update_timeseries_fuzzer + exec_timeout: 150 # 2.5 hours diff --git a/evergreen/functions/task_timeout_determine.sh b/evergreen/functions/task_timeout_determine.sh index 77dd4dca782..49dda4dd4a9 100644 --- a/evergreen/functions/task_timeout_determine.sh +++ b/evergreen/functions/task_timeout_determine.sh @@ -25,6 +25,12 @@ if [[ -n "${burn_in_bypass}" ]]; then build_variant_for_timeout=${burn_in_bypass} fi +if [[ -n "${alias}" ]]; then + evg_alias=${alias} +else + evg_alias="evg-alias-absent" +fi + activate_venv PATH=$PATH:$HOME:/ $python buildscripts/evergreen_task_timeout.py $timeout_factor \ --install-dir "${install_dir}" \ @@ -32,7 +38,7 @@ PATH=$PATH:$HOME:/ $python buildscripts/evergreen_task_timeout.py $timeout_facto --suite-name ${suite_name} \ --project ${project} \ --build-variant $build_variant_for_timeout \ - --evg-alias '${alias}' \ + --evg-alias $evg_alias \ --timeout ${timeout_secs} \ --exec-timeout ${exec_timeout_secs} \ --evg-api-config ./.evergreen.yml \ |