From f188bd47b3e6dc8c0a364797bcfb9006c8b35ccf Mon Sep 17 00:00:00 2001 From: David Bradford Date: Tue, 18 Jun 2019 15:27:00 -0400 Subject: SERVER-41806: Support for global overrides in evergreen_resmoke_job_count --- buildscripts/evergreen_resmoke_job_count.py | 45 +++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 5 deletions(-) (limited to 'buildscripts/evergreen_resmoke_job_count.py') diff --git a/buildscripts/evergreen_resmoke_job_count.py b/buildscripts/evergreen_resmoke_job_count.py index 6d82d251f30..d52471325e1 100644 --- a/buildscripts/evergreen_resmoke_job_count.py +++ b/buildscripts/evergreen_resmoke_job_count.py @@ -13,6 +13,17 @@ CPU_COUNT = psutil.cpu_count() PLATFORM_MACHINE = platform.machine() SYS_PLATFORM = sys.platform +# The following constants define tasks that should override the resmoke jobs in various +# configurations. The factor value will set the max number of resmoke jobs based on the number +# of CPUs a machine has. For example, if the factor is 0.5 and a machine has 8 CPUs, the max resmoke +# jobs would be 4 (8 * 0.5). If the running task has multiple overrides that apply, the lowest +# value will be used. +# +# The task name is specified as a regex. The task name used will be the task executing the test, +# which means if the task has been split to run in sub-tasks, an extra "_0", "_1", ... will be +# appended to the task name. For this reason, most task names should end with a ".*". + +# Apply factor for a task based on the build variant it is running on. VARIANT_TASK_FACTOR_OVERRIDES = { "enterprise-rhel-62-64-bit": [{"task": r"logical_session_cache_replication.*", "factor": 0.75}], "enterprise-rhel-62-64-bit-inmem": [{"task": "secondary_reads_passthrough", "factor": 0.3}] @@ -20,10 +31,33 @@ VARIANT_TASK_FACTOR_OVERRIDES = { TASKS_FACTORS = [{"task": r"replica_sets.*", "factor": 0.5}, {"task": r"sharding.*", "factor": 0.5}] +# Apply factor for a task based on the machine type it is running on. MACHINE_TASK_FACTOR_OVERRIDES = {"aarch64": TASKS_FACTORS} +# Apply factor for a task based on the platform it is running on. PLATFORM_TASK_FACTOR_OVERRIDES = {"win32": TASKS_FACTORS, "cygwin": TASKS_FACTORS} +# Apply factor for a task everywhere it is run. +GLOBAL_TASK_FACTOR_OVERRIDES = { + r"multi_shard_.*multi_stmt_txn_.*jscore_passthrough.*": 0.125, +} + + +def global_task_factor(task_name, overrides, factor): + """ + Check for a global task override and return factor. + + :param task_name: Name of task to check for. + :param overrides: Global override data. + :param factor: Default factor if there is no override. + :return: Factor that should be used based on global overrides. + """ + for task_re, task_factor in overrides.items(): + if re.compile(task_re).match(task_name): + return task_factor + + return factor + def get_task_factor(task_name, overrides, override_type, factor): """Check for task override and return factor.""" @@ -35,11 +69,12 @@ def get_task_factor(task_name, overrides, override_type, factor): def determine_factor(task_name, variant, factor): """Determine the job factor.""" - factors = [] - factors.append( - get_task_factor(task_name, MACHINE_TASK_FACTOR_OVERRIDES, PLATFORM_MACHINE, factor)) - factors.append(get_task_factor(task_name, PLATFORM_TASK_FACTOR_OVERRIDES, SYS_PLATFORM, factor)) - factors.append(get_task_factor(task_name, VARIANT_TASK_FACTOR_OVERRIDES, variant, factor)) + factors = [ + get_task_factor(task_name, MACHINE_TASK_FACTOR_OVERRIDES, PLATFORM_MACHINE, factor), + get_task_factor(task_name, PLATFORM_TASK_FACTOR_OVERRIDES, SYS_PLATFORM, factor), + get_task_factor(task_name, VARIANT_TASK_FACTOR_OVERRIDES, variant, factor), + global_task_factor(task_name, GLOBAL_TASK_FACTOR_OVERRIDES, factor), + ] return min(factors) -- cgit v1.2.1