diff options
| -rw-r--r-- | buildscripts/burn_in_tests.py | 2 | ||||
| -rw-r--r-- | buildscripts/ciconfig/evergreen.py | 36 | ||||
| -rw-r--r-- | buildscripts/tests/ciconfig/test_evergreen.py | 6 | ||||
| -rwxr-xr-x | buildscripts/update_test_lifecycle.py | 2 | ||||
| -rw-r--r-- | buildscripts/util/runcommand.py | 68 | ||||
| -rw-r--r-- | etc/evergreen.yml | 8 |
6 files changed, 109 insertions, 13 deletions
diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index a1875c09995..11a51fd5afb 100644 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -351,7 +351,7 @@ def main(): # Run the executor finder. else: # Parse the Evergreen project configuration file. - evergreen_conf = evergreen.EvergreenProjectConfig(values.evergreen_file) + evergreen_conf = evergreen.parse_evergreen_file(values.evergreen_file) if values.buildvariant is None: print "Option buildVariant must be specified to find changed tests.\n", \ diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py index 9d6dfa23efa..e7ab5654187 100644 --- a/buildscripts/ciconfig/evergreen.py +++ b/buildscripts/ciconfig/evergreen.py @@ -4,21 +4,47 @@ The API also provides methods to access specific fields present in the mongodb/m configuration file. """ +from __future__ import print_function + import datetime +import distutils.spawn # pylint: disable=no-name-in-module import fnmatch +import os import re import yaml +import buildscripts.util.runcommand as runcommand + + +def parse_evergreen_file(path, evergreen_binary="evergreen"): + """Read an Evergreen file and return EvergreenProjectConfig instance.""" + if evergreen_binary: + if not distutils.spawn.find_executable(evergreen_binary): + raise EnvironmentError( + "Executable '{}' does not exist or is not in the PATH.".format(evergreen_binary)) + + # Call 'evergreen evaluate path' to pre-process the project configuration file. + cmd = runcommand.RunCommand(evergreen_binary) + cmd.add("evaluate") + cmd.add_file(path) + error_code, output = cmd.execute() + if error_code: + raise RuntimeError("Unable to evaluate {}: {}".format(path, output)) + config = yaml.load(output) + else: + with open(path, "r") as fstream: + config = yaml.load(fstream) + + return EvergreenProjectConfig(config) + class EvergreenProjectConfig(object): # pylint: disable=too-many-instance-attributes """Represent an Evergreen project configuration file.""" - def __init__(self, path): - """Initialize the EvergreenProjectConfig from a file path name.""" - with open(path, "r") as fstream: - self._conf = yaml.load(fstream) - self.path = path + def __init__(self, conf): + """Initialize the EvergreenProjectConfig from a YML dictionary.""" + self._conf = conf self.tasks = [Task(task_dict) for task_dict in self._conf["tasks"]] self._tasks_by_name = {task.name: task for task in self.tasks} self.task_groups = [ diff --git a/buildscripts/tests/ciconfig/test_evergreen.py b/buildscripts/tests/ciconfig/test_evergreen.py index f1b6740d539..9af34485713 100644 --- a/buildscripts/tests/ciconfig/test_evergreen.py +++ b/buildscripts/tests/ciconfig/test_evergreen.py @@ -18,12 +18,12 @@ class TestEvergreenProjectConfig(unittest.TestCase): @classmethod def setUpClass(cls): - cls.conf = _evergreen.EvergreenProjectConfig(TEST_FILE_PATH) + cls.conf = _evergreen.parse_evergreen_file(TEST_FILE_PATH, evergreen_binary=None) def test_invalid_path(self): invalid_path = "non_existing_file" with self.assertRaises(IOError): - _evergreen.EvergreenProjectConfig(invalid_path) + _evergreen.parse_evergreen_file(invalid_path, evergreen_binary=None) def test_list_tasks(self): self.assertEqual(6, len(self.conf.tasks)) @@ -135,7 +135,7 @@ class TestVariant(unittest.TestCase): @classmethod def setUpClass(cls): - cls.conf = _evergreen.EvergreenProjectConfig(TEST_FILE_PATH) + cls.conf = _evergreen.parse_evergreen_file(TEST_FILE_PATH, evergreen_binary=None) def test_from_dict(self): task = _evergreen.Task({"name": "compile"}) diff --git a/buildscripts/update_test_lifecycle.py b/buildscripts/update_test_lifecycle.py index 9e06d42a086..f813e18412e 100755 --- a/buildscripts/update_test_lifecycle.py +++ b/buildscripts/update_test_lifecycle.py @@ -1051,7 +1051,7 @@ def main(): # pylint: disable=too-many-branches,too-many-locals,too-many-statem logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=options.log_level, filename=options.log_file) - evg_conf = ci_evergreen.EvergreenProjectConfig(options.evergreen_project_config) + evg_conf = ci_evergreen.parse_evergreen_file(options.evergreen_project_config) use_test_tasks_membership = False tasks = options.tasks.split(",") if options.tasks else [] diff --git a/buildscripts/util/runcommand.py b/buildscripts/util/runcommand.py new file mode 100644 index 00000000000..ff9e726cc3e --- /dev/null +++ b/buildscripts/util/runcommand.py @@ -0,0 +1,68 @@ +"""Utility to support running a command in a subprocess.""" + +from __future__ import print_function + +import os +import pipes +import shlex +import sys + +# The subprocess32 module is untested on Windows and thus isn't recommended for use, even when it's +# installed. See https://github.com/google/python-subprocess32/blob/3.2.7/README.md#usage. +if os.name == "posix" and sys.version_info[0] == 2: + try: + import subprocess32 as subprocess + except ImportError: + import warnings + warnings.warn(("Falling back to using the subprocess module because subprocess32 isn't" + " available. When using the subprocess module, a child process may" + " trigger an invalid free(). See SERVER-22219 for more details."), + RuntimeWarning) + import subprocess # type: ignore +else: + import subprocess + + +def execute_cmd(cmd): + """Execute specified 'cmd' and return err_code and output. + + If 'cmd' is specified as a string, convert it to a list of strings. + """ + if isinstance(cmd, str): + cmd = shlex.split(cmd) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output, _ = proc.communicate() + error_code = proc.returncode + return error_code, output + + +class RunCommand(object): + """Class to abstract executing a subprocess.""" + + def __init__(self, string=None): + """Initialize the RunCommand object.""" + self._command = string + + def add(self, string): + """Add a string to the command.""" + self._command = "{}{}{}".format(self._command, self._space(), string) + + def add_file(self, path): + """Add a file path to the command.""" + # For Windows compatability, use pipes.quote around file paths. + self._command = "{}{}{}".format(self._command, self._space(), pipes.quote(path)) + + def execute(self): + """Execute the command. Return (error_code, output).""" + return execute_cmd(self._command) + + @property + def command(self): + """Get the command.""" + return self._command + + def _space(self): + """Return a space if the command has been started to be built.""" + if self._command: + return " " + return "" diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 818c5eae640..af4bc8aa182 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -3540,13 +3540,14 @@ tasks: if [ -n "${burn_in_tests_build_variant|}" ]; then build_variant=${burn_in_tests_build_variant|} fi - $python buildscripts/burn_in_tests.py --branch=${branch_name} --buildVariant=$build_variant --testListOutfile=jstests/new_tests.json --noExec $burn_in_args + # Evergreen executable is in $HOME. + PATH=$PATH:$HOME $python buildscripts/burn_in_tests.py --branch=${branch_name} --buildVariant=$build_variant --testListOutfile=jstests/new_tests.json --noExec $burn_in_args # Copy the results to the src dir. cp jstests/new_tests.json ../src/jstests/new_tests.json - func: "do multiversion setup" - func: "run tests" vars: - task_path_suffix: /data/multiversion + task_path_suffix: /data/multiversion:$HOME resmoke_wrapper: $python buildscripts/burn_in_tests.py --testListFile=jstests/new_tests.json resmoke_args: --repeat=2 run_multiple_jobs: true @@ -6805,7 +6806,8 @@ tasks: # We use a small batch size to avoid hitting the load balancer timeout if the Evergreen # API query is not fast enough. - $python buildscripts/update_test_lifecycle.py \ + # Evergreen executable is in $HOME. + PATH=$PATH:$HOME $python buildscripts/update_test_lifecycle.py \ --project ${project} \ --requestBatchSize 20 \ --commit \ |