SERVER-29888 Use ciconfig.evergreen module in scripts

2 files changed, 33 insertions, 80 deletions

diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py
index 5623f5c2c23..a7593cfbcfe 100644
--- a/buildscripts/burn_in_tests.py
+++ b/buildscripts/burn_in_tests.py
@@ -19,12 +19,15 @@ import sys
 import urlparse
 import yaml
 
-API_SERVER_DEFAULT = "http://evergreen-api.mongodb.com:8080"
 
 # Get relative imports to work when the package is not installed on the PYTHONPATH.
 if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from buildscripts import resmokelib
+from buildscripts.ciconfig import evergreen
+
+
+API_SERVER_DEFAULT = "http://evergreen-api.mongodb.com:8080"
 
 
 def parse_command_line():
@@ -263,69 +266,37 @@ def create_executor_list(suites, exclude_suites):
     return memberships
 
 
-def create_buildvariant_list(evergreen_file):
-    """
-    Parses etc/evergreen.yml. Returns a list of buildvariants.
+def create_task_list(evergreen_conf, buildvariant, suites, exclude_tasks):
     """
+    Finds associated tasks for the specified buildvariant and suites.
 
-    with open(evergreen_file, "r") as fstream:
-        evg = yaml.load(fstream)
-
-    return [li["name"] for li in evg["buildvariants"]]
-
-
-def get_resmoke_args(evg_task):
-    """
-    Returns the resmoke_args from a task in evergreen.
-    """
-    for command in evg_task["commands"]:
-        if ("func" in command and command["func"] == "run tests" and
-                "vars" in command and "resmoke_args" in command["vars"]):
-            return command["vars"]["resmoke_args"]
-    return None
-
-
-def create_task_list(evergreen_file, buildvariant, suites, exclude_tasks):
-    """
-    Parses etc/evergreen.yml to find associated tasks for the specified buildvariant
-    and suites. Returns a dict keyed by task_name, with executor, resmoke_args & tests, i.e.,
+    Returns a dict keyed by task_name, with executor, resmoke_args & tests, i.e.,
     {'jsCore_small_oplog':
         {'resmoke_args': '--suites=core_small_oplog --storageEngine=mmapv1',
          'tests': ['jstests/core/all2.js', 'jstests/core/all3.js']}
     }
     """
 
-    # Check if buildvariant is in the evergreen_file.
-    buildvariants = create_buildvariant_list(evergreen_file)
-    if buildvariant not in buildvariants:
-        print "Buildvariant", buildvariant, "not found in", evergreen_file
+    evg_buildvariant = evergreen_conf.get_variant(buildvariant)
+    if not evg_buildvariant:
+        print "Buildvariant", buildvariant, "not found in", evergreen_conf.path
         sys.exit(1)
 
-    with open(evergreen_file, "r") as fstream:
-        evg = yaml.load(fstream)
-
-    evg_buildvariant = next(item for item in evg["buildvariants"] if item["name"] == buildvariant)
-
-    # Find all the task names for the specified buildvariant.
-    variant_tasks = [li["name"] for li in evg_buildvariant['tasks']]
-
     # Find all the buildvariant task's resmoke_args.
     variant_task_args = {}
-    for task in [a for a in evg["tasks"] if a["name"] in set(variant_tasks) - set(exclude_tasks)]:
-        variant_task_args[task["name"]] = get_resmoke_args(task)
-
-    # Find if the buildvariant has a test_flags expansion, which will be passed onto resmoke.py.
-    test_flags = evg_buildvariant.get("expansions", {}).get("test_flags", "")
+    exclude_tasks_set = set(exclude_tasks)
+    for task in evg_buildvariant.tasks:
+        if task.name not in exclude_tasks_set:
+            # Using 'task.combined_resmoke_args' to include the variant's test_flags and
+            # allow the storage engine to be overridden.
+            variant_task_args[task.name] = task.combined_resmoke_args
 
     # Create the list of tasks to run for the specified suite.
     tasks_to_run = {}
     for suite in suites.keys():
         for task_name, task_arg in variant_task_args.items():
-            # Append the test_flags to the task arguments to match the logic in the "run tests"
-            # function. This allows the storage engine to be overridden.
-            task_arg = "{} {}".format(task_arg, test_flags)
             # Find the resmoke_args for matching suite names.
-            if (re.compile('--suites=' + suite + '(?:\s+|$)').match(task_arg)):
+            if re.compile('--suites=' + suite + '(?:\s+|$)').match(task_arg):
                 tasks_to_run[task_name] = {
                     "resmoke_args": task_arg,
                     "tests": suites[suite]
@@ -389,10 +360,13 @@ def main():
 
     # Run the executor finder.
     else:
+        # Parse the Evergreen project configuration file.
+        evergreen_conf = evergreen.EvergreenProjectConfig(values.evergreen_file)
+
         if values.buildvariant is None:
             print "Option buildVariant must be specified to find changed tests.\n", \
                   "Select from the following: \n" \
-                  "\t", "\n\t".join(sorted(create_buildvariant_list(values.evergreen_file)))
+                  "\t", "\n\t".join(sorted(evergreen_conf.variant_names))
             sys.exit(1)
 
         changed_tests = find_changed_tests(values.branch,
@@ -409,7 +383,7 @@ def main():
             sys.exit(0)
         suites = resmokelib.parser.get_suites(values, changed_tests)
         tests_by_executor = create_executor_list(suites, exclude_suites)
-        tests_by_task = create_task_list(values.evergreen_file,
+        tests_by_task = create_task_list(evergreen_conf,
                                          values.buildvariant,
                                          tests_by_executor,
                                          exclude_tasks)
diff --git a/buildscripts/update_test_lifecycle.py b/buildscripts/update_test_lifecycle.py
index 4b1b030b908..91475e91326 100755
--- a/buildscripts/update_test_lifecycle.py
+++ b/buildscripts/update_test_lifecycle.py
@@ -9,10 +9,8 @@ from __future__ import print_function
 
 import collections
 import copy
-import fnmatch
 import optparse
 import os
-import re
 import subprocess
 import sys
 import textwrap
@@ -22,8 +20,8 @@ import yaml
 # Get relative imports to work when the package is not installed on the PYTHONPATH.
 if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from buildscripts import burn_in_tests
 from buildscripts import resmokelib
+from buildscripts.ciconfig import evergreen
 from buildscripts import test_failures as tf
 
 
@@ -49,23 +47,20 @@ def write_yaml_file(yaml_file, object):
         yaml.dump(object, fstream, default_flow_style=False)
 
 
-def get_suite_tasks_membership(evg_yaml):
+def get_suite_tasks_membership(evg_conf):
     """Return a dictionary with keys of all suites and list of associated tasks."""
-    evg = read_yaml_file(evg_yaml)
     suite_membership = collections.defaultdict(list)
-    for task in evg["tasks"]:
-        resmoke_args = burn_in_tests.get_resmoke_args(task)
-        if resmoke_args:
-            m = re.search("--suites=(?P<suite>\w+)", resmoke_args)
-            if m is not None:
-                suite_membership[m.group("suite")].append(task["name"])
+    for task in evg_conf.tasks:
+        suite = task.resmoke_suite
+        if suite:
+            suite_membership[suite].append(task.name)
     return suite_membership
 
 
-def get_test_tasks_membership(evg_yaml):
+def get_test_tasks_membership(evg_conf):
     """Return a dictionary with keys of all tests and list of associated tasks."""
     test_suites_membership = resmokelib.parser.create_test_membership_map(test_kind="js_test")
-    suite_tasks_membership = get_suite_tasks_membership(evg_yaml)
+    suite_tasks_membership = get_suite_tasks_membership(evg_conf)
     test_tasks_membership = collections.defaultdict(list)
     for test in test_suites_membership.keys():
         for suite in test_suites_membership[test]:
@@ -107,23 +102,6 @@ def create_batch_groups(test_groups, batch_size):
     return batch_groups
 
 
-def get_all_tasks(evg_yaml):
-    """Returns list of tasks from evg_yaml.
-
-       Note that tasks can be excluded in 'test_lifecycle_excluded_tasks'.
-    """
-    evg = read_yaml_file(evg_yaml)
-    all_tasks = [t["name"] for t in evg["tasks"]]
-    # The list of excluded tasks may include "Unix shell-style wildcards",
-    # i.e., 'compile*', which matches 'compile', 'compile_all'
-    excluded_glob_tasks = evg.get("test_lifecycle_excluded_tasks", [])
-    excluded_tasks = []
-    for excluded_glob_task in excluded_glob_tasks:
-        excluded_tasks.extend(fnmatch.filter(all_tasks, excluded_glob_task))
-
-    return list(set(all_tasks) - set(excluded_tasks))
-
-
 def callo(args):
     """Call a program, and capture its output."""
     return subprocess.check_output(args)
@@ -346,12 +324,13 @@ def main():
             parser.print_help()
             parser.error("Missing required option")
 
+    evg_conf = evergreen.EvergreenProjectConfig(options.evergreen_yml)
     use_test_tasks_membership = False
 
     tasks = options.tasks.split(",") if options.tasks else []
     if not tasks:
         # If no tasks are specified, then the list of tasks is all.
-        tasks = get_all_tasks(options.evergreen_yml)
+        tasks = evg_conf.lifecycle_task_names
         use_test_tasks_membership = True
 
     variants = options.variants.split(",") if options.variants else []
@@ -379,7 +358,7 @@ def main():
     orig_lifecycle = read_yaml_file(options.lifecycle_file)
     lifecycle = copy.deepcopy(orig_lifecycle)
 
-    test_tasks_membership = get_test_tasks_membership(options.evergreen_yml)
+    test_tasks_membership = get_test_tasks_membership(evg_conf)
     # If no tests are specified then the list of tests is generated from the list of tasks.
     if not tests:
         tests = get_tests_from_tasks(tasks, test_tasks_membership)