SERVER-33497 remove support for options file in resmoke.py

2 files changed, 95 insertions, 163 deletions

diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index 99841a84c8e..29398c2bfcf 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -35,51 +35,52 @@ MONGO_RUNNER_SUBDIR = "mongorunner"
 
 # Names below correspond to how they are specified via the command line or in the options YAML file.
 DEFAULTS = {
-    "archiveFile": None,
-    "archiveLimitMb": 5000,
-    "archiveLimitTests": 10,
-    "basePort": 20000,
-    "buildloggerUrl": "https://logkeeper.mongodb.org",
-    "continueOnFailure": False,
-    "dbpathPrefix": None,
-    "dbtest": None,
-    "distroId": None,
-    "dryRun": None,
-    "excludeWithAnyTags": None,
-    "executionNumber": 0,
-    "gitRevision": None,
-    "includeWithAnyTags": None,
+    "archive_file": None,
+    "archive_limit_mb": 5000,
+    "archive_limit_tests": 10,
+    "base_port": 20000,
+    "buildlogger_url": "https://logkeeper.mongodb.org",
+    "continue_on_failure": False,
+    "dbpath_prefix": None,
+    "dbtest_executable": None,
+    "distro_id": None,
+    "dry_run": None,
+    "exclude_with_any_tags": None,
+    "execution_number": 0,
+    "git_revision": None,
+    "include_with_any_tags": None,
     "jobs": 1,
-    "mongo": None,
-    "mongod": None,
-    "mongodSetParameters": None,
-    "mongos": None,
-    "mongosSetParameters": None,
-    "nojournal": False,
-    "numClientsPerFixture": 1,
-    "shellPort": None,
-    "shellConnString": None,
-    "patchBuild": False,
-    "projectName": "mongodb-mongo-master",
+    "mongo_executable": None,
+    "mongod_executable": None,
+    "mongod_set_parameters": None,
+    "mongos_executable": None,
+    "mongos_set_parameters": None,
+    "no_journal": False,
+    "num_clients_per_fixture": 1,
+    "patch_build": False,
+    "prealloc_journal": None,  # Default is set on the commandline.
+    "project_name": "mongodb-mongo-master",
     "repeat": 1,
-    "reportFailureStatus": "fail",
-    "reportFile": None,
+    "report_failure_status": "fail",
+    "report_file": None,
     "seed": long(time.time() * 256),  # Taken from random.py code in Python 2.7.
-    "serviceExecutor": None,
-    "shellReadMode": None,
-    "shellWriteMode": None,
+    "service_executor": None,
+    "shell_conn_string": None,
+    "shell_port": None,
+    "shell_read_mode": None,
+    "shell_write_mode": None,
     "shuffle": None,
-    "staggerJobs": None,
-    "storageEngine": None,
-    "storageEngineCacheSizeGB": None,
-    "tagFile": None,
-    "taskId": None,
-    "taskName": None,
-    "transportLayer": None,
-    "variantName": None,
-    "wiredTigerCollectionConfigString": None,
-    "wiredTigerEngineConfigString": None,
-    "wiredTigerIndexConfigString": None
+    "stagger_jobs": None,
+    "storage_engine": None,
+    "storage_engine_cache_size_gb": None,
+    "tag_file": None,
+    "task_id": None,
+    "task_name": None,
+    "transport_layer": None,
+    "variant_name": None,
+    "wt_coll_config": None,
+    "wt_engine_config": None,
+    "wt_index_config": None
 }
 
 
diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py
index b51dc239f35..7f1fe051053 100644
--- a/buildscripts/resmokelib/parser.py
+++ b/buildscripts/resmokelib/parser.py
@@ -13,58 +13,6 @@ from . import utils
 from .. import resmokeconfig
 
 
-# Mapping of the attribute of the parsed arguments (dest) to its key as it appears in the options
-# YAML configuration file. Most should only be converting from snake_case to camelCase.
-DEST_TO_CONFIG = {
-    "archive_file": "archiveFile",
-    "archive_limit_mb": "archiveLimitMb",
-    "archive_limit_tests": "archiveLimitTests",
-    "base_port": "basePort",
-    "buildlogger_url": "buildloggerUrl",
-    "continue_on_failure": "continueOnFailure",
-    "dbpath_prefix": "dbpathPrefix",
-    "dbtest_executable": "dbtest",
-    "distro_id": "distroId",
-    "dry_run": "dryRun",
-    "exclude_with_any_tags": "excludeWithAnyTags",
-    "execution_number": "executionNumber",
-    "git_revision": "gitRevision",
-    "include_with_any_tags": "includeWithAnyTags",
-    "jobs": "jobs",
-    "mongo_executable": "mongo",
-    "mongod_executable": "mongod",
-    "mongod_parameters": "mongodSetParameters",
-    "mongos_executable": "mongos",
-    "mongos_parameters": "mongosSetParameters",
-    "no_journal": "nojournal",
-    "num_clients_per_fixture": "numClientsPerFixture",
-    "patch_build": "patchBuild",
-    "prealloc_journal": "preallocJournal",
-    "project_name": "projectName",
-    "repeat": "repeat",
-    "report_failure_status": "reportFailureStatus",
-    "report_file": "reportFile",
-    "seed": "seed",
-    "service_executor": "serviceExecutor",
-    "shell_conn_string": "shellConnString",
-    "shell_port": "shellPort",
-    "shell_read_mode": "shellReadMode",
-    "shell_write_mode": "shellWriteMode",
-    "shuffle": "shuffle",
-    "stagger_jobs": "staggerJobs",
-    "storage_engine": "storageEngine",
-    "storage_engine_cache_size": "storageEngineCacheSizeGB",
-    "tag_file": "tagFile",
-    "task_id": "taskId",
-    "task_name": "taskName",
-    "transport_layer": "transportLayer",
-    "variant_name": "variantName",
-    "wt_coll_config": "wiredTigerCollectionConfigString",
-    "wt_engine_config": "wiredTigerEngineConfigString",
-    "wt_index_config": "wiredTigerIndexConfigString"
-}
-
-
 def parse_command_line():
     """
     Parses the command line arguments passed to resmoke.py.
@@ -85,9 +33,6 @@ def parse_command_line():
                             " located in the resmokeconfig/suites/ directory, then the basename"
                             " without the .yml extension can be specified, e.g. 'console'."))
 
-    parser.add_option("--options", dest="options_file", metavar="OPTIONS",
-                      help="A YAML file that specifies global options to resmoke.py.")
-
     parser.add_option("--archiveFile", dest="archive_file", metavar="ARCHIVE_FILE",
                       help=("Sets the archive file name for the Evergreen task running the tests."
                             " The archive file is JSON format containing a list of tests that were"
@@ -159,7 +104,7 @@ def parse_command_line():
     parser.add_option("--mongod", dest="mongod_executable", metavar="PATH",
                       help="The path to the mongod executable for resmoke.py to use.")
 
-    parser.add_option("--mongodSetParameters", dest="mongod_parameters",
+    parser.add_option("--mongodSetParameters", dest="mongod_set_parameters",
                       metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
                       help=("Passes one or more --setParameter options to all mongod processes"
                             " started by resmoke.py. The argument is specified as bracketed YAML -"
@@ -168,7 +113,7 @@ def parse_command_line():
     parser.add_option("--mongos", dest="mongos_executable", metavar="PATH",
                       help="The path to the mongos executable for resmoke.py to use.")
 
-    parser.add_option("--mongosSetParameters", dest="mongos_parameters",
+    parser.add_option("--mongosSetParameters", dest="mongos_set_parameters",
                       metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
                       help=("Passes one or more --setParameter options to all mongos processes"
                             " started by resmoke.py. The argument is specified as bracketed YAML -"
@@ -249,7 +194,7 @@ def parse_command_line():
     parser.add_option("--storageEngine", dest="storage_engine", metavar="ENGINE",
                       help="The storage engine used by dbtests and jstests.")
 
-    parser.add_option("--storageEngineCacheSizeGB", dest="storage_engine_cache_size",
+    parser.add_option("--storageEngineCacheSizeGB", dest="storage_engine_cache_size_gb",
                       metavar="CONFIG", help="Sets the storage engine cache size configuration"
                       " setting for all mongod's.")
 
@@ -342,62 +287,60 @@ def get_logging_config(values):
 
 
 def update_config_vars(values):
-    options = _get_options_config(values.options_file)
-
     config = _config.DEFAULTS.copy()
-    config.update(options)
 
-    values = vars(values)
-    for dest in values:
-        if dest not in DEST_TO_CONFIG:
+    # Override `config` with values from command line arguments.
+    cmdline_vars = vars(values)
+    for cmdline_key in cmdline_vars:
+        if cmdline_key not in _config.DEFAULTS:
+            # Ignore options that don't map to values in config.py
             continue
-        config_var = DEST_TO_CONFIG[dest]
-        if values[dest] is not None:
-            config[config_var] = values[dest]
-
-    _config.ARCHIVE_FILE = config.pop("archiveFile")
-    _config.ARCHIVE_LIMIT_MB = config.pop("archiveLimitMb")
-    _config.ARCHIVE_LIMIT_TESTS = config.pop("archiveLimitTests")
-    _config.BASE_PORT = int(config.pop("basePort"))
-    _config.BUILDLOGGER_URL = config.pop("buildloggerUrl")
-    _config.DBPATH_PREFIX = _expand_user(config.pop("dbpathPrefix"))
-    _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest"))
-    _config.DRY_RUN = config.pop("dryRun")
-    _config.EVERGREEN_DISTRO_ID = config.pop("distroId")
-    _config.EVERGREEN_EXECUTION = config.pop("executionNumber")
-    _config.EVERGREEN_PATCH_BUILD = config.pop("patchBuild")
-    _config.EVERGREEN_PROJECT_NAME = config.pop("projectName")
-    _config.EVERGREEN_REVISION = config.pop("gitRevision")
-    _config.EVERGREEN_TASK_ID = config.pop("taskId")
-    _config.EVERGREEN_TASK_NAME = config.pop("taskName")
-    _config.EVERGREEN_VARIANT_NAME = config.pop("variantName")
-    _config.EXCLUDE_WITH_ANY_TAGS = _tags_from_list(config.pop("excludeWithAnyTags"))
-    _config.FAIL_FAST = not config.pop("continueOnFailure")
-    _config.INCLUDE_WITH_ANY_TAGS = _tags_from_list(config.pop("includeWithAnyTags"))
+        if cmdline_vars[cmdline_key] is not None:
+            config[cmdline_key] = cmdline_vars[cmdline_key]
+
+    _config.ARCHIVE_FILE = config.pop("archive_file")
+    _config.ARCHIVE_LIMIT_MB = config.pop("archive_limit_mb")
+    _config.ARCHIVE_LIMIT_TESTS = config.pop("archive_limit_tests")
+    _config.BASE_PORT = int(config.pop("base_port"))
+    _config.BUILDLOGGER_URL = config.pop("buildlogger_url")
+    _config.DBPATH_PREFIX = _expand_user(config.pop("dbpath_prefix"))
+    _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest_executable"))
+    _config.DRY_RUN = config.pop("dry_run")
+    _config.EVERGREEN_DISTRO_ID = config.pop("distro_id")
+    _config.EVERGREEN_EXECUTION = config.pop("execution_number")
+    _config.EVERGREEN_PATCH_BUILD = config.pop("patch_build")
+    _config.EVERGREEN_PROJECT_NAME = config.pop("project_name")
+    _config.EVERGREEN_REVISION = config.pop("git_revision")
+    _config.EVERGREEN_TASK_ID = config.pop("task_id")
+    _config.EVERGREEN_TASK_NAME = config.pop("task_name")
+    _config.EVERGREEN_VARIANT_NAME = config.pop("variant_name")
+    _config.EXCLUDE_WITH_ANY_TAGS = _tags_from_list(config.pop("exclude_with_any_tags"))
+    _config.FAIL_FAST = not config.pop("continue_on_failure")
+    _config.INCLUDE_WITH_ANY_TAGS = _tags_from_list(config.pop("include_with_any_tags"))
     _config.JOBS = config.pop("jobs")
-    _config.MONGO_EXECUTABLE = _expand_user(config.pop("mongo"))
-    _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod"))
-    _config.MONGOD_SET_PARAMETERS = config.pop("mongodSetParameters")
-    _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos"))
-    _config.MONGOS_SET_PARAMETERS = config.pop("mongosSetParameters")
-    _config.NO_JOURNAL = config.pop("nojournal")
-    _config.NO_PREALLOC_JOURNAL = config.pop("preallocJournal") == "off"
-    _config.NUM_CLIENTS_PER_FIXTURE = config.pop("numClientsPerFixture")
+    _config.MONGO_EXECUTABLE = _expand_user(config.pop("mongo_executable"))
+    _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable"))
+    _config.MONGOD_SET_PARAMETERS = config.pop("mongod_set_parameters")
+    _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable"))
+    _config.MONGOS_SET_PARAMETERS = config.pop("mongos_set_parameters")
+    _config.NO_JOURNAL = config.pop("no_journal")
+    _config.NO_PREALLOC_JOURNAL = config.pop("prealloc_journal") == "off"
+    _config.NUM_CLIENTS_PER_FIXTURE = config.pop("num_clients_per_fixture")
     _config.RANDOM_SEED = config.pop("seed")
     _config.REPEAT = config.pop("repeat")
-    _config.REPORT_FAILURE_STATUS = config.pop("reportFailureStatus")
-    _config.REPORT_FILE = config.pop("reportFile")
-    _config.SERVICE_EXECUTOR = config.pop("serviceExecutor")
-    _config.SHELL_READ_MODE = config.pop("shellReadMode")
-    _config.SHELL_WRITE_MODE = config.pop("shellWriteMode")
-    _config.STAGGER_JOBS = config.pop("staggerJobs") == "on"
-    _config.STORAGE_ENGINE = config.pop("storageEngine")
-    _config.STORAGE_ENGINE_CACHE_SIZE = config.pop("storageEngineCacheSizeGB")
-    _config.TAG_FILE = config.pop("tagFile")
-    _config.TRANSPORT_LAYER = config.pop("transportLayer")
-    _config.WT_COLL_CONFIG = config.pop("wiredTigerCollectionConfigString")
-    _config.WT_ENGINE_CONFIG = config.pop("wiredTigerEngineConfigString")
-    _config.WT_INDEX_CONFIG = config.pop("wiredTigerIndexConfigString")
+    _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status")
+    _config.REPORT_FILE = config.pop("report_file")
+    _config.SERVICE_EXECUTOR = config.pop("service_executor")
+    _config.SHELL_READ_MODE = config.pop("shell_read_mode")
+    _config.SHELL_WRITE_MODE = config.pop("shell_write_mode")
+    _config.STAGGER_JOBS = config.pop("stagger_jobs") == "on"
+    _config.STORAGE_ENGINE = config.pop("storage_engine")
+    _config.STORAGE_ENGINE_CACHE_SIZE = config.pop("storage_engine_cache_size_gb")
+    _config.TAG_FILE = config.pop("tag_file")
+    _config.TRANSPORT_LAYER = config.pop("transport_layer")
+    _config.WT_COLL_CONFIG = config.pop("wt_coll_config")
+    _config.WT_ENGINE_CONFIG = config.pop("wt_engine_config")
+    _config.WT_INDEX_CONFIG = config.pop("wt_index_config")
 
     shuffle = config.pop("shuffle")
     if shuffle == "auto":
@@ -408,8 +351,8 @@ def update_config_vars(values):
     else:
         _config.SHUFFLE = shuffle == "on"
 
-    conn_string = config.pop("shellConnString")
-    port = config.pop("shellPort")
+    conn_string = config.pop("shell_conn_string")
+    port = config.pop("shell_port")
 
     if port is not None:
         conn_string = "mongodb://localhost:" + port
@@ -439,18 +382,6 @@ def _get_logging_config(pathname):
     return utils.load_yaml_file(pathname).pop("logging")
 
 
-def _get_options_config(pathname):
-    """
-    Attempts to read a YAML configuration from 'pathname' that describes
-    any modifications to global options.
-    """
-
-    if pathname is None:
-        return {}
-
-    return utils.load_yaml_file(pathname).pop("options")
-
-
 def _expand_user(pathname):
     """
     Wrapper around os.path.expanduser() to do nothing when given None.