SERVER-51877 Check all API V1 commands are defined in IDL

6 files changed, 223 insertions, 25 deletions

diff --git a/buildscripts/idl/check_versioned_api_commands_have_idl_definitions.py b/buildscripts/idl/check_versioned_api_commands_have_idl_definitions.py
new file mode 100644
index 00000000000..0840dfadf8c
--- /dev/null
+++ b/buildscripts/idl/check_versioned_api_commands_have_idl_definitions.py
@@ -0,0 +1,184 @@
+# Copyright (C) 2020-present MongoDB, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+"""Check that mongod's and mongos's Versioned API commands are defined in IDL.
+
+Call listCommands on mongod and mongos to assert they have the same set of commands in the given API
+version, and assert all these commands are defined in IDL.
+"""
+
+import argparse
+import logging
+import os
+import sys
+from tempfile import TemporaryDirectory
+from typing import Dict, List, Set
+
+from pymongo import MongoClient
+
+# Permit imports from "buildscripts".
+sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), '../../..')))
+
+# pylint: disable=wrong-import-position
+from buildscripts.resmokelib import configure_resmoke
+from buildscripts.resmokelib.logging import loggers
+from buildscripts.resmokelib.testing.fixtures import interface
+from buildscripts.resmokelib.testing.fixtures.shardedcluster import ShardedClusterFixture
+from buildscripts.resmokelib.testing.fixtures.standalone import MongoDFixture
+from idl import parser, syntax
+from idl.compiler import CompilerImportResolver
+
+LOGGER_NAME = 'check-idl-definitions'
+LOGGER = logging.getLogger(LOGGER_NAME)
+
+
+def list_idls(directory: str) -> Set[str]:
+    """Find all IDL files in the current directory."""
+    return {
+        os.path.join(dirpath, filename)
+        for dirpath, dirnames, filenames in os.walk(directory) for filename in filenames
+        if filename.endswith('.idl')
+    }
+
+
+def parse_idl(idl_path: str, import_directories: List[str]) -> syntax.IDLParsedSpec:
+    """Parse an IDL file or throw an error."""
+    parsed_doc = parser.parse(open(idl_path), idl_path, CompilerImportResolver(import_directories))
+
+    if parsed_doc.errors:
+        parsed_doc.errors.dump_errors()
+        raise ValueError(f"Cannot parse {idl_path}")
+
+    return parsed_doc
+
+
+def get_command_definitions(api_version: str, directory: str,
+                            import_directories: List[str]) -> Dict[str, syntax.Command]:
+    """Get parsed IDL definitions of commands in a given API version."""
+
+    LOGGER.info("Searching for command definitions in %s", directory)
+
+    def gen():
+        for idl_path in sorted(list_idls(directory)):
+            for command in parse_idl(idl_path, import_directories).spec.symbols.commands:
+                if command.api_version == api_version:
+                    yield command.name, command
+
+    idl_commands = dict(gen())
+    LOGGER.debug("Found %s IDL commands in API Version %s", len(idl_commands), api_version)
+    return idl_commands
+
+
+def list_commands_for_api(api_version: str, mongod_or_mongos: str, install_dir: str) -> Set[str]:
+    """Get a list of commands in a given API version by calling listCommands."""
+    assert mongod_or_mongos in ("mongod", "mongos")
+    logging.info("Calling listCommands on %s", mongod_or_mongos)
+    dbpath = TemporaryDirectory()
+    mongod_executable = os.path.join(install_dir, "mongod")
+    mongos_executable = os.path.join(install_dir, "mongos")
+    if mongod_or_mongos == "mongod":
+        logger = loggers.new_fixture_logger("MongoDFixture", 0)
+        logger.parent = LOGGER
+        fixture: interface.Fixture = MongoDFixture(logger, 0, dbpath_prefix=dbpath.name,
+                                                   mongod_executable=mongod_executable)
+    else:
+        logger = loggers.new_fixture_logger("ShardedClusterFixture", 0)
+        logger.parent = LOGGER
+        fixture = ShardedClusterFixture(logger, 0, dbpath_prefix=dbpath.name,
+                                        mongos_executable=mongos_executable,
+                                        mongod_executable=mongod_executable, mongod_options={})
+
+    fixture.setup()
+    fixture.await_ready()
+
+    try:
+        client = MongoClient(fixture.get_driver_connection_url())
+        reply = client.admin.command('listCommands')
+        commands = {
+            name
+            for name, info in reply['commands'].items() if api_version in info['apiVersions']
+        }
+        logging.info("Found %s commands in API Version %s on %s", len(commands), api_version,
+                     mongod_or_mongos)
+        return commands
+    finally:
+        fixture.teardown()
+
+
+def assert_command_sets_equal(api_version: str, command_sets: Dict[str, Set[str]]):
+    """Check that all sources have the same set of commands for a given API version."""
+    LOGGER.info("Comparing %s command sets", len(command_sets))
+    for name, commands in command_sets.items():
+        LOGGER.info("--------- %s API Version %s commands --------------", name, api_version)
+        for command in sorted(commands):
+            LOGGER.info("%s", command)
+
+    LOGGER.info("--------------------------------------------")
+    it = iter(command_sets.items())
+    name, commands = next(it)
+    for other_name, other_commands in it:
+        if commands != other_commands:
+            if commands - other_commands:
+                LOGGER.error("%s has commands not in %s: %s", name, other_name,
+                             commands - other_commands)
+            if other_commands - commands:
+                LOGGER.error("%s has commands not in %s: %s", other_name, name,
+                             other_commands - commands)
+            # TODO(SERVER-51878): Enable this assertion.
+            # raise AssertionError(
+            #     f"{name} and {other_name} have different commands in API Version {api_version}")
+
+
+def main():
+    """Run the script."""
+    arg_parser = argparse.ArgumentParser(description=__doc__)
+    arg_parser.add_argument("--include", type=str, action="append",
+                            help="Directory to search for IDL import files")
+    arg_parser.add_argument("--installDir", dest="install_dir", metavar="INSTALL_DIR",
+                            help="Directory to search for MongoDB binaries")
+    arg_parser.add_argument("-v", "--verbose", action="count", help="Enable verbose logging")
+    arg_parser.add_argument("api_version", metavar="API_VERSION", help="API Version to check")
+    args = arg_parser.parse_args()
+
+    # pylint: disable=protected-access
+    configure_resmoke._update_config_vars(object)
+    configure_resmoke._set_logging_config()
+
+    # Configure Fixture logging.
+    loggers.configure_loggers()
+    loggers.new_job_logger(sys.argv[0], 0)
+    logging.basicConfig(level=logging.WARNING)
+    logging.getLogger(LOGGER_NAME).setLevel(logging.DEBUG if args.verbose else logging.INFO)
+
+    command_sets = {}
+    command_sets["mongod"] = list_commands_for_api(args.api_version, "mongod", args.install_dir)
+    command_sets["mongos"] = list_commands_for_api(args.api_version, "mongos", args.install_dir)
+    command_sets["idl"] = set(get_command_definitions(args.api_version, os.getcwd(), args.include))
+    assert_command_sets_equal(args.api_version, command_sets)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index 268d9f735cf..5d3eda4275c 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -64,7 +64,7 @@ DEFAULTS = {
     "include_with_any_tags": None,
     "install_dir": None,
     "jobs": 1,
-    "logger_file": None,
+    "logger_file": "console",
     "mongo_executable": None,
     "mongod_executable": None,
     "mongod_set_parameters": [],
diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py
index 93dd66c3fca..01ca76e889d 100644
--- a/buildscripts/resmokelib/testing/fixtures/replicaset.py
+++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py
@@ -28,16 +28,18 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
     _INTERRUPTED_DUE_TO_REPL_STATE_CHANGE = 11602
 
     def __init__(  # pylint: disable=too-many-arguments, too-many-locals
-            self, logger, job_num, mongod_options=None, dbpath_prefix=None, preserve_dbpath=False,
-            num_nodes=2, start_initial_sync_node=False, write_concern_majority_journal_default=None,
-            auth_options=None, replset_config_options=None, voting_secondaries=True,
-            all_nodes_electable=False, use_replica_set_connection_string=None, linear_chain=False,
-            mixed_bin_versions=None, default_read_concern=None, default_write_concern=None,
-            shard_logging_prefix=None, replicaset_logging_prefix=None):
+            self, logger, job_num, mongod_executable=None, mongod_options=None, dbpath_prefix=None,
+            preserve_dbpath=False, num_nodes=2, start_initial_sync_node=False,
+            write_concern_majority_journal_default=None, auth_options=None,
+            replset_config_options=None, voting_secondaries=True, all_nodes_electable=False,
+            use_replica_set_connection_string=None, linear_chain=False, mixed_bin_versions=None,
+            default_read_concern=None, default_write_concern=None, shard_logging_prefix=None,
+            replicaset_logging_prefix=None):
         """Initialize ReplicaSetFixture."""
 
         interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix)
 
+        self.mongod_executable = mongod_executable
         self.mongod_options = utils.default_if_none(mongod_options, {})
         self.preserve_dbpath = preserve_dbpath
         self.start_initial_sync_node = start_initial_sync_node
@@ -62,8 +64,8 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
         self.num_nodes = num_replset_nodes if num_replset_nodes else num_nodes
 
         if self.mixed_bin_versions is not None:
-            mongod_executable = utils.default_if_none(config.MONGOD_EXECUTABLE,
-                                                      config.DEFAULT_MONGOD_EXECUTABLE)
+            mongod_executable = utils.default_if_none(
+                self.mongod_executable, config.MONGOD_EXECUTABLE, config.DEFAULT_MONGOD_EXECUTABLE)
             latest_mongod = mongod_executable
             # The last-lts binary is currently expected to live in '/data/multiversion', which is
             # part of the PATH.
@@ -591,8 +593,8 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
 
     def _new_mongod(self, index, replset_name):
         """Return a standalone.MongoDFixture configured to be used as replica-set member."""
-        mongod_executable = None if self.mixed_bin_versions is None else self.mixed_bin_versions[
-            index]
+        mongod_executable = (self.mongod_executable
+                             if self.mixed_bin_versions is None else self.mixed_bin_versions[index])
         mongod_logger = self._get_logger_for_mongod(index)
         mongod_options = self.mongod_options.copy()
         mongod_options["replSet"] = replset_name
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index 1c4699c8b11..dd8164c034c 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -25,10 +25,11 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
     _SHARD_REPLSET_NAME_PREFIX = "shard-rs"
 
     def __init__(  # pylint: disable=too-many-arguments,too-many-locals
-            self, logger, job_num, mongos_executable=None, mongos_options=None, mongod_options=None,
-            dbpath_prefix=None, preserve_dbpath=False, num_shards=1, num_rs_nodes_per_shard=1,
-            num_mongos=1, enable_sharding=None, enable_balancer=True, enable_autosplit=True,
-            auth_options=None, configsvr_options=None, shard_options=None, mixed_bin_versions=None):
+            self, logger, job_num, mongos_executable=None, mongos_options=None,
+            mongod_executable=None, mongod_options=None, dbpath_prefix=None, preserve_dbpath=False,
+            num_shards=1, num_rs_nodes_per_shard=1, num_mongos=1, enable_sharding=None,
+            enable_balancer=True, enable_autosplit=True, auth_options=None, configsvr_options=None,
+            shard_options=None, mixed_bin_versions=None):
         """Initialize ShardedClusterFixture with different options for the cluster processes."""
 
         interface.Fixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix)
@@ -39,6 +40,7 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         self.mongos_executable = mongos_executable
         self.mongos_options = utils.default_if_none(mongos_options, {})
         self.mongod_options = utils.default_if_none(mongod_options, {})
+        self.mongod_executable = mongod_executable
         self.mongod_options["set_parameters"] = mongod_options.get("set_parameters", {}).copy()
         self.mongod_options["set_parameters"]["migrationLockAcquisitionMaxWaitMS"] = \
                 mongod_options["set_parameters"].get("migrationLockAcquisitionMaxWaitMS", 30000)
@@ -276,8 +278,9 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
 
         return replicaset.ReplicaSetFixture(
             mongod_logger, self.job_num, mongod_options=mongod_options,
-            preserve_dbpath=preserve_dbpath, num_nodes=num_nodes, auth_options=auth_options,
-            mixed_bin_versions=None, replset_config_options=replset_config_options,
+            mongod_executable=self.mongod_executable, preserve_dbpath=preserve_dbpath,
+            num_nodes=num_nodes, auth_options=auth_options, mixed_bin_versions=None,
+            replset_config_options=replset_config_options,
             shard_logging_prefix=shard_logging_prefix, **configsvr_options)
 
     def _new_rs_shard(self, index, num_rs_nodes_per_shard):
@@ -308,11 +311,11 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         mongod_options["replSet"] = ShardedClusterFixture._SHARD_REPLSET_NAME_PREFIX + str(index)
 
         return replicaset.ReplicaSetFixture(
-            mongod_logger, self.job_num, mongod_options=mongod_options,
-            preserve_dbpath=preserve_dbpath, num_nodes=num_rs_nodes_per_shard,
-            auth_options=auth_options, replset_config_options=replset_config_options,
-            mixed_bin_versions=mixed_bin_versions, shard_logging_prefix=shard_logging_prefix,
-            **shard_options)
+            mongod_logger, self.job_num, mongod_executable=self.mongod_executable,
+            mongod_options=mongod_options, preserve_dbpath=preserve_dbpath,
+            num_nodes=num_rs_nodes_per_shard, auth_options=auth_options,
+            replset_config_options=replset_config_options, mixed_bin_versions=mixed_bin_versions,
+            shard_logging_prefix=shard_logging_prefix, **shard_options)
 
     def _new_standalone_shard(self, index):
         """Return a standalone.MongoDFixture configured as a shard in a sharded cluster."""
@@ -330,6 +333,7 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "shard{}".format(index))
 
         return standalone.MongoDFixture(mongod_logger, self.job_num, mongod_options=mongod_options,
+                                        mongod_executable=self.mongod_executable,
                                         preserve_dbpath=preserve_dbpath, **shard_options)
 
     def _new_mongos(self, index, total):
diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py
index 65311c9c3bc..2768e041c05 100644
--- a/buildscripts/resmokelib/utils/__init__.py
+++ b/buildscripts/resmokelib/utils/__init__.py
@@ -32,9 +32,13 @@ def open_or_use_stdout(filename):
         fp.close()
 
 
-def default_if_none(value, default):
-    """Set default if value is 'None'."""
-    return value if value is not None else default
+def default_if_none(*values):
+    """Return the first argument that is not 'None'."""
+    for value in values:
+        if value is not None:
+            return value
+
+    return None
 
 
 def rmtree(path, **kwargs):
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 103936c831f..854cef44d85 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -4181,12 +4181,15 @@ tasks:
         targets: lint-errorcodes
 
 - name: test_api_version_compatibility
+  depends_on:
+    - name: compile
   commands:
     - command: manifest.load
     - func: "git get project"
     - func: "set task expansion macros"
     - func: "set up virtualenv"
     - func: "upload pip requirements"
+    - func: "do setup"
     - command: shell.exec
       type: test
       params:
@@ -4197,6 +4200,7 @@ tasks:
           set -o verbose
           ${activate_virtualenv}
 
+          $python buildscripts/idl/check_versioned_api_commands_have_idl_definitions.py -v --include src --installDir dist-test/bin 1
           $python buildscripts/idl/checkout_idl_files_from_past_releases.py -v idls
 
 - name: burn_in_tests_gen