SERVER-53212 Create an evergreen task for IDL compatibility check

6 files changed, 162 insertions, 18 deletions

diff --git a/buildscripts/idl/checkout_idl_files_from_past_releases.py b/buildscripts/idl/checkout_idl_files_from_past_releases.py
index 50af4472325..1e0fd100b1d 100644
--- a/buildscripts/idl/checkout_idl_files_from_past_releases.py
+++ b/buildscripts/idl/checkout_idl_files_from_past_releases.py
@@ -35,7 +35,8 @@ from typing import List
 
 from packaging.version import Version
 
-FIRST_API_V1_RELEASE = '4.9.0-alpha'
+# TODO (SERVER-55203): Change FIRST_API_V1_RELEASE to r5.0.0-rc0.
+FIRST_API_V1_RELEASE = '4.9.0-alpha5'
 LOGGER_NAME = 'checkout-idl'
 LOGGER = logging.getLogger(LOGGER_NAME)
 
diff --git a/buildscripts/idl/idl_check_compatibility.py b/buildscripts/idl/idl_check_compatibility.py
index a7fab961ff4..cc285fe2e7d 100644
--- a/buildscripts/idl/idl_check_compatibility.py
+++ b/buildscripts/idl/idl_check_compatibility.py
@@ -37,18 +37,19 @@ directories containing the old IDL files from previous releases.
 """
 
 import argparse
-import logging
 import os
 import sys
 from typing import Dict, List, Optional, Tuple, Union
 
 from idl import parser, syntax, errors, common
 from idl.compiler import CompilerImportResolver
-from idl_compatibility_errors import IDLCompatibilityContext, IDLCompatibilityErrorCollection
+from idl_compatibility_errors import IDLCompatibilityContext, IDLCompatibilityErrorCollection, dump_errors
 
 ALLOW_ANY_TYPE_LIST: List[str] = [
-    "commandAllowedAnyTypes", "commandAllowedAnyTypes-param-anyTypeParam",
-    "commandAllowedAnyTypes-reply-anyTypeField", "oldTypeBsonAnyAllowList",
+    "commandAllowedAnyTypes",
+    "commandAllowedAnyTypes-param-anyTypeParam",
+    "commandAllowedAnyTypes-reply-anyTypeField",
+    "oldTypeBsonAnyAllowList",
     "newTypeBsonAnyAllowList",
     "oldReplyFieldTypeBsonAnyAllowList-reply-oldBsonSerializationTypeAnyReplyField",
     "newReplyFieldTypeBsonAnyAllowList-reply-newBsonSerializationTypeAnyReplyField",
@@ -59,11 +60,78 @@ ALLOW_ANY_TYPE_LIST: List[str] = [
     "replyFieldTypeBsonAnyWithVariantWithArray-reply-bsonSerializationTypeAnyStructField",
     "parameterFieldTypeBsonAnyWithVariant-param-bsonSerializationTypeAnyStructField",
     "parameterFieldTypeBsonAnyWithVariantWithArray-param-bsonSerializationTypeAnyStructField",
-    "commandTypeBsonAnyWithVariant", "commandTypeBsonAnyWithVariantWithArray",
-    "replyFieldCppTypeNotEqual-reply-cppTypeNotEqualReplyField", "commandCppTypeNotEqual",
-    "commandParameterCppTypeNotEqual-param-cppTypeNotEqualParam"
+    "commandTypeBsonAnyWithVariant",
+    "commandTypeBsonAnyWithVariantWithArray",
+    "replyFieldCppTypeNotEqual-reply-cppTypeNotEqualReplyField",
+    "commandCppTypeNotEqual",
+    "commandParameterCppTypeNotEqual-param-cppTypeNotEqualParam",
+
+    # TODO (SERVER-54956): Decide what to do with commands: (create, createIndexes).
+    'create-param-backwards',
+    'createIndexes-param-commitQuorum',
+    'createIndexes-reply-commitQuorum',
+
+    # TODO (SERVER-54923): Decide what to do with commands: (saslStart, saslContinue).
+    'saslStart-param-payload',
+    'explain-param-use44SortKeys',
+    'explain-param-useNewUpsert',
+    'saslStart-param-payload',
+    'saslStart-reply-payload',
+    'saslContinue-param-payload',
+    'saslContinue-reply-payload',
+
+    # TODO (SERVER-54925): Decide what to do with commands:
+    # (aggregate, find, update, delete, findAndModify, explain).
+    'aggregate-param-pipeline',
+    'aggregate-param-explain',
+    'aggregate-param-allowDiskUse',
+    'aggregate-param-cursor',
+    'aggregate-param-hint',
+    'aggregate-param-needsMerge',
+    'aggregate-param-fromMongos',
+    'aggregate-param-$_requestReshardingResumeToken',
+    'aggregate-param-isMapReduceCommand',
+    'find-param-filter',
+    'find-param-projection',
+    'find-param-sort',
+    'find-param-hint',
+    'find-param-collation',
+    'find-param-singleBatch',
+    'find-param-allowDiskUse',
+    'find-param-min',
+    'find-param-max',
+    'find-param-returnKey',
+    'find-param-showRecordId',
+    'find-param-$queryOptions',
+    'find-param-tailable',
+    'find-param-oplogReplay',
+    'find-param-noCursorTimeout',
+    'find-param-awaitData',
+    'find-param-allowPartialResults',
+    'find-param-readOnce',
+    'find-param-allowSpeculativeMajorityRead',
+    'find-param-$_requestResumeToken',
+    'find-param-$_resumeAfter',
+    'find-param-maxTimeMS',
+    'update-param-u',
+    'update-param-hint',
+    'update-param-upsertSupplied',
+    'update-reply-_id',
+    'delete-param-limit',
+    'delete-param-hint',
+    'findAndModify-param-hint',
+    'findAndModify-param-update',
+    'findAndModify-reply-upserted',
+    'explain-param-collation',
+    'explain-param-use44SortKeys',
+    'explain-param-useNewUpsert',
+
+    # TODO (SERVER-54927): Decide what to do with commands: (hello).
+    'hello-param-saslSupportedMechs'
 ]
 
+SKIPPED_FILES = ["unittest.idl"]
+
 
 def get_new_commands(
         ctxt: IDLCompatibilityContext, new_idl_dir: str, import_directories: List[str]
@@ -75,7 +143,7 @@ def get_new_commands(
 
     for dirpath, _, filenames in os.walk(new_idl_dir):
         for new_filename in filenames:
-            if not new_filename.endswith('.idl'):
+            if not new_filename.endswith('.idl') or new_filename in SKIPPED_FILES:
                 continue
 
             new_idl_file_path = os.path.join(dirpath, new_filename)
@@ -634,7 +702,10 @@ def check_error_reply(old_basic_types_path: str, new_basic_types_path: str,
                     check_reply_fields(ctxt, old_error_reply_struct, new_error_reply_struct, "n/a",
                                        old_idl_file, new_idl_file, old_basic_types_path,
                                        new_basic_types_path)
-    ctxt.errors.dump_errors()
+
+    # TODO (SERVER-55203): Remove error_skipped logic.
+    ctxt.errors.remove_skipped_errors_and_dump_all_errors("ErrorReply", old_basic_types_path,
+                                                          new_basic_types_path)
     return ctxt.errors
 
 
@@ -677,7 +748,7 @@ def check_compatibility(old_idl_dir: str, new_idl_dir: str,
     old_commands: Dict[str, syntax.Command] = dict()
     for dirpath, _, filenames in os.walk(old_idl_dir):
         for old_filename in filenames:
-            if not old_filename.endswith('.idl'):
+            if not old_filename.endswith('.idl') or old_filename in SKIPPED_FILES:
                 continue
 
             old_idl_file_path = os.path.join(dirpath, old_filename)
@@ -733,7 +804,9 @@ def check_compatibility(old_idl_dir: str, new_idl_dir: str,
                     check_security_access_check(ctxt, old_cmd.access_check, new_cmd.access_check,
                                                 old_cmd.command_name, new_idl_file_path)
 
-    ctxt.errors.dump_errors()
+    # TODO (SERVER-55203): Remove error_skipped logic.
+    ctxt.errors.remove_skipped_errors_and_dump_all_errors("Commands", old_idl_dir, new_idl_dir)
+
     return ctxt.errors
 
 
@@ -741,19 +814,21 @@ def main():
     """Run the script."""
     arg_parser = argparse.ArgumentParser(description=__doc__)
     arg_parser.add_argument("-v", "--verbose", action="count", help="Enable verbose logging")
+    arg_parser.add_argument("--include", type=str, action="append",
+                            help="Directory to search for IDL import files")
     arg_parser.add_argument("old_idl_dir", metavar="OLD_IDL_DIR",
                             help="Directory where old IDL files are located")
     arg_parser.add_argument("new_idl_dir", metavar="NEW_IDL_DIR",
                             help="Directory where new IDL files are located")
     args = arg_parser.parse_args()
 
-    error_coll = check_compatibility(args.old_idl_dir, args.new_idl_dir, [])
+    error_coll = check_compatibility(args.old_idl_dir, args.new_idl_dir, args.include)
     if error_coll.has_errors():
         sys.exit(1)
 
     old_basic_types_path = os.path.join(args.old_idl_dir, "mongo/idl/basic_types.idl")
     new_basic_types_path = os.path.join(args.new_idl_dir, "mongo/idl/basic_types.idl")
-    error_reply_coll = check_error_reply(old_basic_types_path, new_basic_types_path, [])
+    error_reply_coll = check_error_reply(old_basic_types_path, new_basic_types_path, args.include)
     if error_reply_coll.has_errors():
         sys.exit(1)
 
diff --git a/buildscripts/idl/idl_compatibility_errors.py b/buildscripts/idl/idl_compatibility_errors.py
index d07920dc9dc..fefb3f6bb7a 100644
--- a/buildscripts/idl/idl_compatibility_errors.py
+++ b/buildscripts/idl/idl_compatibility_errors.py
@@ -103,6 +103,14 @@ ERROR_ID_RESOURCE_PATTERN_NOT_EQUAL = "ID0059"
 ERROR_ID_NEW_ACTION_TYPES_NOT_SUBSET = "ID0060"
 ERROR_ID_TYPE_NOT_ARRAY = "ID0061"
 
+# TODO (SERVER-55203): Remove SKIPPED_COMMANDS logic.
+# Any breaking changes added to API V1 before releasing 5.0 should be added to SKIPPED_COMMANDS to
+# be skipped from compatibility checking.
+SKIPPED_COMMANDS = {
+    "invalidReplySkippedCommand": [ERROR_ID_NEW_REPLY_FIELD_MISSING],
+    "dropDatabase": [ERROR_ID_NEW_REPLY_FIELD_MISSING]
+}
+
 
 class IDLCompatibilityCheckerError(Exception):
     """Base class for all IDL Compatibility Checker exceptions."""
@@ -148,6 +156,15 @@ class IDLCompatibilityError(object):
         return msg
 
 
+def dump_errors(header_msg: str, errors: List[IDLCompatibilityError]) -> None:
+    """Print the list of errors."""
+    error_list = [str(error) for error in errors]
+    print(header_msg + "\n: %s errors:" % (len(error_list)))
+    for error_msg in error_list:
+        print("%s\n\n" % error_msg)
+    print("------------------------------------------------")
+
+
 class IDLCompatibilityErrorCollection(object):
     """Collection of IDL compatibility errors with source context information."""
 
@@ -197,16 +214,40 @@ class IDLCompatibilityErrorCollection(object):
         """Get all the errors in the error collection with the command command_name."""
         return [a for a in self._errors if a.command_name == command_name]
 
+    def extract_skipped_errors(self) -> List[IDLCompatibilityError]:
+        """Remove all the errors in the error collection that are skipped and return them."""
+        skipped_errors = [
+            c for c in self._errors
+            if c.command_name in SKIPPED_COMMANDS and c.error_id in SKIPPED_COMMANDS[c.command_name]
+        ]
+        self._errors = [
+            c for c in self._errors if not (c.command_name in SKIPPED_COMMANDS
+                                            and c.error_id in SKIPPED_COMMANDS[c.command_name])
+        ]
+        return skipped_errors
+
     def to_list(self) -> List[str]:
         """Return a list of formatted error messages."""
         return [str(error) for error in self._errors]
 
     def dump_errors(self) -> None:
         """Print the list of errors."""
-        print("Errors found while checking IDL compatibility")
-        for error_msg in self.to_list():
-            print("%s\n\n" % error_msg)
-        print("Found %s errors" % (len(self.to_list())))
+        dump_errors("Errors found while checking IDL compatibility", self._errors)
+
+    def remove_skipped_errors_and_dump_all_errors(self, checking_target: str, old_path: str,
+                                                  new_path: str) -> None:
+        """Remove skipped errors from errors list and dump skipped and found errors."""
+        skipped_errors = self.extract_skipped_errors()
+        suffix_msg = "for (%s) between old directory (%s) and the new directory (%s)" % (
+            checking_target, old_path, new_path)
+        dump_errors(
+            "(Note: Skipped errors should be fixed before 5.0 release as it won't be skipped" +
+            " after.)\n" + "Errors skipped while checking IDL compatibility " + suffix_msg,
+            skipped_errors)
+        dump_errors("Errors found while checking IDL compatibility " + suffix_msg, self._errors)
+        print("(Note: Any breaking changes added to API V1 before releasing 5.0 should be added to "
+              + "SKIPPED_COMMANDS list at (/buildscripts/idl/idl_compatibility_errors.py) to be " +
+              "skipped from compatibility checking.)")
 
     def count(self) -> int:
         """Return the count of errors."""
diff --git a/buildscripts/idl/tests/compatibility_test_fail/new/compatibility_test_fail_new.idl b/buildscripts/idl/tests/compatibility_test_fail/new/compatibility_test_fail_new.idl
index 0ebdda46d4e..595b460ab3f 100644
--- a/buildscripts/idl/tests/compatibility_test_fail/new/compatibility_test_fail_new.idl
+++ b/buildscripts/idl/tests/compatibility_test_fail/new/compatibility_test_fail_new.idl
@@ -1817,3 +1817,14 @@ commands:
             parameterStruct:
                 type: array<StructType>
         reply_type: ArrayTypeStruct
+
+    # TODO (SERVER-55203): Remove InvalidReplySkippedCommand.
+    InvalidReplySkippedCommand:
+        description: "new command should fail because it has invalid reply, but this command will
+                      be added to skipped command list."
+        command_name: invalidReplySkippedCommand
+        namespace: ignored
+        cpp_name: InvalidReplySkippedCommand
+        strict: true
+        api_version: "1"
+        reply_type: OkReply
\ No newline at end of file
diff --git a/buildscripts/idl/tests/compatibility_test_fail/old/compatibility_test_fail_old.idl b/buildscripts/idl/tests/compatibility_test_fail/old/compatibility_test_fail_old.idl
index 4497b150d0a..174f33c452c 100644
--- a/buildscripts/idl/tests/compatibility_test_fail/old/compatibility_test_fail_old.idl
+++ b/buildscripts/idl/tests/compatibility_test_fail/old/compatibility_test_fail_old.idl
@@ -1806,3 +1806,14 @@ commands:
             parameterStruct:
                 type: array<ArrayTypeStruct>
         reply_type: ArrayTypeStruct
+
+    # TODO (SERVER-55203): Remove InvalidReplySkippedCommand.
+    InvalidReplySkippedCommand:
+        description: "new command should fail because it has invalid reply, but this command will
+                      be added to skipped command list."
+        command_name: invalidReplySkippedCommand
+        namespace: ignored
+        cpp_name: InvalidReplySkippedCommand
+        strict: true
+        api_version: "1"
+        reply_type: NotStructFieldReply
\ No newline at end of file
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 583fbd11a94..84712703197 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -4077,6 +4077,11 @@ tasks:
 
           $python buildscripts/idl/check_versioned_api_commands_have_idl_definitions.py -v --include src --include src/mongo/db/modules/enterprise/src --installDir dist-test/bin 1
           $python buildscripts/idl/checkout_idl_files_from_past_releases.py -v idls
+          find idls -maxdepth 1 -mindepth 1 -type d | while read dir;
+          do
+            echo "Performing idl check compatibility with release: $dir:";
+            $python buildscripts/idl/idl_check_compatibility.py -v --include src --include src/mongo/db/modules/enterprise/src "$dir/src" src;
+          done
 
 - name: burn_in_tests_gen
   commands: