SERVER-54145 Handle variant types in reply fields in IDL compatibility checker script

7 files changed, 482 insertions, 15 deletions

diff --git a/buildscripts/idl/idl_check_compatibility.py b/buildscripts/idl/idl_check_compatibility.py
index 00857a60bea..fe8a9b9cd47 100644
--- a/buildscripts/idl/idl_check_compatibility.py
+++ b/buildscripts/idl/idl_check_compatibility.py
@@ -120,6 +120,71 @@ def check_type_superset(ctxt: IDLCompatibilityContext, cmd_name: str, type_name:
         ctxt.add_command_type_not_superset_error(cmd_name, type_name, file_path)
 
 
+def check_reply_field_type_recursive(
+        ctxt: IDLCompatibilityContext, old_field_type: syntax.Type,
+        new_field_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]], cmd_name: str,
+        field_name: str, old_idl_file: syntax.IDLParsedSpec, new_idl_file: syntax.IDLParsedSpec,
+        old_idl_file_path: str, new_idl_file_path: str) -> None:
+    # pylint: disable=too-many-arguments,too-many-branches
+    """Check compatibility between old and new reply field type if old field type is a syntax.Type instance."""
+    if not isinstance(new_field_type, syntax.Type):
+        ctxt.add_new_reply_field_type_enum_or_struct_error(
+            cmd_name, field_name, new_field_type.name, old_field_type.name, new_idl_file_path)
+        return
+
+    if "any" in old_field_type.bson_serialization_type:
+        ctxt.add_old_reply_field_bson_any_error(cmd_name, field_name, old_field_type.name,
+                                                old_idl_file_path)
+        return
+    if "any" in new_field_type.bson_serialization_type:
+        ctxt.add_new_reply_field_bson_any_error(cmd_name, field_name, new_field_type.name,
+                                                new_idl_file_path)
+        return
+
+    if isinstance(old_field_type, syntax.VariantType):
+        # If the new type is not variant just check the single type.
+        new_variant_types = new_field_type.variant_types if isinstance(
+            new_field_type, syntax.VariantType) else [new_field_type]
+        old_variant_types = old_field_type.variant_types
+
+        # Check that new variant types are a subset of old variant types.
+        for new_variant_type in new_variant_types:
+            old_variant_type_exists = False
+            for old_variant_type in old_variant_types:
+                if old_variant_type.name == new_variant_type.name:
+                    old_variant_type_exists = True
+                    # Check that the old and new version of each variant type is also compatible.
+                    check_reply_field_type_recursive(
+                        ctxt, old_variant_type, new_variant_type, cmd_name, field_name,
+                        old_idl_file, new_idl_file, old_idl_file_path, new_idl_file_path)
+
+            if not old_variant_type_exists:
+                ctxt.add_new_reply_field_variant_type_not_subset_error(
+                    cmd_name, field_name, new_field_type.name, new_idl_file_path)
+
+        # If new type is variant and has a struct as a variant type, compare old and new variant_struct_type.
+        # Since enums can't be part of variant types, we don't explicitly check for enums.
+        if isinstance(new_field_type,
+                      syntax.VariantType) and new_field_type.variant_struct_type is not None:
+            if old_field_type.variant_struct_type is None:
+                ctxt.add_new_reply_field_variant_type_not_subset_error(
+                    cmd_name, field_name, new_field_type.variant_struct_type.name,
+                    new_idl_file_path)
+            else:
+                check_reply_fields(ctxt, old_field_type.variant_struct_type,
+                                   new_field_type.variant_struct_type, cmd_name, old_idl_file,
+                                   new_idl_file, old_idl_file_path, new_idl_file_path)
+
+    else:
+        if isinstance(new_field_type, syntax.VariantType):
+            ctxt.add_new_reply_field_variant_type_error(cmd_name, field_name, old_field_type.name,
+                                                        new_field_type.name, new_idl_file_path)
+        else:
+            check_subset(ctxt, cmd_name, field_name, new_field_type.name,
+                         new_field_type.bson_serialization_type,
+                         old_field_type.bson_serialization_type, new_idl_file_path)
+
+
 def check_reply_field_type(ctxt: IDLCompatibilityContext,
                            old_field_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]],
                            new_field_type: Optional[Union[syntax.Enum, syntax.Struct, syntax.Type]],
@@ -138,20 +203,10 @@ def check_reply_field_type(ctxt: IDLCompatibilityContext,
         sys.exit(1)
 
     if isinstance(old_field_type, syntax.Type):
-        if isinstance(new_field_type, syntax.Type):
-            if "any" in old_field_type.bson_serialization_type:
-                ctxt.add_old_reply_field_bson_any_error(cmd_name, field_name, old_field_type.name,
-                                                        old_idl_file_path)
-            elif "any" in new_field_type.bson_serialization_type:
-                ctxt.add_new_reply_field_bson_any_error(cmd_name, field_name, new_field_type.name,
-                                                        new_idl_file_path)
-            else:
-                check_subset(ctxt, cmd_name, field_name, new_field_type.name,
-                             new_field_type.bson_serialization_type,
-                             old_field_type.bson_serialization_type, new_idl_file_path)
-        else:
-            ctxt.add_new_reply_field_type_enum_or_struct_error(
-                cmd_name, field_name, new_field_type.name, old_field_type.name, new_idl_file_path)
+        check_reply_field_type_recursive(ctxt, old_field_type, new_field_type, cmd_name, field_name,
+                                         old_idl_file, new_idl_file, old_idl_file_path,
+                                         new_idl_file_path)
+
     elif isinstance(old_field_type, syntax.Enum):
         if isinstance(new_field_type, syntax.Enum):
             check_subset(ctxt, cmd_name, field_name, new_field_type.name, new_field_type.values,
diff --git a/buildscripts/idl/idl_compatibility_errors.py b/buildscripts/idl/idl_compatibility_errors.py
index 94b63ab7466..da19af30328 100644
--- a/buildscripts/idl/idl_compatibility_errors.py
+++ b/buildscripts/idl/idl_compatibility_errors.py
@@ -66,6 +66,8 @@ ERROR_ID_NEW_COMMAND_TYPE_NOT_STRUCT = "ID0022"
 ERROR_ID_NEW_COMMAND_TYPE_NOT_ENUM = "ID0023"
 ERROR_ID_NEW_COMMAND_TYPE_ENUM_OR_STRUCT = "ID0024"
 ERROR_ID_MISSING_ERROR_REPLY_STRUCT = "ID0025"
+ERROR_ID_NEW_REPLY_FIELD_VARIANT_TYPE = "ID0026"
+ERROR_ID_NEW_REPLY_FIELD_VARIANT_TYPE_NOT_SUBSET = "ID0027"
 
 
 class IDLCompatibilityCheckerError(Exception):
@@ -361,6 +363,25 @@ class IDLCompatibilityContext(object):
             "'%s' has an unstable reply field '%s' that was stable in the old command." %
             (command_name, field_name), file)
 
+    def add_new_reply_field_variant_type_error(self, command_name: str, field_name: str,
+                                               new_field_type: str, old_field_type: str,
+                                               file: str) -> None:
+        # pylint: disable=too-many-arguments
+        """Add an error about the new reply field type being variant when the old one is not."""
+        self._add_error(
+            ERROR_ID_NEW_REPLY_FIELD_VARIANT_TYPE, command_name,
+            ("'%s' has a reply field '%s' of type '%s' that is variant while the corresponding "
+             "old reply field type '%s' is not.") % (command_name, field_name, new_field_type,
+                                                     old_field_type), file)
+
+    def add_new_reply_field_variant_type_not_subset_error(self, command_name: str, field_name: str,
+                                                          type_name: str, file: str) -> None:
+        # pylint: disable=too-many-arguments
+        """Add an error about the new reply field variant types not being a subset of the old variant types."""
+        self._add_error(ERROR_ID_NEW_REPLY_FIELD_VARIANT_TYPE_NOT_SUBSET, command_name, (
+            "'%s' has a reply field '%s' with variant alternative type '%s' that is not a subset of the corresponding "
+            "old reply field type") % (command_name, field_name, type_name), file)
+
     def add_old_command_type_bson_any_error(self, command_name: str, old_type: str,
                                             file: str) -> None:
         """Add an error about the old command type's bson serialization type being of type "any"."""
diff --git a/buildscripts/idl/tests/compatibility_test_fail/new/compatibility_test_fail_new.idl b/buildscripts/idl/tests/compatibility_test_fail/new/compatibility_test_fail_new.idl
index aaf7cd22272..acafaa0271f 100644
--- a/buildscripts/idl/tests/compatibility_test_fail/new/compatibility_test_fail_new.idl
+++ b/buildscripts/idl/tests/compatibility_test_fail/new/compatibility_test_fail_new.idl
@@ -195,6 +195,46 @@ structs:
         fields:
             fieldOne:
                 type: BsonNotSubsetReply
+    
+    NewVariantTypeReply:
+        description: "This reply contains a new field that has a variant type while the old field
+                      is not a variant type"
+        fields:
+            newVariantTypeReplyField:
+                type: 
+                    variant: [int, string]
+
+    NewVariantNotSubsetReply:
+        description: "This reply contains a field whose new variant types are not a subset
+                      of the old variant types"
+        fields:
+            variantNotSubsetReplyField:
+                type: 
+                    variant: [int, bool, string]
+    
+    VariantRecursiveReply:
+        description: "This reply contains a field that has a new variant type that is not compatible
+                      with the old variant type"
+        fields:
+            variantRecursiveReplyField:
+                type: 
+                    variant: [int, intStringToIntStringBool]
+    
+    NewVariantStructNotSubsetReply:
+        description: "This reply contains a field whose new variant types are not a subset
+                      of the old variant types"
+        fields:
+            variantStructNotSubsetReplyField:
+                type: 
+                    variant: [int, string, StructType]
+    
+    VariantStructRecursiveReply:
+        description: "This reply contains a field that has a new variant struct type that is not compatible
+                      with the old variant struct type"
+        fields:
+            variantStructRecursiveReplyField:
+                type: 
+                    variant: [int, StructFieldTypeRecursiveReplyTwo]
 
 commands:
     invalidAPIVersionNew:
@@ -536,3 +576,53 @@ commands:
         strict: true
         api_version: "1"
         reply_type: OkReply
+
+    newReplyFieldVariantType:
+        description: "new command fails because its reply field type is a variant type while
+                      the old reply field is not a variant type"
+        command_name: newReplyFieldVariantType
+        namespace: ignored
+        cpp_name: newReplyFieldVariantType
+        strict: true
+        api_version: "1"
+        reply_type: NewVariantTypeReply
+    
+    newReplyFieldVariantNotSubset:
+        description: "new command fails because its reply field type is a variant type that is not
+                      a subset of the old reply field variant types"
+        command_name: newReplyFieldVariantNotSubset
+        namespace: ignored
+        cpp_name: newReplyFieldVariantNotSubset
+        strict: true
+        api_version: "1"
+        reply_type: NewVariantNotSubsetReply
+
+    replyFieldVariantRecursive:
+        description: "new command fails because its reply field type is a variant type that is not
+                      compatible with the old reply field variant type"
+        command_name: replyFieldVariantRecursive
+        namespace: ignored
+        cpp_name: replyFieldVariantRecursive
+        strict: true
+        api_version: "1"
+        reply_type: VariantRecursiveReply
+
+    newReplyFieldVariantStructNotSubset:
+        description: "new command fails because its reply field type is a variant type that is not
+                      a subset of the old reply field variant types"
+        command_name: newReplyFieldVariantStructNotSubset
+        namespace: ignored
+        cpp_name: newReplyFieldVariantStructNotSubset
+        strict: true
+        api_version: "1"
+        reply_type: NewVariantStructNotSubsetReply
+    
+    replyFieldVariantStructRecursive:
+        description: "new command fails because its reply field type has a variant struct type that is not
+                      compatible with the old reply field variant struct type"
+        command_name: replyFieldVariantStructRecursive
+        namespace: ignored
+        cpp_name: replyFieldVariantStructRecursive
+        strict: true
+        api_version: "1"
+        reply_type: VariantStructRecursiveReply
\ No newline at end of file
diff --git a/buildscripts/idl/tests/compatibility_test_fail/old/compatibility_test_fail_old.idl b/buildscripts/idl/tests/compatibility_test_fail/old/compatibility_test_fail_old.idl
index 6a0fb8ff267..0f320bc7799 100644
--- a/buildscripts/idl/tests/compatibility_test_fail/old/compatibility_test_fail_old.idl
+++ b/buildscripts/idl/tests/compatibility_test_fail/old/compatibility_test_fail_old.idl
@@ -199,6 +199,45 @@ structs:
             fieldOne:
                 type: BsonNotSubsetReply
 
+    NewVariantTypeReply:
+        description: "This reply contains a new field that has a variant type while the old field
+                      is not a variant type"
+        fields:
+            newVariantTypeReplyField:
+                type: int
+
+    NewVariantNotSubsetReply:
+        description: "This reply contains a field whose new variant types are not a subset
+                      of the old variant types"
+        fields:
+            variantNotSubsetReplyField:
+                type: 
+                    variant: [int, string]
+    
+    VariantRecursiveReply:
+        description: "This reply contains a field that has a new variant type that is not compatible
+                      with the old variant type"
+        fields:
+            variantRecursiveReplyField:
+                type: 
+                    variant: [int, intStringToIntStringBool]
+    
+    NewVariantStructNotSubsetReply:
+        description: "This reply contains a field whose new variant types are not a subset
+                      of the old variant types"
+        fields:
+            variantStructNotSubsetReplyField:
+                type: 
+                    variant: [int, string]
+
+    VariantStructRecursiveReply:
+        description: "This reply contains a field that has a new variant struct type that is not compatible
+                      with the old variant struct type"
+        fields:
+            variantStructRecursiveReplyField:
+                type: 
+                    variant: [int, StructFieldTypeRecursiveReplyTwo]
+
 commands:
     invalidAPIVersionOld:
         description: "old command fails because of invalid API version"
@@ -529,3 +568,53 @@ commands:
         strict: true
         api_version: "1"
         reply_type: OkReply
+    
+    newReplyFieldVariantType:
+        description: "new command fails because its reply field type is a variant type while
+                      the old reply field is not a variant type"
+        command_name: newReplyFieldVariantType
+        namespace: ignored
+        cpp_name: newReplyFieldVariantType
+        strict: true
+        api_version: "1"
+        reply_type: NewVariantTypeReply
+    
+    newReplyFieldVariantNotSubset:
+        description: "new command fails because its reply field type is a variant type that is not
+                      a subset of the old reply field variant types"
+        command_name: newReplyFieldVariantNotSubset
+        namespace: ignored
+        cpp_name: newReplyFieldVariantNotSubset
+        strict: true
+        api_version: "1"
+        reply_type: NewVariantNotSubsetReply
+
+    replyFieldVariantRecursive:
+        description: "new command fails because its reply field type is a variant type that is not
+                      compatible with the old reply field variant type"
+        command_name: replyFieldVariantRecursive
+        namespace: ignored
+        cpp_name: replyFieldVariantRecursive
+        strict: true
+        api_version: "1"
+        reply_type: VariantRecursiveReply
+
+    newReplyFieldVariantStructNotSubset:
+        description: "new command fails because its reply field type is a variant type that is not
+                      a subset of the old reply field variant types"
+        command_name: newReplyFieldVariantStructNotSubset
+        namespace: ignored
+        cpp_name: newReplyFieldVariantStructNotSubset
+        strict: true
+        api_version: "1"
+        reply_type: NewVariantStructNotSubsetReply
+    
+    replyFieldVariantStructRecursive:
+        description: "new command fails because its reply field type has a variant struct type that is not
+                      compatible with the old reply field variant struct type"
+        command_name: replyFieldVariantStructRecursive
+        namespace: ignored
+        cpp_name: replyFieldVariantStructRecursive
+        strict: true
+        api_version: "1"
+        reply_type: VariantStructRecursiveReply
diff --git a/buildscripts/idl/tests/compatibility_test_pass/new/compatibility_test_pass_new.idl b/buildscripts/idl/tests/compatibility_test_pass/new/compatibility_test_pass_new.idl
index 5509de88591..b0c926e90ef 100644
--- a/buildscripts/idl/tests/compatibility_test_pass/new/compatibility_test_pass_new.idl
+++ b/buildscripts/idl/tests/compatibility_test_pass/new/compatibility_test_pass_new.idl
@@ -155,6 +155,46 @@ structs:
         fields:
             fieldOne:
                 type: BsonSubsetReply
+    
+    OldVariantTypeReply:
+        description: "This reply contains an old field that has a variant type while the new field
+                      is not a variant type"
+        fields:
+            oldVariantTypeReplyField:
+                type: int
+
+    NewVariantSubsetReply:
+        description: "This reply contains a field whose new variant types are a subset
+                      of the old variant types"
+        fields:
+            variantSubsetReplyField:
+                type: 
+                    variant: [int, string]
+    
+    VariantRecursiveReply:
+        description: "This reply contains a field that has a new variant type that is compatible
+                      with the old variant type"
+        fields:
+            variantRecursiveReplyField:
+                type: 
+                    variant: [int, intStringBoolToIntString]
+    
+    OldVariantStructReply:
+        description: "This reply contains a field whose new variant type does not have a variant
+                      struct type while the old one does"
+        fields:
+            variantStructReplyField:
+                type: 
+                    variant: [int, string]
+    
+    VariantStructRecursiveReply:
+        description: "This reply contains a field that has a new variant struct type that is compatible
+                      with the old variant struct type"
+        fields:
+            variantStructRecursiveReplyField:
+                type: 
+                    variant: [int, StructFieldTypeRecursiveReplyTwo]
+
 
 commands:
     testCommand:
@@ -355,3 +395,53 @@ commands:
         strict: true
         api_version: "1"
         reply_type: OkReply
+    
+    oldReplyFieldVariantType:
+        description: "new command passes when its reply field type is not a variant type while
+                      the old reply field is a variant type"
+        command_name: oldReplyFieldVariantType
+        namespace: ignored
+        cpp_name: oldReplyFieldVariantType
+        strict: true
+        api_version: "1"
+        reply_type: OldVariantTypeReply
+    
+    newReplyFieldVariantSubset:
+        description: "new command when its reply field type is a variant type that is
+                      a subset of the old reply field variant types"
+        command_name: newReplyFieldVariantSubset
+        namespace: ignored
+        cpp_name: newReplyFieldVariantSubset
+        strict: true
+        api_version: "1"
+        reply_type: NewVariantSubsetReply
+
+    replyFieldVariantRecursive:
+        description: "new command passes when its reply field type is a variant type that is
+                      compatible with the old reply field variant type"
+        command_name: replyFieldVariantRecursive
+        namespace: ignored
+        cpp_name: replyFieldVariantRecursive
+        strict: true
+        api_version: "1"
+        reply_type: VariantRecursiveReply
+    
+    oldReplyFieldVariantStruct:
+        description: "new command passes if it doesn't have a variant struct type while the 
+                      old command does"
+        command_name: oldReplyFieldVariantStruct
+        namespace: ignored
+        cpp_name: oldReplyFieldVariantStruct
+        strict: true
+        api_version: "1"
+        reply_type: OldVariantStructReply
+    
+    replyFieldVariantStructRecursive:
+        description: "new command passes when its reply field type has a variant struct type that is
+                      compatible with the old reply field variant struct type"
+        command_name: replyFieldVariantStructRecursive
+        namespace: ignored
+        cpp_name: replyFieldVariantStructRecursive
+        strict: true
+        api_version: "1"
+        reply_type: VariantStructRecursiveReply
diff --git a/buildscripts/idl/tests/compatibility_test_pass/old/compatibility_test_pass_old.idl b/buildscripts/idl/tests/compatibility_test_pass/old/compatibility_test_pass_old.idl
index 99bc674cccd..5f52041b6e1 100644
--- a/buildscripts/idl/tests/compatibility_test_pass/old/compatibility_test_pass_old.idl
+++ b/buildscripts/idl/tests/compatibility_test_pass/old/compatibility_test_pass_old.idl
@@ -154,6 +154,46 @@ structs:
         fields:
             fieldOne:
                 type: BsonSubsetReply
+    
+    OldVariantTypeReply:
+        description: "This reply contains an old field that has a variant type while the new field
+                      is not a variant type"
+        fields:
+            oldVariantTypeReplyField:
+                type: 
+                    variant: [int, string]
+
+    NewVariantSubsetReply:
+        description: "This reply contains a field whose new variant types are a subset
+                      of the old variant types"
+        fields:
+            variantSubsetReplyField:
+                type: 
+                    variant: [int, bool, string]
+    
+    VariantRecursiveReply:
+        description: "This reply contains a field that has a new variant type that is compatible
+                      with the old variant type"
+        fields:
+            variantRecursiveReplyField:
+                type: 
+                    variant: [int, intStringBoolToIntString]
+    
+    OldVariantStructReply:
+        description: "This reply contains a field whose new variant type does not have a variant
+                      struct type while the old one does"
+        fields:
+            variantStructReplyField:
+                type: 
+                    variant: [int, string, StructType]
+    
+    VariantStructRecursiveReply:
+        description: "This reply contains a field that has a new variant struct type that is compatible
+                      with the old variant struct type"
+        fields:
+            variantStructRecursiveReplyField:
+                type: 
+                    variant: [int, StructFieldTypeRecursiveReplyTwo]
 
 commands:
     testCommand:
@@ -357,3 +397,53 @@ commands:
         strict: true
         api_version: "1"
         reply_type: OkReply
+
+    oldReplyFieldVariantType:
+        description: "new command passes when its reply field type is not a variant type while
+                      the old reply field is a variant type"
+        command_name: oldReplyFieldVariantType
+        namespace: ignored
+        cpp_name: oldReplyFieldVariantType
+        strict: true
+        api_version: "1"
+        reply_type: OldVariantTypeReply
+    
+    newReplyFieldVariantSubset:
+        description: "new command when its reply field type is a variant type that is
+                      a subset of the old reply field variant types"
+        command_name: newReplyFieldVariantSubset
+        namespace: ignored
+        cpp_name: newReplyFieldVariantSubset
+        strict: true
+        api_version: "1"
+        reply_type: NewVariantSubsetReply
+
+    replyFieldVariantRecursive:
+        description: "new command passes when its reply field type is a variant type that is
+                      compatible with the old reply field variant type"
+        command_name: replyFieldVariantRecursive
+        namespace: ignored
+        cpp_name: replyFieldVariantRecursive
+        strict: true
+        api_version: "1"
+        reply_type: VariantRecursiveReply
+    
+    oldReplyFieldVariantStruct:
+        description: "new command passes if it doesn't have a variant struct type while the 
+                      old command does"
+        command_name: oldReplyFieldVariantStruct
+        namespace: ignored
+        cpp_name: oldReplyFieldVariantStruct
+        strict: true
+        api_version: "1"
+        reply_type: OldVariantStructReply
+
+    replyFieldVariantStructRecursive:
+        description: "new command passes when its reply field type has a variant struct type that is
+                      compatible with the old reply field variant struct type"
+        command_name: replyFieldVariantStructRecursive
+        namespace: ignored
+        cpp_name: replyFieldVariantStructRecursive
+        strict: true
+        api_version: "1"
+        reply_type: VariantStructRecursiveReply
diff --git a/buildscripts/idl/tests/test_compatibility.py b/buildscripts/idl/tests/test_compatibility.py
index 7a6d3b1fd12..fdc87cb3198 100644
--- a/buildscripts/idl/tests/test_compatibility.py
+++ b/buildscripts/idl/tests/test_compatibility.py
@@ -69,7 +69,7 @@ class TestIDLCompatibilityChecker(unittest.TestCase):
             path.join(dir_path, "compatibility_test_fail/new"), ["src"])
 
         self.assertTrue(error_collection.has_errors())
-        self.assertTrue(error_collection.count() == 34)
+        self.assertTrue(error_collection.count() == 39)
 
         invalid_api_version_new_error = error_collection.get_error_by_command_name(
             "invalidAPIVersionNew")
@@ -255,6 +255,38 @@ class TestIDLCompatibilityChecker(unittest.TestCase):
             idl_compatibility_errors.ERROR_ID_NEW_COMMAND_TYPE_FIELD_MISSING)
         self.assertRegex(str(new_type_field_missing_error), "newTypeFieldMissing")
 
+        new_reply_field_variant_type_error = error_collection.get_error_by_error_id(
+            idl_compatibility_errors.ERROR_ID_NEW_REPLY_FIELD_VARIANT_TYPE)
+        self.assertRegex(str(new_reply_field_variant_type_error), "newReplyFieldVariantType")
+
+        new_reply_field_variant_not_subset_error = error_collection.get_error_by_command_name(
+            "newReplyFieldVariantNotSubset")
+        self.assertTrue(new_reply_field_variant_not_subset_error.error_id ==
+                        idl_compatibility_errors.ERROR_ID_NEW_REPLY_FIELD_VARIANT_TYPE_NOT_SUBSET)
+        self.assertRegex(
+            str(new_reply_field_variant_not_subset_error), "newReplyFieldVariantNotSubset")
+
+        new_reply_field_variant_recursive_error = error_collection.get_error_by_command_name(
+            "replyFieldVariantRecursive")
+        self.assertTrue(new_reply_field_variant_recursive_error.error_id ==
+                        idl_compatibility_errors.ERROR_ID_COMMAND_NOT_SUBSET)
+        self.assertRegex(str(new_reply_field_variant_recursive_error), "replyFieldVariantRecursive")
+
+        new_reply_field_variant_struct_not_subset_error = error_collection.get_error_by_command_name(
+            "newReplyFieldVariantStructNotSubset")
+        self.assertTrue(new_reply_field_variant_struct_not_subset_error.error_id ==
+                        idl_compatibility_errors.ERROR_ID_NEW_REPLY_FIELD_VARIANT_TYPE_NOT_SUBSET)
+        self.assertRegex(
+            str(new_reply_field_variant_struct_not_subset_error),
+            "newReplyFieldVariantStructNotSubset")
+
+        new_reply_field_variant_struct_recursive_error = error_collection.get_error_by_command_name(
+            "replyFieldVariantStructRecursive")
+        self.assertTrue(new_reply_field_variant_struct_recursive_error.error_id ==
+                        idl_compatibility_errors.ERROR_ID_COMMAND_NOT_SUBSET)
+        self.assertRegex(
+            str(new_reply_field_variant_struct_recursive_error), "replyFieldVariantStructRecursive")
+
     def test_error_reply(self):
         """Tests the compatibility checker with the ErrorReply struct."""
         dir_path = path.dirname(path.realpath(__file__))