SERVER-64089 Ignore checking fields in IGNORE_UNSTABLE_LIST in IDL compatibility checker

3 files changed, 540 insertions, 42 deletions

diff --git a/buildscripts/idl/idl_check_compatibility.py b/buildscripts/idl/idl_check_compatibility.py
index 82428ef7e3b..9b6d0f2aaff 100644
--- a/buildscripts/idl/idl_check_compatibility.py
+++ b/buildscripts/idl/idl_check_compatibility.py
@@ -164,7 +164,29 @@ ALLOW_ANY_TYPE_LIST: List[str] = [
 ]
 
 # Do not add user visible fields already released in earlier versions.
-IGNORE_UNSTABLE_LIST: List[str] = [
+# We generally don't allow changing a field from stable to unstable, but we permit it in special cases,
+# such as when we want to avoid making internal fields part of the stable API.
+IGNORE_STABLE_TO_UNSTABLE_LIST: List[str] = [
+    # This list is only used in unit-tests.
+    'newReplyFieldUnstableIgnoreList-reply-unstableNewFieldIgnoreList',
+    'newTypeFieldUnstableIgnoreList-param-unstableNewFieldIgnoreList',
+    'newTypeEnumOrStructIgnoreList-reply-unstableNewFieldIgnoreList',
+    'commandParameterUnstableIgnoreList-param-newUnstableParameterIgnoreList',
+    'newReplyFieldUnstableOptionalIgnoreList-reply-unstableOptionalNewFieldIgnoreList',
+    'newReplyTypeEnumOrStructIgnoreList-reply-newReplyTypeEnumOrStructIgnoreList',
+    'newReplyFieldVariantNotSubsetIgnoreList-reply-variantNotSubsetReplyFieldIgnoreList',
+    'replyFieldVariantDifferentStructIgnoreList-reply-variantStructRecursiveReplyFieldIgnoreList',
+    'replyFieldNonVariantToVariantIgnoreList-reply-nonVariantToVariantReplyFieldIgnoreList',
+    'replyFieldNonEnumToEnumIgnoreList-reply-nonEnumToEnumReplyIgnoreList',
+    'newUnstableParamTypeChangesIgnoreList-param-newUnstableTypeChangesParamIgnoreList',
+    'newUnstableTypeChangesIgnoreList',
+    'newUnstableTypeChangesIgnoreList-param-newUnstableTypeChangesFieldIgnoreList',
+    'newUnstableReplyFieldTypeChangesIgnoreList-reply-newUnstableTypeChangesReplyFieldIgnoreList',
+    'newReplyFieldTypeStructIgnoreList-reply-structReplyField',
+    'newReplyFieldTypeStructIgnoreList-reply-unstableNewFieldIgnoreList',
+
+    # Real use cases for changing a field from 'stable' to 'unstable'.
+
     # The 'originalSpec' field was introduced in v5.1 behind a disabled feature flag and is not user
     # visible. This is part of the listIndexes output when executed against system.bucket.*
     # collections, which users should avoid doing.
@@ -350,11 +372,13 @@ def check_reply_field_type_recursive(ctxt: IDLCompatibilityContext,
     cmd_name = field_pair.cmd_name
     field_name = field_pair.field_name
 
+    ignore_list_name: str = cmd_name + "-reply-" + field_name
+
     # If the old field is unstable, we only add errors related to the use of 'any' as the
     # bson_serialization_type. For all other errors, we check that the old field is stable
     # before adding an error.
     if not isinstance(new_field_type, syntax.Type):
-        if not old_field.unstable:
+        if not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
             ctxt.add_new_reply_field_type_enum_or_struct_error(
                 cmd_name, field_name, new_field_type.name, old_field_type.name,
                 new_field.idl_file_path)
@@ -372,11 +396,9 @@ def check_reply_field_type_recursive(ctxt: IDLCompatibilityContext,
                                                 new_field_type.name, new_field.idl_file_path)
         return
 
-    allow_name: str = cmd_name + "-reply-" + field_name
-
     if "any" in old_field_type.bson_serialization_type:
         # If 'any' is not explicitly allowed as the bson_serialization_type.
-        if allow_name not in ALLOW_ANY_TYPE_LIST:
+        if ignore_list_name not in ALLOW_ANY_TYPE_LIST:
             ctxt.add_old_reply_field_bson_any_not_allowed_error(
                 cmd_name, field_name, old_field_type.name, old_field.idl_file_path)
             return
@@ -387,12 +409,12 @@ def check_reply_field_type_recursive(ctxt: IDLCompatibilityContext,
                                                           new_field.idl_file_path)
 
         # If serializer is changed, it's a potential breaking change.
-        if (not old_field.unstable) and old_field_type.serializer != new_field_type.serializer:
+        if not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST and old_field_type.serializer != new_field_type.serializer:
             ctxt.add_reply_field_serializer_not_equal_error(
                 cmd_name, field_name, new_field_type.name, new_field.idl_file_path)
 
         # If deserializer is changed, it's a potential breaking change.
-        if (not old_field.unstable) and old_field_type.deserializer != new_field_type.deserializer:
+        if not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST and old_field_type.deserializer != new_field_type.deserializer:
             ctxt.add_reply_field_deserializer_not_equal_error(
                 cmd_name, field_name, new_field_type.name, new_field.idl_file_path)
 
@@ -419,7 +441,7 @@ def check_reply_field_type_recursive(ctxt: IDLCompatibilityContext,
 
             else:
                 # new_variant_type was not found in old_variant_types.
-                if not old_field.unstable:
+                if not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
                     ctxt.add_new_reply_field_variant_type_not_subset_error(
                         cmd_name, field_name, new_variant_type.name, new_field.idl_file_path)
 
@@ -427,7 +449,7 @@ def check_reply_field_type_recursive(ctxt: IDLCompatibilityContext,
         # Since enums can't be part of variant types, we don't explicitly check for enums.
         if isinstance(new_field_type,
                       syntax.VariantType) and new_field_type.variant_struct_type is not None:
-            if old_field_type.variant_struct_type is None and not old_field.unstable:
+            if old_field_type.variant_struct_type is None and not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
                 ctxt.add_new_reply_field_variant_type_not_subset_error(
                     cmd_name, field_name, new_field_type.variant_struct_type.name,
                     new_field.idl_file_path)
@@ -437,7 +459,7 @@ def check_reply_field_type_recursive(ctxt: IDLCompatibilityContext,
                                    new_field.idl_file, old_field.idl_file_path,
                                    new_field.idl_file_path)
 
-    elif not old_field.unstable:
+    elif not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
         if isinstance(new_field_type, syntax.VariantType):
             ctxt.add_new_reply_field_variant_type_error(cmd_name, field_name, old_field_type.name,
                                                         new_field.idl_file_path)
@@ -452,6 +474,8 @@ def check_reply_field_type(ctxt: IDLCompatibilityContext, field_pair: FieldCompa
     # pylint: disable=too-many-branches
     old_field = field_pair.old
     new_field = field_pair.new
+    cmd_name = field_pair.cmd_name
+    field_name = field_pair.field_name
     array_check = check_array_type(ctxt, "reply_field", old_field.field_type, new_field.field_type,
                                    field_pair.cmd_name, 'type', old_field.idl_file_path,
                                    new_field.idl_file_path, old_field.unstable)
@@ -475,10 +499,14 @@ def check_reply_field_type(ctxt: IDLCompatibilityContext, field_pair: FieldCompa
         ctxt.errors.dump_errors()
         sys.exit(1)
 
+    ignore_list_name: str = cmd_name + "-reply-" + field_name
+
     if isinstance(old_field_type, syntax.Type):
         check_reply_field_type_recursive(ctxt, field_pair)
 
-    elif isinstance(old_field_type, syntax.Enum) and not old_field.unstable:
+    elif isinstance(
+            old_field_type, syntax.Enum
+    ) and not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
         if isinstance(new_field_type, syntax.Enum):
             check_subset(ctxt, cmd_name, field_name, new_field_type.name, new_field_type.values,
                          old_field_type.values, new_field.idl_file_path)
@@ -491,7 +519,7 @@ def check_reply_field_type(ctxt: IDLCompatibilityContext, field_pair: FieldCompa
             check_reply_fields(ctxt, old_field_type, new_field_type, cmd_name, old_field.idl_file,
                                new_field.idl_file, old_field.idl_file_path, new_field.idl_file_path)
         else:
-            if not old_field.unstable:
+            if not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
                 ctxt.add_new_reply_field_type_not_struct_error(
                     cmd_name, field_name, new_field_type.name, old_field_type.name,
                     new_field.idl_file_path)
@@ -536,9 +564,9 @@ def check_reply_field(ctxt: IDLCompatibilityContext, old_field: syntax.Field,
                                                 and old_field_type.name == "optionalBool")
     new_field_optional = new_field.optional or (new_field_type
                                                 and new_field_type.name == "optionalBool")
-    field_name: str = cmd_name + "-reply-" + new_field.name
-    if not old_field.unstable and field_name not in IGNORE_UNSTABLE_LIST:
-        if new_field.unstable and field_name not in IGNORE_UNSTABLE_LIST:
+    ignore_list_name: str = cmd_name + "-reply-" + new_field.name
+    if not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
+        if new_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
             ctxt.add_new_reply_field_unstable_error(cmd_name, new_field.name, new_idl_file_path)
         if new_field_optional and not old_field_optional:
             ctxt.add_new_reply_field_optional_error(cmd_name, new_field.name, new_idl_file_path)
@@ -652,19 +680,19 @@ def check_param_or_command_type_recursive(ctxt: IDLCompatibilityContext,
     cmd_name = field_pair.cmd_name
     param_name = field_pair.field_name
 
+    ignore_list_name: str = cmd_name + "-param-" + param_name if is_command_parameter else cmd_name
+
     # If the old field is unstable, we only add errors related to the use of 'any' as the
     # bson_serialization_type. For all other errors, we check that the old field is stable
     # before adding an error.
 
     if not isinstance(new_type, syntax.Type):
-        if not old_field.unstable:
+        if not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
             ctxt.add_new_command_or_param_type_enum_or_struct_error(
                 cmd_name, new_type.name, old_type.name, new_field.idl_file_path, param_name,
                 is_command_parameter)
         return
 
-    allow_name: str = cmd_name + "-param-" + param_name if is_command_parameter else cmd_name
-
     # If bson_serialization_type switches from 'any' to non-any type.
     if "any" in old_type.bson_serialization_type and "any" not in new_type.bson_serialization_type:
         ctxt.add_old_command_or_param_type_bson_any_error(cmd_name, old_type.name, new_type.name,
@@ -681,7 +709,7 @@ def check_param_or_command_type_recursive(ctxt: IDLCompatibilityContext,
 
     if "any" in old_type.bson_serialization_type:
         # If 'any' is not explicitly allowed as the bson_serialization_type.
-        if allow_name not in ALLOW_ANY_TYPE_LIST:
+        if ignore_list_name not in ALLOW_ANY_TYPE_LIST:
             ctxt.add_old_command_or_param_type_bson_any_not_allowed_error(
                 cmd_name, old_type.name, old_field.idl_file_path, param_name, is_command_parameter)
             return
@@ -692,18 +720,20 @@ def check_param_or_command_type_recursive(ctxt: IDLCompatibilityContext,
                 cmd_name, new_type.name, new_field.idl_file_path, param_name, is_command_parameter)
 
         # If serializer is changed, it's a potential breaking change.
-        if (not old_field.unstable) and old_type.serializer != new_type.serializer:
+        if (not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST
+            ) and old_type.serializer != new_type.serializer:
             ctxt.add_command_or_param_serializer_not_equal_error(
                 cmd_name, new_type.name, new_field.idl_file_path, param_name, is_command_parameter)
 
         # If deserializer is changed, it's a potential breaking change.
-        if (not old_field.unstable) and old_type.deserializer != new_type.deserializer:
+        if (not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST
+            ) and old_type.deserializer != new_type.deserializer:
             ctxt.add_command_or_param_deserializer_not_equal_error(
                 cmd_name, new_type.name, new_field.idl_file_path, param_name, is_command_parameter)
 
     if isinstance(old_type, syntax.VariantType):
         if not isinstance(new_type, syntax.VariantType):
-            if not old_field.unstable:
+            if not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
                 ctxt.add_new_command_or_param_type_not_variant_type_error(
                     cmd_name, new_type.name, new_field.idl_file_path, param_name,
                     is_command_parameter)
@@ -730,7 +760,7 @@ def check_param_or_command_type_recursive(ctxt: IDLCompatibilityContext,
                             is_command_parameter)
                         break
                 else:
-                    if not old_field.unstable:
+                    if not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
                         # old_variant_type was not found in new_variant_types.
                         ctxt.add_new_command_or_param_variant_type_not_superset_error(
                             cmd_name, old_variant_type.name, new_field.idl_file_path, param_name,
@@ -746,12 +776,12 @@ def check_param_or_command_type_recursive(ctxt: IDLCompatibilityContext,
                         new_field.idl_file_path, is_command_parameter)
 
                 # If old type has a variant struct type and new type does not have a variant struct type.
-                elif not old_field.unstable:
+                elif not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
                     ctxt.add_new_command_or_param_variant_type_not_superset_error(
                         cmd_name, old_type.variant_struct_type.name, new_field.idl_file_path,
                         param_name, is_command_parameter)
 
-    elif not old_field.unstable:
+    elif not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
         check_superset(ctxt, cmd_name, new_type.name, new_type.bson_serialization_type,
                        old_type.bson_serialization_type, new_field.idl_file_path, param_name,
                        is_command_parameter)
@@ -763,10 +793,12 @@ def check_param_or_command_type(ctxt: IDLCompatibilityContext, field_pair: Field
     # pylint: disable=too-many-branches
     old_field = field_pair.old
     new_field = field_pair.new
+    field_name = field_pair.field_name
+    cmd_name = field_pair.cmd_name
     array_check = check_array_type(
         ctxt, "command_parameter" if is_command_parameter else "command_namespace",
         old_field.field_type, new_field.field_type, field_pair.cmd_name,
-        field_pair.field_name if is_command_parameter else "type", old_field.idl_file_path,
+        field_name if is_command_parameter else "type", old_field.idl_file_path,
         new_field.idl_file_path, old_field.unstable)
     if array_check == ArrayTypeCheckResult.INVALID:
         return
@@ -778,40 +810,42 @@ def check_param_or_command_type(ctxt: IDLCompatibilityContext, field_pair: Field
     old_type = old_field.field_type
     new_type = new_field.field_type
     if old_type is None:
-        ctxt.add_command_or_param_type_invalid_error(field_pair.cmd_name, old_field.idl_file_path,
+        ctxt.add_command_or_param_type_invalid_error(cmd_name, old_field.idl_file_path,
                                                      field_pair.field_name, is_command_parameter)
         ctxt.errors.dump_errors()
         sys.exit(1)
     if new_type is None:
-        ctxt.add_command_or_param_type_invalid_error(field_pair.cmd_name, new_field.idl_file_path,
+        ctxt.add_command_or_param_type_invalid_error(cmd_name, new_field.idl_file_path,
                                                      field_pair.field_name, is_command_parameter)
         ctxt.errors.dump_errors()
         sys.exit(1)
 
+    ignore_list_name: str = cmd_name + "-param-" + field_name
+
     if isinstance(old_type, syntax.Type):
         check_param_or_command_type_recursive(ctxt, field_pair, is_command_parameter)
 
     # Only add type errors if the old field is stable.
-    elif isinstance(old_type, syntax.Enum) and not old_field.unstable:
+    elif isinstance(
+            old_type, syntax.Enum
+    ) and not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
         if isinstance(new_type, syntax.Enum):
-            check_superset(ctxt, field_pair.cmd_name, new_type.name, new_type.values,
-                           old_type.values, new_field.idl_file_path, field_pair.field_name,
-                           is_command_parameter)
+            check_superset(ctxt, cmd_name, new_type.name, new_type.values, old_type.values,
+                           new_field.idl_file_path, field_pair.field_name, is_command_parameter)
         else:
             ctxt.add_new_command_or_param_type_not_enum_error(
-                field_pair.cmd_name, new_type.name, old_type.name, new_field.idl_file_path,
+                cmd_name, new_type.name, old_type.name, new_field.idl_file_path,
                 field_pair.field_name, is_command_parameter)
 
     elif isinstance(old_type, syntax.Struct):
         if isinstance(new_type, syntax.Struct):
             check_command_params_or_type_struct_fields(
-                ctxt, old_type, new_type, field_pair.cmd_name, old_field.idl_file,
-                new_field.idl_file, old_field.idl_file_path, new_field.idl_file_path,
-                is_command_parameter)
+                ctxt, old_type, new_type, cmd_name, old_field.idl_file, new_field.idl_file,
+                old_field.idl_file_path, new_field.idl_file_path, is_command_parameter)
         else:
-            if not old_field.unstable:
+            if not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
                 ctxt.add_new_command_or_param_type_not_struct_error(
-                    field_pair.cmd_name, new_type.name, old_type.name, new_field.idl_file_path,
+                    cmd_name, new_type.name, old_type.name, new_field.idl_file_path,
                     field_pair.field_name, is_command_parameter)
 
 
@@ -948,8 +982,8 @@ def check_command_param_or_type_struct_field(
         is_command_parameter: bool):
     """Check compatibility between the old and new command parameter or command type struct field."""
     # pylint: disable=too-many-arguments
-    field_name: str = cmd_name + "-param-" + new_field.name
-    if not old_field.unstable and new_field.unstable and field_name not in IGNORE_UNSTABLE_LIST:
+    ignore_list_name: str = cmd_name + "-param-" + new_field.name
+    if not old_field.unstable and new_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
         ctxt.add_new_param_or_command_type_field_unstable_error(
             cmd_name, old_field.name, old_idl_file_path, type_name, is_command_parameter)
     # If old field is unstable and new field is stable, the new field should either be optional or
@@ -964,11 +998,11 @@ def check_command_param_or_type_struct_field(
         ctxt.add_new_param_or_command_type_field_stable_required_no_default_error(
             cmd_name, old_field.name, old_idl_file_path, type_name, is_command_parameter)
 
-    if old_field_optional and not new_field_optional:
+    if not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST and old_field_optional and not new_field_optional:
         ctxt.add_new_param_or_command_type_field_required_error(
             cmd_name, old_field.name, old_idl_file_path, type_name, is_command_parameter)
 
-    if not old_field.unstable:
+    if not old_field.unstable and ignore_list_name not in IGNORE_STABLE_TO_UNSTABLE_LIST:
         check_param_or_type_validator(ctxt, old_field, new_field, cmd_name, new_idl_file_path,
                                       type_name, is_command_parameter)
 
diff --git a/buildscripts/idl/tests/compatibility_test_pass/new/compatibility_test_pass_new.idl b/buildscripts/idl/tests/compatibility_test_pass/new/compatibility_test_pass_new.idl
index da2011a57b9..5302dd01ed0 100644
--- a/buildscripts/idl/tests/compatibility_test_pass/new/compatibility_test_pass_new.idl
+++ b/buildscripts/idl/tests/compatibility_test_pass/new/compatibility_test_pass_new.idl
@@ -100,6 +100,22 @@ structs:
                 type: string
                 unstable: false
 
+    UnstableNewFieldReplyIgnoreList:
+        description: "This reply contains a field that is stable in the old command and is
+                      unstable in the new command, but the change is explicitly allowed."
+        fields:
+            unstableNewFieldIgnoreList:
+                type: string
+                unstable: true
+
+    NewReplyTypeEnumOrStructIgnoreList:
+        description: "the type is a non-enum or struct type in the old command, and an enum or struct
+                      in the new command, but the change is explicitly allowed"
+        fields:
+            newReplyTypeEnumOrStructIgnoreList:
+                type: StructType
+                unstable: true
+
     RequiredNewFieldReply:
         description: "This reply contains a field that is optional in the old command but is
                       required in the new command."
@@ -133,6 +149,15 @@ structs:
                 unstable: true
                 optional: true
 
+    UnstableOptionalNewFieldReplyIgnoreList:
+        description: "This reply contains a field that is stable in the old command and is
+                      unstable and optional in the new command, but the change is allowed"
+        fields:
+            unstableOptionalNewFieldIgnoreList:
+                type: string
+                unstable: true
+                optional: true
+
     EnumSubsetReply:
         description: "This reply contains an enum field where the new enum values is a subset
                       of the old reply type's enum values"
@@ -157,6 +182,14 @@ structs:
                 type: intStringBoolToIntString
                 unstable: false
 
+    StructFieldTypeRecursiveReplyIgnoreList:
+        description: "This reply contains a field whose new type is a struct that is not
+                      compatible with the old field type, but this is explicitly allowed"
+        fields:
+            structReplyField:
+                type: UnstableNewFieldReplyIgnoreList
+                unstable: true
+
     StructFieldTypeRecursiveReplyOne:
         description: "This reply contains a field whose new type is a struct that is
                       compatible with the old field type"
@@ -189,6 +222,15 @@ structs:
                 type: int
                 unstable: false
 
+    NewVariantNotSubsetReplyIgnoreList:
+        description: "This reply contains a field whose new variant types are not a subset
+                      of the old variant types"
+        fields:
+            variantNotSubsetReplyFieldIgnoreList:
+                type:
+                    variant: [int, string]
+                unstable: true
+
     NewVariantSubsetReply:
         description: "This reply contains a field whose new variant types are a subset
                       of the old variant types"
@@ -236,6 +278,32 @@ structs:
                     variant: [int, StructFieldTypeRecursiveReplyTwo,
                               array<StructFieldTypeRecursiveReplyTwo>, array<string>]
 
+    VariantDifferentStructReplyIgnoreList:
+        description: "This reply contains a field that has a new variant struct type that is
+                      different from the old variant struct type"
+        fields:
+            variantStructRecursiveReplyFieldIgnoreList:
+                unstable: true
+                type:
+                    variant: [int, StructFieldTypeRecursiveReplyTwo]
+
+    NonVariantToVariantReplyIgnoreList:
+        description: "This reply contains a field that changes from a non-variant type to a variant
+                      type, but the field is in the ignore list"
+        fields:
+            nonVariantToVariantReplyFieldIgnoreList:
+                unstable: true
+                type:
+                    variant: [int, StructFieldTypeRecursiveReplyOne]
+
+    NonEnumToEnumReplyIgnoreList:
+        description: "This reply contains a field that changes from a non-enum type to an enum
+                      type, but the field is in the ignore list"
+        fields:
+            nonEnumToEnumReplyIgnoreList:
+                type: EnumSubsetReply
+                unstable: true
+
     CommandParamStructRecursiveOne:
         description: "This command parameter struct type contains a stable and optional
                       field while the old struct field is unstable"
@@ -412,6 +480,16 @@ structs:
                 validator:
                     lt: 0
 
+    NewUnstableTypeChangesReplyIgnoreList:
+        description: "This reply contains a field that is stable in the old version and has type changes,
+                      but is also in the ignore list"
+        fields:
+            newUnstableTypeChangesReplyFieldIgnoreList:
+                unstable: true
+                type: intStringToIntStringBool
+                validator:
+                    lt: 0
+
     NewlyAddedBsonSerializationTypeAnyStruct:
         description: "This struct contains a newly added field whose type has a bson_serialization_type
                         that contains 'any' that is explicitly allowed"
@@ -431,6 +509,16 @@ structs:
                 validator:
                     lt: 0
 
+    NewUnstableTypeChangesStructIgnoreList:
+        description: "This struct contains a field that is stable in the old version and has type,
+                      changes but is also in the ignore list"
+        fields:
+            newUnstableTypeChangesFieldIgnoreList:
+                type: intStringBoolToIntString
+                unstable: true
+                validator:
+                    lt: 0
+
     BsonSerializationTypeAnyWithVariantReply:
         description: "This reply contains a new reply field with variant types where one of the
                       bson serialization types is 'any' and is explicitly allowed"
@@ -616,6 +704,20 @@ commands:
                 default: ""
                 unstable: false
 
+    commandParameterUnstableIgnoreList:
+        description: "new unstable command parameter is stable in the corresponding old
+                      command, but the change is explcitly allowed"
+        command_name: commandParameterUnstableIgnoreList
+        namespace: ignored
+        cpp_name: commandParameterUnstableIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: OkReply
+        fields:
+            newUnstableParameterIgnoreList:
+                type: string
+                unstable: true
+
     removeCommandParameterUnstable:
         description: "new command removes parameter that is unstable
                       in the corresponding old command and still passes"
@@ -785,6 +887,22 @@ commands:
                 validator:
                     lt: 0
 
+    newUnstableParamTypeChangesIgnoreList:
+        description: "command has param with incompatible type changes, but is in the stable-to-unstable
+                      ignore list"
+        command_name: newUnstableParamTypeChangesIgnoreList
+        namespace: ignored
+        cpp_name: newUnstableParamTypeChangesIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: OkReply
+        fields:
+            newUnstableTypeChangesParamIgnoreList:
+                type: intStringBoolToIntString
+                unstable: true
+                validator:
+                    lt: 0
+
     newlyAddedTypeFieldBsonAnyAllowList:
         description: "command passes when its type field is newly added and has bson type 'any'
                       that is explicitly allowed"
@@ -806,6 +924,17 @@ commands:
         api_version: "1"
         reply_type: OkReply
 
+    newUnstableTypeChangesIgnoreList:
+        description: "command has type with incompatible changes, but is in the stable-to-unstable
+                      ignore list"
+        command_name: newUnstableTypeChangesIgnoreList
+        namespace: type
+        type: NewUnstableTypeChangesStructIgnoreList
+        cpp_name: newUnstableTypeChangesIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: OkReply
+
     oldCommandParameterValidator:
         description: "new command passes when it contains a parameter that does not contain a validator
                       that is present in the old parameter"
@@ -872,6 +1001,16 @@ commands:
         api_version: "1"
         reply_type: StableNewFieldReply
 
+    newReplyFieldUnstableIgnoreList:
+        description: "new command contains an unstable reply field that is stable
+                      in the corresponding old command but is explicitly allowed"
+        command_name: newReplyFieldUnstableIgnoreList
+        namespace: ignored
+        cpp_name: newReplyFieldUnstableIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: UnstableNewFieldReplyIgnoreList
+
     newReplyFieldRequired:
         description: "new command contains a required reply field that is optional
                       in the corresponding old command and still passes"
@@ -901,6 +1040,16 @@ commands:
         api_version: "1"
         reply_type: UnstableOldFieldReply
 
+    newReplyFieldUnstableOptionalIgnoreList:
+        description: "old reply field is stable but is included in the ignore list so new commmand 
+                      passes even if its new reply field is unstable and optional"
+        command_name: newReplyFieldUnstableOptionalIgnoreList
+        namespace: ignored
+        cpp_name: newReplyFieldUnstableOptionalIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: UnstableOptionalNewFieldReplyIgnoreList
+
     importedReplyCommand:
         description: "reply is imported and should pass"
         command_name: importedReplyCommand
@@ -962,6 +1111,16 @@ commands:
         api_version: "1"
         reply_type: StructFieldTypeRecursiveReplyTwo
 
+    newReplyFieldTypeStructIgnoreList:
+        description: "command has a reply contains a field whose new type is a struct that is not
+                      compatible with the old field type, but this is explicitly allowed"
+        command_name: newReplyFieldTypeStructIgnoreList
+        namespace: ignored
+        cpp_name: newReplyFieldTypeStructIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: StructFieldTypeRecursiveReplyIgnoreList
+
     newNamespaceIgnored:
         description: "new command passes when its namespace is changed to ignored"
         command_name: newNamespaceIgnored
@@ -1032,6 +1191,17 @@ commands:
         api_version: "1"
         reply_type: OkReply
 
+    newReplyTypeEnumOrStructIgnoreList:
+        description: "the type is a non-enum or struct type in the old command, and an enum or struct
+                     in the new command, but the change is explicitly allowed"
+        command_name: newReplyTypeEnumOrStructIgnoreList
+        namespace: type
+        type: namespacestring
+        cpp_name: newReplyTypeEnumOrStructIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: NewReplyTypeEnumOrStructIgnoreList
+
     newTypeFieldOptional:
         description: "new command type contains an optional field that is required
                       in the corresponding old command and still passes"
@@ -1075,6 +1245,17 @@ commands:
         api_version: "1"
         reply_type: OkReply
 
+    newTypeFieldUnstableIgnoreList:
+        description: "new command contains an unstable type field that is stable in the corresponding
+                      old command but that is explicitly allowed"
+        command_name: newTypeFieldUnstableIgnoreList
+        namespace: type
+        type: UnstableNewFieldReplyIgnoreList
+        cpp_name: newTypeFieldUnstableIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: OkReply
+
     removeTypeFieldUnstable:
         description: "new command removes type field that is unstable
                       in the corresponding old command and still passes"
@@ -1118,6 +1299,16 @@ commands:
         api_version: "1"
         reply_type: OldVariantTypeReply
 
+    newReplyFieldVariantNotSubsetIgnoreList:
+        description: "the command's reply field type is a variant type that is not a subset of the old reply
+                      field variant types, but it's also on the unstable ignore list"
+        command_name: newReplyFieldVariantNotSubsetIgnoreList
+        namespace: ignored
+        cpp_name: newReplyFieldVariantNotSubsetIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: NewVariantNotSubsetReplyIgnoreList
+
     newReplyFieldVariantSubset:
         description: "new command when its reply field type is a variant type that is
                       a subset of the old reply field variant types"
@@ -1168,6 +1359,36 @@ commands:
         api_version: "1"
         reply_type: VariantStructRecursiveReply
 
+    replyFieldVariantDifferentStructIgnoreList:
+        description: "the old field has a non-variant type and the new field has a variant type but the
+                      change is explicitly allowed"
+        command_name: replyFieldVariantDifferentStructIgnoreList
+        namespace: ignored
+        cpp_name: replyFieldVariantDifferentStructIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: VariantDifferentStructReplyIgnoreList
+
+    replyFieldNonVariantToVariantIgnoreList:
+        description: "the old and new field have different variant struct types but the change is
+                      explicitly allowed"
+        command_name: replyFieldNonVariantToVariantIgnoreList
+        namespace: ignored
+        cpp_name: replyFieldNonVariantToVariantIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: NonVariantToVariantReplyIgnoreList
+
+    replyFieldNonEnumToEnumIgnoreList:
+        description: "the old field has a non-enum type and the new field has an enum type but the
+                      change is explicitly allowed"
+        command_name: replyFieldNonEnumToEnumIgnoreList
+        namespace: ignored
+        cpp_name: replyFieldNonEnumToEnumIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: NonEnumToEnumReplyIgnoreList
+
     newlyAddedReplyFieldTypeBsonAnyAllowed:
         description: "command passes because it has a newly added reply field type has a bson_serialization_type
                       that contains 'any' that is explicitly allowed"
@@ -1187,6 +1408,16 @@ commands:
         api_version: "1"
         reply_type: OldUnstableTypeChangesReply
 
+    newUnstableReplyFieldTypeChangesIgnoreList:
+        description: "command has an old stable reply field with incompatible type changes but it is also
+                      in the ignore list"
+        command_name: newUnstableReplyFieldTypeChangesIgnoreList
+        namespace: ignored
+        cpp_name: newUnstableReplyFieldTypeChangesIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: NewUnstableTypeChangesReplyIgnoreList
+
     commandAllowedAnyTypes:
         description: "new command that has parameter and reply type with
                       explicitly allowed 'any' bson serialization type passes"
diff --git a/buildscripts/idl/tests/compatibility_test_pass/old/compatibility_test_pass_old.idl b/buildscripts/idl/tests/compatibility_test_pass/old/compatibility_test_pass_old.idl
index 704ea71b0a1..f442e45a20e 100644
--- a/buildscripts/idl/tests/compatibility_test_pass/old/compatibility_test_pass_old.idl
+++ b/buildscripts/idl/tests/compatibility_test_pass/old/compatibility_test_pass_old.idl
@@ -101,6 +101,22 @@ structs:
                 type: string
                 unstable: true
 
+    UnstableNewFieldReplyIgnoreList:
+        description: "This reply contains a field that is stable in the old command and is
+                      unstable in the new command, but the change is explicitly allowed."
+        fields:
+            unstableNewFieldIgnoreList:
+                type: string
+                unstable: false
+
+    NewReplyTypeEnumOrStructIgnoreList:
+        description: "the type is a non-enum or struct type in the old command, and an enum or struct
+                      in the new command, but the change is explicitly allowed"
+        fields:
+            newReplyTypeEnumOrStructIgnoreList:
+                type: string
+                unstable: false
+
     RequiredNewFieldReply:
         description: "This reply contains a field that is optional in the old command but is
                       required in the new command."
@@ -129,6 +145,14 @@ structs:
                 type: string
                 unstable: true
 
+    UnstableOptionalNewFieldReplyIgnoreList:
+        description: "This reply contains a field that is stable in the old command and is
+                      unstable and optional in the new command, but the change is allowed"
+        fields:
+            unstableOptionalNewFieldIgnoreList:
+                unstable: false
+                type: string
+
     EnumSubsetReply:
         description: "This reply contains an enum field where the new enum values is a subset
                       of the old reply type's enum values"
@@ -153,6 +177,14 @@ structs:
                 type: intStringBoolToIntString
                 unstable: false
 
+    StructFieldTypeRecursiveReplyIgnoreList:
+        description: "This reply contains a field whose new type is a struct that is not
+                      compatible with the old field type, but this is explicitly allowed"
+        fields:
+            structReplyField:
+                type: UnstableNewFieldReplyIgnoreList
+                unstable: false
+
     StructFieldTypeRecursiveReplyOne:
         description: "This reply contains a field whose new type is a struct that is
                       compatible with the old field type"
@@ -177,6 +209,14 @@ structs:
                 type: BsonSubsetReply
                 unstable: false
 
+    StructTypeUnstable:
+        description: "This struct contains a field whose new type is compatible with the
+                      old field type and is unstable in both the old and new versions"
+        fields:
+            fieldOne:
+                type: BsonSubsetReply
+                unstable: true
+
     OldVariantTypeReply:
         description: "This reply contains an old field that has a variant type while the new field
                       is not a variant type"
@@ -186,6 +226,15 @@ structs:
                 type:
                     variant: [int, string, array<string>]
 
+    NewVariantNotSubsetReplyIgnoreList:
+        description: "This reply contains a field whose new variant types are not a subset
+                      of the old variant types"
+        fields:
+            variantNotSubsetReplyFieldIgnoreList:
+                unstable: false
+                type:
+                    variant: [int, bool, string]
+
     NewVariantSubsetReply:
         description: "This reply contains a field whose new variant types are a subset
                       of the old variant types"
@@ -233,6 +282,31 @@ structs:
                     variant: [int, StructFieldTypeRecursiveReplyTwo,
                               array<StructFieldTypeRecursiveReplyTwo>, array<string>]
 
+    VariantDifferentStructReplyIgnoreList:
+        description: "This reply contains a field that has a new variant struct type that is
+                      different from the old variant struct type"
+        fields:
+            variantStructRecursiveReplyFieldIgnoreList:
+                unstable: false
+                type:
+                    variant: [int, StructFieldTypeRecursiveReplyOne]
+
+    NonVariantToVariantReplyIgnoreList:
+        description: "This reply contains a field that changes from a non-variant type to a variant
+                      type, but the field is in the ignore list"
+        fields:
+            nonVariantToVariantReplyFieldIgnoreList:
+                unstable: false
+                type: StructType
+
+    NonEnumToEnumReplyIgnoreList:
+        description: "This reply contains a field that changes from a non-enum type to an enum
+                      type, but the field is in the ignore list"
+        fields:
+            nonEnumToEnumReplyIgnoreList:
+                type: StructTypeUnstable
+                unstable: false
+
     NewCommandParameterStruct:
         description: "The new command parameter's type and the
                      old command parameter's type are both structs"
@@ -396,6 +470,15 @@ structs:
                 unstable: true
                 type: intStringToIntStringBool
 
+    NewUnstableTypeChangesReplyIgnoreList:
+        description: "This reply contains a field that is stable in the old version and has type changes,
+                      but is also in the ignore list"
+        fields:
+            newUnstableTypeChangesReplyFieldIgnoreList:
+                unstable: false
+                type: intStringToIntStringBool
+                optional: true
+
     NewlyAddedBsonSerializationTypeAnyStruct:
         description: "This struct contains a newly added field whose type has a bson_serialization_type
                         that contains 'any' that is explicitly allowed"
@@ -407,6 +490,15 @@ structs:
                 unstable: true
                 type: intStringBoolToIntString
 
+    NewUnstableTypeChangesStructIgnoreList:
+        description: "This struct contains a field that is stable in the old version and has type,
+                      changes but is also in the ignore list"
+        fields:
+            newUnstableTypeChangesFieldIgnoreList:
+                type: intStringBoolToIntString
+                unstable: false
+                optional: true
+
     BsonSerializationTypeAnyWithVariantReply:
         description: "This reply contains a new reply field with variant types where one of the
                       bson serialization types is 'any' and is explicitly allowed"
@@ -577,6 +669,20 @@ commands:
                 type: string
                 unstable: true
 
+    commandParameterUnstableIgnoreList:
+        description: "new unstable command parameter is stable in the corresponding old
+                      command, but the change is explcitly allowed"
+        command_name: commandParameterUnstableIgnoreList
+        namespace: ignored
+        cpp_name: commandParameterUnstableIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: OkReply
+        fields:
+            newUnstableParameterIgnoreList:
+                type: string
+                unstable: false
+
     removeCommandParameterUnstable:
         description: "new command removes parameter that is unstable
                       in the corresponding old command and still passes"
@@ -738,6 +844,21 @@ commands:
                 type: intStringBoolToIntString
                 unstable: true
 
+    newUnstableParamTypeChangesIgnoreList:
+        description: "command has param with incompatible type changes, but is in the stable-to-unstable
+                      ignore list"
+        command_name: newUnstableParamTypeChangesIgnoreList
+        namespace: ignored
+        cpp_name: newUnstableParamTypeChangesIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: OkReply
+        fields:
+            newUnstableTypeChangesParamIgnoreList:
+                type: intStringBoolToIntString
+                unstable: false
+                optional: true
+
     newlyAddedTypeFieldBsonAnyAllowList:
         description: "command passes when its type field is newly added and has bson type 'any'
                       that is explicitly allowed"
@@ -759,6 +880,17 @@ commands:
         api_version: "1"
         reply_type: OkReply
 
+    newUnstableTypeChangesIgnoreList:
+        description: "command has type with incompatible changes, but is in the stable-to-unstable
+                      ignore list"
+        command_name: newUnstableTypeChangesIgnoreList
+        namespace: type
+        type: NewUnstableTypeChangesStructIgnoreList
+        cpp_name: newUnstableTypeChangesIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: OkReply
+
     oldCommandParameterValidator:
         description: "new command passes when it contains a parameter that does not contain a validator
                       that is present in the old parameter"
@@ -827,6 +959,16 @@ commands:
         api_version: "1"
         reply_type: StableNewFieldReply
 
+    newReplyFieldUnstableIgnoreList:
+        description: "new command contains an unstable reply field that is stable
+                      in the corresponding old command but is explicitly allowed"
+        command_name: newReplyFieldUnstableIgnoreList
+        namespace: ignored
+        cpp_name: newReplyFieldUnstableIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: UnstableNewFieldReplyIgnoreList
+
     newReplyFieldRequired:
         description: "new command contains a required reply field that is optional
                       in the corresponding old command and still passes"
@@ -856,6 +998,16 @@ commands:
         api_version: "1"
         reply_type: UnstableOldFieldReply
 
+    newReplyFieldUnstableOptionalIgnoreList:
+        description: "old reply field is stable but is included in the ignore list so new commmand 
+                      passes even if its new reply field is unstable and optional"
+        command_name: newReplyFieldUnstableOptionalIgnoreList
+        namespace: ignored
+        cpp_name: newReplyFieldUnstableOptionalIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: UnstableOptionalNewFieldReplyIgnoreList
+
     importedReplyCommand:
         description: "reply is imported"
         command_name: importedReplyCommand
@@ -917,6 +1069,16 @@ commands:
         api_version: "1"
         reply_type: StructFieldTypeRecursiveReplyTwo
 
+    newReplyFieldTypeStructIgnoreList:
+        description: "command has a reply contains a field whose new type is a struct that is not
+                      compatible with the old field type, but this is explicitly allowed"
+        command_name: newReplyFieldTypeStructIgnoreList
+        namespace: ignored
+        cpp_name: newReplyFieldTypeStructIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: StructFieldTypeRecursiveReplyIgnoreList
+
     newNamespaceIgnored:
         description: "new command passes when its namespace is changed to ignored"
         command_name: newNamespaceIgnored
@@ -990,6 +1152,17 @@ commands:
         api_version: "1"
         reply_type: OkReply
 
+    newReplyTypeEnumOrStructIgnoreList:
+        description: "the type is a non-enum or struct type in the old command, and an enum or struct
+                     in the new command, but the change is explicitly allowed"
+        command_name: newReplyTypeEnumOrStructIgnoreList
+        namespace: type
+        type: namespacestring
+        cpp_name: newReplyTypeEnumOrStructIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: NewReplyTypeEnumOrStructIgnoreList
+
     newTypeFieldOptional:
         description: "new command type contains an optional field that is required
                       in the corresponding old command and still passes"
@@ -1033,6 +1206,17 @@ commands:
         api_version: "1"
         reply_type: OkReply
 
+    newTypeFieldUnstableIgnoreList:
+        description: "new command contains an unstable type field that is stable in the corresponding
+                      old command but that is explicitly allowed"
+        command_name: newTypeFieldUnstableIgnoreList
+        namespace: type
+        type: UnstableNewFieldReplyIgnoreList
+        cpp_name: newTypeFieldUnstableIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: OkReply
+
     removeTypeFieldUnstable:
         description: "new command removes type field that is unstable
                       in the corresponding old command and still passes"
@@ -1076,6 +1260,15 @@ commands:
         api_version: "1"
         reply_type: OldVariantTypeReply
 
+    newReplyFieldVariantNotSubsetIgnoreList:
+        description: "the command's reply field type is a variant type that is not a subset of the old reply field variant types"
+        command_name: newReplyFieldVariantNotSubsetIgnoreList
+        namespace: ignored
+        cpp_name: newReplyFieldVariantNotSubsetIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: NewVariantNotSubsetReplyIgnoreList
+
     newReplyFieldVariantSubset:
         description: "new command when its reply field type is a variant type that is
                       a subset of the old reply field variant types"
@@ -1126,6 +1319,36 @@ commands:
         api_version: "1"
         reply_type: VariantStructRecursiveReply
 
+    replyFieldVariantDifferentStructIgnoreList:
+        description: "the old field has a non-variant type and the new field has a variant type but the
+                      change is explicitly allowed"
+        command_name: replyFieldVariantDifferentStructIgnoreList
+        namespace: ignored
+        cpp_name: replyFieldVariantDifferentStructIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: VariantDifferentStructReplyIgnoreList
+
+    replyFieldNonVariantToVariantIgnoreList:
+        description: "the old and new field have different variant struct types but the change is
+                      explicitly allowed"
+        command_name: replyFieldNonVariantToVariantIgnoreList
+        namespace: ignored
+        cpp_name: replyFieldNonVariantToVariantIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: NonVariantToVariantReplyIgnoreList
+
+    replyFieldNonEnumToEnumIgnoreList:
+        description: "the old field has a non-enum type and the new field has an enum type but the
+                      change is explicitly allowed"
+        command_name: replyFieldNonEnumToEnumIgnoreList
+        namespace: ignored
+        cpp_name: replyFieldNonEnumToEnumIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: NonEnumToEnumReplyIgnoreList
+
     newlyAddedReplyFieldTypeBsonAnyAllowed:
         description: "command passes because it has a newly added reply field type has a bson_serialization_type
                       that contains 'any' that is explicitly allowed"
@@ -1145,6 +1368,16 @@ commands:
         api_version: "1"
         reply_type: OldUnstableTypeChangesReply
 
+    newUnstableReplyFieldTypeChangesIgnoreList:
+        description: "command has an old stable reply field with incompatible type changes but it is also
+                      in the ignore list"
+        command_name: newUnstableReplyFieldTypeChangesIgnoreList
+        namespace: ignored
+        cpp_name: newUnstableReplyFieldTypeChangesIgnoreList
+        strict: true
+        api_version: "1"
+        reply_type: NewUnstableTypeChangesReplyIgnoreList
+
     commandAllowedAnyTypes:
         description: "new command that has parameter and reply type with
                       explicitly allowed 'any' bson serialization type passes"