Squashed 'json/' changes from 6b3cac42b..d21ed578a

d21ed578a Merge pull request #628 from json-schema-org/catch-unknown-keywords
42b4c3645 Ensure we don't use unknown keywords unless intended.

git-subtree-dir: json
git-subtree-split: d21ed578aeb13b55dfe3604db53adb4750e1abc3

2 files changed, 357 insertions, 25 deletions

diff --git a/json/bin/jsonschema_suite b/json/bin/jsonschema_suite
index 33d4c56..1288f04 100755
--- a/json/bin/jsonschema_suite
+++ b/json/bin/jsonschema_suite
@@ -88,6 +88,23 @@ def url_for_path(path):
     )
 
 
+def versions_and_validators():
+    """
+    All versions we can validate schemas from.
+    """
+
+    for version in SUITE_ROOT_DIR.iterdir():
+        if not version.is_dir():
+            continue
+
+            Validator = VALIDATORS.get(version.name)
+            if Validator is None:
+                warnings.warn(f"No schema validator for {version.name}")
+                continue
+
+            yield version, Validator
+
+
 class SanityTests(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -218,31 +235,346 @@ class SanityTests(unittest.TestCase):
         """
         All schemas are valid under their metaschemas.
         """
-        for version in SUITE_ROOT_DIR.iterdir():
-            if not version.is_dir():
+        for version, Validator in versions_and_validators():
+            # Valid (optional test) schemas contain regexes which
+            # aren't valid Python regexes, so skip checking it
+            Validator.FORMAT_CHECKER.checkers.pop("regex", None)
+
+            test_files = collect(version)
+            for case in cases(test_files):
+                with self.subTest(case=case):
+                    try:
+                        Validator.check_schema(
+                            case["schema"],
+                            format_checker=Validator.FORMAT_CHECKER,
+                        )
+                    except jsonschema.SchemaError:
+                        self.fail(
+                            "Found an invalid schema. "
+                            "See the traceback for details on why."
+                        )
+
+    @unittest.skipIf(jsonschema is None, "Validation library not present!")
+    def test_arbitrary_schemas_do_not_use_unknown_keywords(self):
+        """
+        Test cases do not use unknown keywords.
+
+        (Unless they specifically are testing the specified behavior for
+        unknown keywords).
+
+        This helps prevent accidental leakage of newer keywords into older
+        drafts where they didn't exist.
+        """
+
+        KNOWN = {
+            "draft2020-12": {
+                "$anchor",
+                "$comment",
+                "$defs",
+                "$dynamicAnchor",
+                "$dynamicRef",
+                "$id",
+                "$ref",
+                "$schema",
+                "$vocabulary",
+                "additionalProperties",
+                "allOf",
+                "allOf",
+                "anyOf",
+                "const",
+                "contains",
+                "contentEncoding",
+                "contentMediaType",
+                "contentSchema",
+                "dependencies",
+                "dependentRequired",
+                "dependentSchemas",
+                "description",
+                "else",
+                "enum",
+                "exclusiveMaximum",
+                "exclusiveMinimum",
+                "format",
+                "if",
+                "items",
+                "maxContains",
+                "maxItems",
+                "maxItems",
+                "maxLength",
+                "maxProperties",
+                "maximum",
+                "minContains",
+                "minItems",
+                "minLength",
+                "minProperties",
+                "minimum",
+                "multipleOf",
+                "not",
+                "oneOf",
+                "pattern",
+                "patternProperties",
+                "prefixItems",
+                "properties",
+                "propertyNames",
+                "required",
+                "then",
+                "title",
+                "type",
+                "unevaluatedItems",
+                "unevaluatedProperties",
+                "uniqueItems",
+            },
+            "draft2019-09": {
+                "$anchor",
+                "$comment",
+                "$defs",
+                "$id",
+                "$recursiveAnchor",
+                "$recursiveRef",
+                "$ref",
+                "$schema",
+                "$vocabulary",
+                "additionalItems",
+                "additionalProperties",
+                "allOf",
+                "anyOf",
+                "const",
+                "contains",
+                "contentEncoding",
+                "contentMediaType",
+                "contentSchema",
+                "dependencies",
+                "dependentRequired",
+                "dependentSchemas",
+                "description",
+                "else",
+                "enum",
+                "exclusiveMaximum",
+                "exclusiveMinimum",
+                "format",
+                "if",
+                "items",
+                "maxContains",
+                "maxItems",
+                "maxLength",
+                "maxProperties",
+                "maximum",
+                "minContains",
+                "minItems",
+                "minLength",
+                "minProperties",
+                "minimum",
+                "multipleOf",
+                "not",
+                "oneOf",
+                "pattern",
+                "patternProperties",
+                "properties",
+                "propertyNames",
+                "required",
+                "then",
+                "title",
+                "type",
+                "unevaluatedItems",
+                "unevaluatedProperties",
+                "uniqueItems",
+            },
+            "draft7": {
+                "$comment",
+                "$id",
+                "$ref",
+                "$schema",
+                "additionalItems",
+                "additionalProperties",
+                "allOf",
+                "anyOf",
+                "const",
+                "contains",
+                "contentEncoding",
+                "contentMediaType",
+                "definitions",
+                "dependencies",
+                "description",
+                "else",
+                "enum",
+                "exclusiveMaximum",
+                "exclusiveMinimum",
+                "format",
+                "if",
+                "items",
+                "maxItems",
+                "maxLength",
+                "maxProperties",
+                "maximum",
+                "minItems",
+                "minLength",
+                "minProperties",
+                "minimum",
+                "multipleOf",
+                "not",
+                "oneOf",
+                "pattern",
+                "patternProperties",
+                "properties",
+                "propertyNames",
+                "required",
+                "then",
+                "title",
+                "type",
+                "type",
+                "uniqueItems",
+            },
+            "draft6": {
+                "$comment",
+                "$id",
+                "$ref",
+                "$schema",
+                "additionalItems",
+                "additionalProperties",
+                "allOf",
+                "anyOf",
+                "const",
+                "contains",
+                "definitions",
+                "dependencies",
+                "description",
+                "enum",
+                "exclusiveMaximum",
+                "exclusiveMinimum",
+                "format",
+                "items",
+                "maxItems",
+                "maxLength",
+                "maxProperties",
+                "maximum",
+                "minItems",
+                "minLength",
+                "minProperties",
+                "minimum",
+                "multipleOf",
+                "not",
+                "oneOf",
+                "pattern",
+                "patternProperties",
+                "properties",
+                "propertyNames",
+                "required",
+                "title",
+                "type",
+                "uniqueItems",
+            },
+            "draft4": {
+                "$ref",
+                "$schema",
+                "additionalItems",
+                "additionalItems",
+                "additionalProperties",
+                "allOf",
+                "anyOf",
+                "definitions",
+                "dependencies",
+                "description",
+                "enum",
+                "exclusiveMaximum",
+                "exclusiveMinimum",
+                "format",
+                "id",
+                "items",
+                "maxItems",
+                "maxLength",
+                "maxProperties",
+                "maximum",
+                "minItems",
+                "minLength",
+                "minProperties",
+                "minimum",
+                "multipleOf",
+                "not",
+                "oneOf",
+                "pattern",
+                "patternProperties",
+                "properties",
+                "required",
+                "title",
+                "type",
+                "uniqueItems",
+            },
+            "draft3": {
+                "$ref",
+                "$schema",
+                "additionalItems",
+                "additionalProperties",
+                "definitions",
+                "dependencies",
+                "description",
+                "disallow",
+                "divisibleBy",
+                "enum",
+                "exclusiveMaximum",
+                "exclusiveMinimum",
+                "extends",
+                "format",
+                "id",
+                "items",
+                "maxItems",
+                "maxLength",
+                "maximum",
+                "minItems",
+                "minLength",
+                "minimum",
+                "pattern",
+                "patternProperties",
+                "properties",
+                "title",
+                "type",
+                "uniqueItems",
+            },
+        }
+
+        def missing(d):
+            from collections.abc import Mapping
+
+            class BlowUpForUnknownProperties(Mapping):
+                def __iter__(this):
+                    return iter(d)
+
+                def __getitem__(this, k):
+                    if k not in KNOWN[version.name]:
+                        self.fail(
+                            f"{k} is not a known keyword for {version.name}. "
+                            "If this test is testing behavior related to "
+                            "unknown keywords you may need to add it to the "
+                            "allowlist in the jsonschema_suite checker. "
+                            "Otherwise it may contain a typo!"
+                        )
+                    return d[k]
+
+                def __len__(this):
+                    return len(d)
+
+            return BlowUpForUnknownProperties()
+
+        for version, Validator in versions_and_validators():
+            if version.name == "latest":
                 continue
 
-            Validator = VALIDATORS.get(version.name)
-            if Validator is not None:
-                # Valid (optional test) schemas contain regexes which
-                # aren't valid Python regexes, so skip checking it
-                Validator.FORMAT_CHECKER.checkers.pop("regex", None)
-
-                test_files = collect(version)
-                for case in cases(test_files):
-                    with self.subTest(case=case):
-                        try:
-                            Validator.check_schema(
-                                case["schema"],
-                                format_checker=Validator.FORMAT_CHECKER,
-                            )
-                        except jsonschema.SchemaError:
-                            self.fail(
-                                "Found an invalid schema. "
-                                "See the traceback for details on why."
-                            )
-            else:
-                warnings.warn(f"No schema validator for {version.name}")
+            self.addCleanup(
+                setattr, Validator, "VALIDATORS", Validator.VALIDATORS,
+            )
+            Validator.VALIDATORS = missing(dict(Validator.VALIDATORS))
+
+            test_files = [
+                each for each in collect(version)
+                if each.stem != "refOfUnknownKeyword"
+            ]
+            for case in cases(test_files):
+                if "unknown keyword" in case["description"]:
+                    continue
+                with self.subTest(case=case, version=version.name):
+                    try:
+                        Validator(case["schema"]).is_valid(12)
+                    except jsonschema.exceptions.RefResolutionError:
+                        pass
 
     @unittest.skipIf(jsonschema is None, "Validation library not present!")
     def test_suites_are_valid(self):
@@ -269,7 +601,7 @@ class SanityTests(unittest.TestCase):
             with self.subTest(path=path):
                 try:
                     validator.validate(cases)
-                except jsonschema.exceptions.RefResolutionError as error:
+                except jsonschema.exceptions.RefResolutionError:
                     # python-jsonschema/jsonschema#884
                     pass
                 except jsonschema.ValidationError as error:
diff --git a/json/tests/draft3/ref.json b/json/tests/draft3/ref.json
index 924db76..609eaa4 100644
--- a/json/tests/draft3/ref.json
+++ b/json/tests/draft3/ref.json
@@ -215,7 +215,7 @@
                     "type": "number"
                 }
             },
-            "allOf": [
+            "extends": [
                 {
                     "$comment": "$ref resolves to http://localhost:1234/sibling_id/base/foo.json, not http://localhost:1234/sibling_id/foo.json",
                     "id": "http://localhost:1234/sibling_id/",