Merge commit 'ade1faa768471bb95b14533f20a504fe8fc326c9'

* commit 'ade1faa768471bb95b14533f20a504fe8fc326c9':
  Squashed 'json/' changes from 78c888273..6b3cac42b

20 files changed, 901 insertions, 53 deletions

diff --git a/json/.editorconfig b/json/.editorconfig
new file mode 100644
index 0000000..6db6a5b
--- /dev/null
+++ b/json/.editorconfig
@@ -0,0 +1 @@
+insert_final_newline = true
diff --git a/json/bin/jsonschema_suite b/json/bin/jsonschema_suite
index 8cc2850..33d4c56 100755
--- a/json/bin/jsonschema_suite
+++ b/json/bin/jsonschema_suite
@@ -30,11 +30,15 @@ else:
 
 ROOT_DIR = Path(__file__).parent.parent
 SUITE_ROOT_DIR = ROOT_DIR / "tests"
+OUTPUT_ROOT_DIR = ROOT_DIR / "output-tests"
 
 REMOTES_DIR = ROOT_DIR / "remotes"
 REMOTES_BASE_URL = "http://localhost:1234/"
 
-TESTSUITE_SCHEMA = json.loads((ROOT_DIR / "test-schema.json").read_text())
+TEST_SCHEMA = json.loads(ROOT_DIR.joinpath("test-schema.json").read_text())
+OUTPUT_TEST_SCHEMA = json.loads(
+    ROOT_DIR.joinpath("output-test-schema.json").read_text(),
+)
 
 
 def files(paths):
@@ -67,7 +71,7 @@ def collect(root_dir):
     """
     All of the test file paths within the given root directory, recursively.
     """
-    return root_dir.glob("**/*.json")
+    return root_dir.rglob("*.json")
 
 
 def url_for_path(path):
@@ -80,7 +84,7 @@ def url_for_path(path):
 
     return urljoin(
         REMOTES_BASE_URL,
-        str(path.relative_to(REMOTES_DIR)).replace("\\", "/")  # Windows...
+        str(path.relative_to(REMOTES_DIR)).replace("\\", "/"),  # Windows...
     )
 
 
@@ -88,12 +92,21 @@ class SanityTests(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         print(f"Looking for tests in {SUITE_ROOT_DIR}")
+        print(f"Looking for output tests in {OUTPUT_ROOT_DIR}")
         print(f"Looking for remotes in {REMOTES_DIR}")
 
         cls.test_files = list(collect(SUITE_ROOT_DIR))
         assert cls.test_files, "Didn't find the test files!"
         print(f"Found {len(cls.test_files)} test files")
 
+        cls.output_test_files = [
+            each
+            for each in collect(OUTPUT_ROOT_DIR)
+            if each.name != "output-schema.json"
+        ]
+        assert cls.output_test_files, "Didn't find the output test files!"
+        print(f"Found {len(cls.output_test_files)} output test files")
+
         cls.remote_files = list(collect(REMOTES_DIR))
         assert cls.remote_files, "Didn't find the remote files!"
         print(f"Found {len(cls.remote_files)} remote files")
@@ -131,22 +144,11 @@ class SanityTests(unittest.TestCase):
         self.assertNotRegex(description, r"\bshould\b", message)
         self.assertNotRegex(description, r"(?i)\btest(s)? that\b", message)
 
-    def test_all_test_files_are_valid_json(self):
-        """
-        All test files contain valid JSON.
-        """
-        for path in self.test_files:
-            with self.subTest(path=path):
-                try:
-                    json.loads(path.read_text())
-                except ValueError as error:
-                    self.fail(f"{path} contains invalid JSON ({error})")
-
-    def test_all_remote_files_are_valid_json(self):
+    def test_all_json_files_are_valid(self):
         """
-        All remote files contain valid JSON.
+        All files (tests, output tests, remotes, etc.) contain valid JSON.
         """
-        for path in self.remote_files:
+        for path in collect(ROOT_DIR):
             with self.subTest(path=path):
                 try:
                     json.loads(path.read_text())
@@ -157,24 +159,26 @@ class SanityTests(unittest.TestCase):
         """
         All cases have reasonably long descriptions.
         """
-        for case in cases(self.test_files):
+        for case in cases(self.test_files + self.output_test_files):
             with self.subTest(description=case["description"]):
                 self.assertLess(
                     len(case["description"]),
                     150,
-                    "Description is too long (keep it to less than 150 chars)."
+                    "Description is too long (keep it to less than 150 chars).",
                 )
 
     def test_all_test_descriptions_have_reasonable_length(self):
         """
         All tests have reasonably long descriptions.
         """
-        for count, test in enumerate(tests(self.test_files)):
+        for count, test in enumerate(
+            tests(self.test_files + self.output_test_files)
+        ):
             with self.subTest(description=test["description"]):
                 self.assertLess(
                     len(test["description"]),
                     70,
-                    "Description is too long (keep it to less than 70 chars)."
+                    "Description is too long (keep it to less than 70 chars).",
                 )
         print(f"Found {count} tests.")
 
@@ -182,7 +186,7 @@ class SanityTests(unittest.TestCase):
         """
         All cases have unique descriptions in their files.
         """
-        for path, cases in files(self.test_files):
+        for path, cases in files(self.test_files + self.output_test_files):
             with self.subTest(path=path):
                 self.assertUnique(case["description"] for case in cases)
 
@@ -190,7 +194,9 @@ class SanityTests(unittest.TestCase):
         """
         All test cases have unique test descriptions in their tests.
         """
-        for count, case in enumerate(cases(self.test_files)):
+        for count, case in enumerate(
+            cases(self.test_files + self.output_test_files)
+        ):
             with self.subTest(description=case["description"]):
                 self.assertUnique(
                     test["description"] for test in case["tests"]
@@ -198,12 +204,12 @@ class SanityTests(unittest.TestCase):
         print(f"Found {count} test cases.")
 
     def test_case_descriptions_do_not_use_modal_verbs(self):
-        for case in cases(self.test_files):
+        for case in cases(self.test_files + self.output_test_files):
             with self.subTest(description=case["description"]):
                 self.assertFollowsDescriptionStyle(case["description"])
 
     def test_test_descriptions_do_not_use_modal_verbs(self):
-        for test in tests(self.test_files):
+        for test in tests(self.test_files + self.output_test_files):
             with self.subTest(description=test["description"]):
                 self.assertFollowsDescriptionStyle(test["description"])
 
@@ -218,14 +224,21 @@ class SanityTests(unittest.TestCase):
 
             Validator = VALIDATORS.get(version.name)
             if Validator is not None:
+                # Valid (optional test) schemas contain regexes which
+                # aren't valid Python regexes, so skip checking it
+                Validator.FORMAT_CHECKER.checkers.pop("regex", None)
+
                 test_files = collect(version)
                 for case in cases(test_files):
                     with self.subTest(case=case):
                         try:
-                            Validator.check_schema(case["schema"])
+                            Validator.check_schema(
+                                case["schema"],
+                                format_checker=Validator.FORMAT_CHECKER,
+                            )
                         except jsonschema.SchemaError:
                             self.fail(
-                                "Found an invalid schema."
+                                "Found an invalid schema. "
                                 "See the traceback for details on why."
                             )
             else:
@@ -236,8 +249,8 @@ class SanityTests(unittest.TestCase):
         """
         All test files are valid under test-schema.json.
         """
-        Validator = jsonschema.validators.validator_for(TESTSUITE_SCHEMA)
-        validator = Validator(TESTSUITE_SCHEMA)
+        Validator = jsonschema.validators.validator_for(TEST_SCHEMA)
+        validator = Validator(TEST_SCHEMA)
         for path, cases in files(self.test_files):
             with self.subTest(path=path):
                 try:
@@ -245,6 +258,23 @@ class SanityTests(unittest.TestCase):
                 except jsonschema.ValidationError as error:
                     self.fail(str(error))
 
+    @unittest.skipIf(jsonschema is None, "Validation library not present!")
+    def test_output_suites_are_valid(self):
+        """
+        All output test files are valid under output-test-schema.json.
+        """
+        Validator = jsonschema.validators.validator_for(OUTPUT_TEST_SCHEMA)
+        validator = Validator(OUTPUT_TEST_SCHEMA)
+        for path, cases in files(self.output_test_files):
+            with self.subTest(path=path):
+                try:
+                    validator.validate(cases)
+                except jsonschema.exceptions.RefResolutionError as error:
+                    # python-jsonschema/jsonschema#884
+                    pass
+                except jsonschema.ValidationError as error:
+                    self.fail(str(error))
+
 
 def main(arguments):
     if arguments.command == "check":
@@ -277,7 +307,9 @@ def main(arguments):
         try:
             import flask
         except ImportError:
-            print(textwrap.dedent("""
+            print(
+                textwrap.dedent(
+                    """
                 The Flask library is required to serve the remote schemas.
 
                 You can install it by running `pip install Flask`.
@@ -285,7 +317,11 @@ def main(arguments):
                 Alternatively, see the `jsonschema_suite remotes` or
                 `jsonschema_suite dump_remotes` commands to create static files
                 that can be served with your own web server.
-            """.strip("\n")))
+            """.strip(
+                        "\n"
+                    )
+                )
+            )
             sys.exit(1)
 
         app = flask.Flask(__name__)
@@ -309,7 +345,7 @@ check = subparsers.add_parser("check", help="Sanity check the test suite.")
 
 flatten = subparsers.add_parser(
     "flatten",
-    help="Output a flattened file containing a selected version's test cases."
+    help="Output a flattened file containing a selected version's test cases.",
 )
 flatten.add_argument(
     "--randomize",
@@ -317,17 +353,19 @@ flatten.add_argument(
     help="Randomize the order of the outputted cases.",
 )
 flatten.add_argument(
-    "version", help="The directory containing the version to output",
+    "version",
+    help="The directory containing the version to output",
 )
 
 remotes = subparsers.add_parser(
     "remotes",
     help="Output the expected URLs and their associated schemas for remote "
-         "ref tests as a JSON object."
+    "ref tests as a JSON object.",
 )
 
 dump_remotes = subparsers.add_parser(
-    "dump_remotes", help="Dump the remote ref schemas into a file tree",
+    "dump_remotes",
+    help="Dump the remote ref schemas into a file tree",
 )
 dump_remotes.add_argument(
     "--update",
@@ -343,7 +381,7 @@ dump_remotes.add_argument(
 
 serve = subparsers.add_parser(
     "serve",
-    help="Start a webserver to serve schemas used by remote ref tests."
+    help="Start a webserver to serve schemas used by remote ref tests.",
 )
 
 if __name__ == "__main__":
diff --git a/json/output-test-schema.json b/json/output-test-schema.json
new file mode 100644
index 0000000..02c51ef
--- /dev/null
+++ b/json/output-test-schema.json
@@ -0,0 +1,70 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://json-schema.org/tests/output-test-schema",
+    "description": "A schema for files contained within this suite",
+
+    "type": "array",
+    "minItems": 1,
+    "items": {
+        "description": "An individual test case, containing multiple tests of a single schema's behavior",
+
+        "type": "object",
+        "required": [ "description", "schema", "tests" ],
+        "properties": {
+            "description": {
+                "description": "The test case description",
+                "type": "string"
+            },
+            "comment": {
+                "description": "Any additional comments about the test case",
+                "type": "string"
+            },
+            "schema": {
+                "description": "A valid JSON Schema (one written for the corresponding version directory that the file sits within)."
+            },
+            "tests": {
+                "description": "A set of related tests all using the same schema",
+                "type": "array",
+                "items": { "$ref": "#/$defs/test" },
+                "minItems": 1
+            }
+        },
+        "additionalProperties": false
+    },
+
+    "$defs": {
+        "test": {
+            "description": "A single output test",
+
+            "type": "object",
+            "required": [ "description", "data", "output" ],
+            "properties": {
+                "description": {
+                    "description": "The test description, briefly explaining which behavior it exercises",
+                    "type": "string"
+                },
+                "comment": {
+                    "description": "Any additional comments about the test",
+                    "type": "string"
+                },
+                "data": {
+                    "description": "The instance which should be validated against the schema in \"schema\"."
+                },
+                "output": {
+                    "description": "schemas that are used to verify output",
+                    "type": "object",
+                    "properties": {
+                        "flag": { "$ref": "https://json-schema.org/draft/2020-12/schema" },
+                        "basic": { "$ref": "https://json-schema.org/draft/2020-12/schema" },
+                        "detailed": { "$ref": "https://json-schema.org/draft/2020-12/schema" },
+                        "verbose": { "$ref": "https://json-schema.org/draft/2020-12/schema" },
+                        "list": { "$ref": "https://json-schema.org/draft/2020-12/schema" },
+                        "hierarchy": { "$ref": "https://json-schema.org/draft/2020-12/schema" }
+                    },
+                    "minProperties": 1,
+                    "additionalProperties": false
+                }
+            }
+        }
+    }
+}
diff --git a/json/output-tests/README.md b/json/output-tests/README.md
new file mode 100644
index 0000000..0cd7c87
--- /dev/null
+++ b/json/output-tests/README.md
@@ -0,0 +1,63 @@
+These tests are intended to validate that implementations are correctly generating output in accordance with the specification.
+
+Output was initially specified with draft 2019-09.  It remained largely unchanged for draft 2020-12, but will receive an update with the next release.
+
+***NOTE** Although the formats didn't change between 2019-09 and 2020-12, the tests are replicated for 2020-12 because the `$schema` is different and implementations may (but shouldn't) produce different output.*
+
+## Organization
+
+The tests are organized by specification release and then into two categories: content and structure.
+
+Content tests verify that the keywords are producing the correct annotations and/or error messages.  Since there are no requirements on the content of error messages, there's not much that can be verified for them, but it is possible to identify when a error message _could_ be present.  Primarily, these tests need to extensively cover the annotation behaviors of each keyword.  The only output format needed for these tests is `basic` for 2019-09/2020/12 and `list` for later versions.
+
+Structure tests verify that the structures of the various formats (i.e. `flag`, `basic`, `detailed`, `verbose` for 2019/2020 and `flag`, `list`, `hierarchical` for later versions) are correct.  These tests don't need to cover each keyword; rather they need to sufficiently cover the various aspects of building the output structures by using whatever keywords are necessary to do so.
+
+In each release folder, you'll also find an _output-schema.json_ file that contains the schema from the specification repo that describes output for that release.  This schema will need to be loaded as the tests reference it.
+
+## Test Files
+
+The content of a test file is the same as the validation tests in `tests/`, however an `output` property has been added to each test case.
+
+The `output` property itself has a property for each of the output formats where the value is a schema that will successfully validate for compliant output.  For the content tests, only `basic`/`list` needs to be present.
+
+## Other notes
+
+### Ambiguity around 2020-09/2020-12 `basic`
+
+The 2019-09/2020-12 specs don't define the structure of `basic` very thoroughly.  Specifically there is a nuance where if the list contains a single output node, there are two possible structures, given the text:
+
+- the output node for the root schema appears in the list with a containing node that just has a `valid` property
+    ```json
+    {
+        "valid": false,
+        "errors": [
+            {
+                "valid": false,
+                "keywordLocation": "",
+                "absoluteKeywordLocation": "https://json-schema.org/tests/content/draft2019-09/general/0",
+                "instanceLocation": ""
+            }
+        ]
+    }
+    ```
+- the entire structure is collapsed to just the root output node as `detailed` would do.
+    ```json
+    {
+        "valid": false,
+        "keywordLocation": "",
+        "absoluteKeywordLocation": "https://json-schema.org/tests/content/draft2019-09/general/0",
+        "instanceLocation": ""
+    }
+    ```
+As the Test Suite should not prefer one interpretation over another, these cases need to be tested another way.
+
+A simple solution (though there are likely others) is to force a second output unit by adding an `"anyOf": [ true ]`.  This has no impact on the validation result while adding superfluous structure to the output that avoids the above ambiguous scenario.  The test schema should still be targeted on what's being tested and ignore any output units generated by this extra keyword.
+
+## Contributing
+
+Of course, first and foremost, follow the [Contributing guide](/CONTRIBUTING.md).
+
+When writing test cases, try to keep output validation schemas targeted to verify a single requirement.  Where possible (and where it makes sense), create multiple tests to cover multiple requirements.  This will help keep the output validation schemas small and increase readability.  (It also increases your test count. 😉)
+
+For the content tests, there is also a _general.json_ file that contains tests that do not necessarily pertain to any single keyword.
+<!-- This general.json file may be added to the structure tests later, but I haven't gotten to them yet, so I don't know. -->
diff --git a/json/output-tests/draft-next/content/general.json b/json/output-tests/draft-next/content/general.json
new file mode 100644
index 0000000..27082ed
--- /dev/null
+++ b/json/output-tests/draft-next/content/general.json
@@ -0,0 +1,43 @@
+[
+    {
+        "description": "failed validation produces no annotations",
+        "schema": {
+            "$schema": "https://json-schema.org/draft/next/schema",
+            "$id": "https://json-schema.org/tests/content/draft-next/general/0",
+            "type": "string",
+            "readOnly": true
+        },
+        "tests": [
+            {
+                "description": "dropped annotations MAY appear in droppedAnnotations",
+                "data": 1,
+                "output": {
+                    "list": {
+                        "$id": "https://json-schema.org/tests/content/draft-next/general/0/tests/0/basic",
+                        "$ref": "/draft/next/output/schema",
+                        "properties": {
+                            "details": {
+                                "contains": {
+                                    "properties": {
+                                        "evaluationPath": {"const": ""},
+                                        "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/general/0"},
+                                        "instanceLocation": {"const": ""},
+                                        "annotations": false,
+                                        "droppedAnnotations": {
+                                            "properties": {
+                                                "readOnly": {"const": true}
+                                            },
+                                            "required": ["readOnly"]
+                                        }
+                                    },
+                                    "required": ["evaluationPath", "schemaLocation", "instanceLocation"]
+                                }
+                            }
+                        },
+                        "required": ["details"]
+                    }
+                }
+            }
+        ]
+    }
+]
diff --git a/json/output-tests/draft-next/content/readOnly.json b/json/output-tests/draft-next/content/readOnly.json
new file mode 100644
index 0000000..d387d93
--- /dev/null
+++ b/json/output-tests/draft-next/content/readOnly.json
@@ -0,0 +1,41 @@
+[
+    {
+        "description": "readOnly generates its value as an annotation",
+        "schema": {
+            "$schema": "https://json-schema.org/draft/next/schema",
+            "$id": "https://json-schema.org/tests/content/draft-next/readOnly/0",
+            "readOnly": true
+        },
+        "tests": [
+            {
+                "description": "readOnly is true",
+                "data": 1,
+                "output": {
+                    "list": {
+                        "$id": "https://json-schema.org/tests/content/draft-next/readOnly/0/tests/0/basic",
+                        "$ref": "/draft/next/output/schema",
+                        "properties": {
+                            "details": {
+                                "contains": {
+                                    "properties": {
+                                        "evaluationPath": {"const": ""},
+                                        "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/readOnly/0"},
+                                        "instanceLocation": {"const": ""},
+                                        "annotations": {
+                                            "properties": {
+                                                "readOnly": {"const": true}
+                                            },
+                                            "required": ["readOnly"]
+                                        }
+                                    },
+                                    "required": ["evaluationPath", "schemaLocation", "instanceLocation", "annotations"]
+                                }
+                            }
+                        },
+                        "required": ["details"]
+                    }
+                }
+            }
+        ]
+    }
+]
diff --git a/json/output-tests/draft-next/content/type.json b/json/output-tests/draft-next/content/type.json
new file mode 100644
index 0000000..e17f1f5
--- /dev/null
+++ b/json/output-tests/draft-next/content/type.json
@@ -0,0 +1,39 @@
+[
+    {
+        "description": "incorrect type",
+        "schema": {
+            "$schema": "https://json-schema.org/draft/next/schema",
+            "$id": "https://json-schema.org/tests/content/draft-next/type/0",
+            "type": "string"
+        },
+        "tests": [
+            {
+                "description": "incorrect type must be reported, but a message is not required",
+                "data": 1,
+                "output": {
+                    "list": {
+                        "$id": "https://json-schema.org/tests/content/draft-next/type/0/tests/0/basic",
+                        "$ref": "/draft/next/output/schema",
+                        "properties": {
+                            "details": {
+                                "contains": {
+                                    "properties": {
+                                        "evaluationPath": {"const": ""},
+                                        "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/type/0"},
+                                        "instanceLocation": {"const": ""},
+                                        "annotations": false,
+                                        "errors": {
+                                            "required": ["type"]
+                                        }
+                                    },
+                                    "required": ["evaluationPath", "schemaLocation", "instanceLocation"]
+                                }
+                            }
+                        },
+                        "required": ["details"]
+                    }
+                }
+            }
+        ]
+    }
+]
diff --git a/json/output-tests/draft-next/output-schema.json b/json/output-tests/draft-next/output-schema.json
new file mode 100644
index 0000000..26286fa
--- /dev/null
+++ b/json/output-tests/draft-next/output-schema.json
@@ -0,0 +1,95 @@
+{
+  "$schema": "https://json-schema.org/draft/next/schema",
+  "$id": "https://json-schema.org/draft/next/output/schema",
+  "description": "A schema that validates the minimum requirements for validation output",
+
+  "anyOf": [
+    { "$ref": "#/$defs/flag" },
+    { "$ref": "#/$defs/basic" },
+    { "$ref": "#/$defs/hierarchical" }
+  ],
+  "$defs": {
+    "outputUnit":{
+      "properties": {
+        "valid": { "type": "boolean" },
+        "evaluationPath": {
+          "type": "string",
+          "format": "json-pointer"
+        },
+        "schemaLocation": {
+          "type": "string",
+          "format": "uri"
+        },
+        "instanceLocation": {
+          "type": "string",
+          "format": "json-pointer"
+        },
+        "details": {
+          "$ref": "#/$defs/outputUnitArray"
+        },
+        "annotations": {
+          "type": "object",
+          "additionalProperties": true
+        },
+        "droppedAnnotations": {
+          "type": "object",
+          "additionalProperties": true
+        },
+        "errors": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        }
+      },
+      "required": [ "valid", "evaluationPath", "schemaLocation", "instanceLocation" ],
+      "allOf": [
+        {
+          "if": {
+            "anyOf": [
+              {
+                "required": [ "errors" ]
+              },
+              {
+                "required": [ "droppedAnnotations" ]
+              }
+            ]
+          },
+          "then": {
+            "properties": {
+              "valid": { "const": false }
+            }
+          }
+        },
+        {
+          "if": {
+            "required": [ "annotations" ]
+          },
+          "then": {
+            "properties": {
+              "valid": { "const": true }
+            }
+          }
+        }
+      ]
+    },
+    "outputUnitArray": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/outputUnit" }
+    },
+    "flag": {
+      "properties": {
+        "valid": { "type": "boolean" }
+      },
+      "required": [ "valid" ]
+    },
+    "basic": {
+      "properties": {
+        "valid": { "type": "boolean" },
+        "details": {
+          "$ref": "#/$defs/outputUnitArray"
+        }
+      },
+      "required": [ "valid", "details" ]
+    },
+    "hierarchical": { "$ref": "#/$defs/outputUnit" }
+  }
+}
diff --git a/json/output-tests/draft2019-09/content/general.json b/json/output-tests/draft2019-09/content/general.json
new file mode 100644
index 0000000..9194170
--- /dev/null
+++ b/json/output-tests/draft2019-09/content/general.json
@@ -0,0 +1,34 @@
+[
+    {
+        "description": "failed validation produces no annotations",
+        "schema": {
+            "$schema": "https://json-schema.org/draft/2019-09/schema",
+            "$id": "https://json-schema.org/tests/content/draft2019-09/general/0",
+            "type": "string",
+            "readOnly": true
+        },
+        "tests": [
+            {
+                "description": "readOnly annotation is dropped",
+                "data": 1,
+                "output": {
+                    "basic": {
+                        "$id": "https://json-schema.org/tests/content/draft2019-09/general/0/tests/0/basic",
+                        "$ref": "/draft/2019-09/output/schema",
+                        "properties": {
+                            "errors": {
+                                "items": {
+                                    "properties": {
+                                        "annotation": false
+                                    }
+                                }
+                            },
+                            "annotations": false
+                        },
+                        "required": ["errors"]
+                    }
+                }
+            }
+        ]
+    }
+]
diff --git a/json/output-tests/draft2019-09/content/readOnly.json b/json/output-tests/draft2019-09/content/readOnly.json
new file mode 100644
index 0000000..62db1a8
--- /dev/null
+++ b/json/output-tests/draft2019-09/content/readOnly.json
@@ -0,0 +1,38 @@
+[
+    {
+        "description": "readOnly generates its value as an annotation",
+        "schema": {
+            "$schema": "https://json-schema.org/draft/2019-09/schema",
+            "$id": "https://json-schema.org/tests/content/draft2019-09/readOnly/0",
+            "readOnly": true
+        },
+        "tests": [
+            {
+                "description": "readOnly is true",
+                "data": 1,
+                "output": {
+                    "basic": {
+                        "$id": "https://json-schema.org/tests/content/draft2019-09/readOnly/0/tests/0/basic",
+                        "$ref": "/draft/2019-09/output/schema",
+                        "properties": {
+                            "annotations": {
+                                "contains": {
+                                    "type": "object",
+                                    "properties": {
+                                        "keywordLocation": {"const": "/readOnly"},
+                                        "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2019-09/readOnly/0#/readOnly"},
+                                        "instanceLocation": {"const": ""},
+                                        "annotation": {"const": true}
+                                    },
+                                    "required": ["keywordLocation", "instanceLocation", "annotation"]
+                                }
+                            },
+                            "errors": false
+                        },
+                        "required": ["annotations"]
+                    }
+                }
+            }
+        ]
+    }
+]
diff --git a/json/output-tests/draft2019-09/content/type.json b/json/output-tests/draft2019-09/content/type.json
new file mode 100644
index 0000000..cff77a7
--- /dev/null
+++ b/json/output-tests/draft2019-09/content/type.json
@@ -0,0 +1,63 @@
+[
+    {
+        "description": "validating type",
+        "schema": {
+            "$schema": "https://json-schema.org/draft/2019-09/schema",
+            "$id": "https://json-schema.org/tests/content/draft2019-09/type/0",
+            "type": "string",
+            "anyOf": [ true ]
+        },
+        "tests": [
+            {
+                "description": "incorrect type must be reported, but a message is not required",
+                "data": 1,
+                "output": {
+                    "basic": {
+                        "$id": "https://json-schema.org/tests/content/draft2019-09/type/0/tests/0/basic",
+                        "$ref": "/draft/2019-09/output/schema",
+                        "properties": {
+                            "errors": {
+                                "contains": {
+                                    "properties": {
+                                        "keywordLocation": {"const": "/type"},
+                                        "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2019-09/type/0#/type"},
+                                        "instanceLocation": {"const": ""},
+                                        "annotation": false
+                                    },
+                                    "required": ["keywordLocation", "instanceLocation"]
+                                }
+                            }
+                        },
+                        "required": ["errors"]
+                    }
+                }
+            },
+            {
+                "description": "correct type yields an output unit",
+                "data": "a string",
+                "output": {
+                    "basic": {
+                        "$id": "https://json-schema.org/tests/content/draft2019-09/type/0/tests/1/basic",
+                        "$ref": "/draft/2019-09/output/schema",
+                        "properties": {
+                            "annotations": {
+                                "contains": {
+                                    "properties": {
+                                        "valid": {"const": true},
+                                        "keywordLocation": {"const": "/type"},
+                                        "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2019-09/type/0#/type"},
+                                        "instanceLocation": {"const": ""},
+                                        "annotation": false,
+                                        "error": false
+                                    },
+                                    "required": ["keywordLocation", "instanceLocation"]
+                                }
+                            }
+                        },
+                        "required": ["annotations"]
+                    }
+                }
+            }
+        ]
+    }
+]
diff --git a/json/output-tests/draft2019-09/output-schema.json b/json/output-tests/draft2019-09/output-schema.json
new file mode 100644
index 0000000..0a65f20
--- /dev/null
+++ b/json/output-tests/draft2019-09/output-schema.json
@@ -0,0 +1,96 @@
+{
+  "$schema": "https://json-schema.org/draft/2019-09/schema",
+  "$id": "https://json-schema.org/draft/2019-09/output/schema",
+  "description": "A schema that validates the minimum requirements for validation output",
+
+  "anyOf": [
+    { "$ref": "#/$defs/flag" },
+    { "$ref": "#/$defs/basic" },
+    { "$ref": "#/$defs/detailed" },
+    { "$ref": "#/$defs/verbose" }
+  ],
+  "$defs": {
+    "outputUnit":{
+      "properties": {
+        "valid": { "type": "boolean" },
+        "keywordLocation": {
+          "type": "string",
+          "format": "json-pointer"
+        },
+        "absoluteKeywordLocation": {
+          "type": "string",
+          "format": "uri"
+        },
+        "instanceLocation": {
+          "type": "string",
+          "format": "json-pointer"
+        },
+        "error": {
+          "type": "string"
+        },
+        "errors": {
+          "$ref": "#/$defs/outputUnitArray"
+        },
+        "annotations": {
+          "$ref": "#/$defs/outputUnitArray"
+        }
+      },
+      "required": [ "valid", "keywordLocation", "instanceLocation" ],
+      "allOf": [
+        {
+          "if": {
+            "properties": {
+              "valid": { "const": false }
+            }
+          },
+          "then": {
+            "anyOf": [
+              {
+                "required": [ "error" ]
+              },
+              {
+                "required": [ "errors" ]
+              }
+            ]
+          }
+        },
+        {
+          "if": {
+            "anyOf": [
+              {
+                "properties": {
+                  "keywordLocation": {
+                    "pattern": "/\\$ref/"
+                  }
+                }
+              },
+              {
+                "properties": {
+                  "keywordLocation": {
+                    "pattern": "/\\$dynamicRef/"
+                  }
+                }
+              }
+            ]
+          },
+          "then": {
+            "required": [ "absoluteKeywordLocation" ]
+          }
+        }
+      ]
+    },
+    "outputUnitArray": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/outputUnit" }
+    },
+    "flag": {
+      "properties": {
+        "valid": { "type": "boolean" }
+      },
+      "required": [ "valid" ]
+    },
+    "basic": { "$ref": "#/$defs/outputUnit" },
+    "detailed": { "$ref": "#/$defs/outputUnit" },
+    "verbose": { "$ref": "#/$defs/outputUnit" }
+  }
+}
diff --git a/json/output-tests/draft2020-12/content/general.json b/json/output-tests/draft2020-12/content/general.json
new file mode 100644
index 0000000..1f2b370
--- /dev/null
+++ b/json/output-tests/draft2020-12/content/general.json
@@ -0,0 +1,34 @@
+[
+    {
+        "description": "failed validation produces no annotations",
+        "schema": {
+            "$schema": "https://json-schema.org/draft/2020-12/schema",
+            "$id": "https://json-schema.org/tests/content/draft2020-12/general/0",
+            "type": "string",
+            "readOnly": true
+        },
+        "tests": [
+            {
+                "description": "readOnly annotation is dropped",
+                "data": 1,
+                "output": {
+                    "basic": {
+                        "$id": "https://json-schema.org/tests/content/draft2020-12/general/0/tests/0/basic",
+                        "$ref": "/draft/2020-12/output/schema",
+                        "properties": {
+                            "errors": {
+                                "items": {
+                                    "properties": {
+                                        "annotation": false
+                                    }
+                                }
+                            },
+                            "annotations": false
+                        },
+                        "required": ["errors"]
+                    }
+                }
+            }
+        ]
+    }
+]
diff --git a/json/output-tests/draft2020-12/content/readOnly.json b/json/output-tests/draft2020-12/content/readOnly.json
new file mode 100644
index 0000000..9baf48d
--- /dev/null
+++ b/json/output-tests/draft2020-12/content/readOnly.json
@@ -0,0 +1,37 @@
+[
+    {
+        "description": "readOnly generates its value as an annotation",
+        "schema": {
+            "$schema": "https://json-schema.org/draft/2020-12/schema",
+            "$id": "https://json-schema.org/tests/content/draft2020-12/readOnly/0",
+            "readOnly": true
+        },
+        "tests": [
+            {
+                "description": "readOnly is true",
+                "data": 1,
+                "output": {
+                    "basic": {
+                        "$id": "https://json-schema.org/tests/content/draft2020-12/readOnly/0/tests/0/basic",
+                        "$ref": "/draft/2020-12/output/schema",
+                        "properties": {
+                            "annotations": {
+                                "contains": {
+                                    "properties": {
+                                        "keywordLocation": {"const": "/readOnly"},
+                                        "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/readOnly/0#/readOnly"},
+                                        "instanceLocation": {"const": ""},
+                                        "annotation": {"const": true}
+                                    },
+                                    "required": ["keywordLocation", "instanceLocation", "annotation"]
+                                }
+                            },
+                            "errors": false
+                        },
+                        "required": ["annotations"]
+                    }
+                }
+            }
+        ]
+    }
+]
diff --git a/json/output-tests/draft2020-12/content/type.json b/json/output-tests/draft2020-12/content/type.json
new file mode 100644
index 0000000..710475b
--- /dev/null
+++ b/json/output-tests/draft2020-12/content/type.json
@@ -0,0 +1,63 @@
+[
+    {
+        "description": "validating type",
+        "schema": {
+            "$schema": "https://json-schema.org/draft/2020-12/schema",
+            "$id": "https://json-schema.org/tests/content/draft2020-12/type/0",
+            "type": "string",
+            "anyOf": [ true ]
+        },
+        "tests": [
+            {
+                "description": "incorrect type must be reported, but a message is not required",
+                "data": 1,
+                "output": {
+                    "basic": {
+                        "$id": "https://json-schema.org/tests/content/draft2020-12/type/0/tests/0/basic",
+                        "$ref": "/draft/2020-12/output/schema",
+                        "properties": {
+                            "errors": {
+                                "contains": {
+                                    "properties": {
+                                        "keywordLocation": {"const": "/type"},
+                                        "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/type/0#/type"},
+                                        "instanceLocation": {"const": ""},
+                                        "annotation": false
+                                    },
+                                    "required": ["keywordLocation", "instanceLocation"]
+                                }
+                            }
+                        },
+                        "required": ["errors"]
+                    }
+                }
+            },
+            {
+                "description": "correct type yields an output unit",
+                "data": "a string",
+                "output": {
+                    "basic": {
+                        "$id": "https://json-schema.org/tests/content/draft2020-12/type/0/tests/1/basic",
+                        "$ref": "/draft/2020-12/output/schema",
+                        "properties": {
+                            "annotations": {
+                                "contains": {
+                                    "properties": {
+                                        "valid": {"const": true},
+                                        "keywordLocation": {"const": "/type"},
+                                        "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/type/0#/type"},
+                                        "instanceLocation": {"const": ""},
+                                        "annotation": false,
+                                        "error": false
+                                    },
+                                    "required": ["keywordLocation", "instanceLocation"]
+                                }
+                            }
+                        },
+                        "required": ["annotations"]
+                    }
+                }
+            }
+        ]
+    }
+]
diff --git a/json/output-tests/draft2020-12/output-schema.json b/json/output-tests/draft2020-12/output-schema.json
new file mode 100644
index 0000000..1eef288
--- /dev/null
+++ b/json/output-tests/draft2020-12/output-schema.json
@@ -0,0 +1,96 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://json-schema.org/draft/2020-12/output/schema",
+  "description": "A schema that validates the minimum requirements for validation output",
+
+  "anyOf": [
+    { "$ref": "#/$defs/flag" },
+    { "$ref": "#/$defs/basic" },
+    { "$ref": "#/$defs/detailed" },
+    { "$ref": "#/$defs/verbose" }
+  ],
+  "$defs": {
+    "outputUnit":{
+      "properties": {
+        "valid": { "type": "boolean" },
+        "keywordLocation": {
+          "type": "string",
+          "format": "json-pointer"
+        },
+        "absoluteKeywordLocation": {
+          "type": "string",
+          "format": "uri"
+        },
+        "instanceLocation": {
+          "type": "string",
+          "format": "json-pointer"
+        },
+        "error": {
+          "type": "string"
+        },
+        "errors": {
+          "$ref": "#/$defs/outputUnitArray"
+        },
+        "annotations": {
+          "$ref": "#/$defs/outputUnitArray"
+        }
+      },
+      "required": [ "valid", "keywordLocation", "instanceLocation" ],
+      "allOf": [
+        {
+          "if": {
+            "properties": {
+              "valid": { "const": false }
+            }
+          },
+          "then": {
+            "anyOf": [
+              {
+                "required": [ "error" ]
+              },
+              {
+                "required": [ "errors" ]
+              }
+            ]
+          }
+        },
+        {
+          "if": {
+            "anyOf": [
+              {
+                "properties": {
+                  "keywordLocation": {
+                    "pattern": "/\\$ref/"
+                  }
+                }
+              },
+              {
+                "properties": {
+                  "keywordLocation": {
+                    "pattern": "/\\$dynamicRef/"
+                  }
+                }
+              }
+            ]
+          },
+          "then": {
+            "required": [ "absoluteKeywordLocation" ]
+          }
+        }
+      ]
+    },
+    "outputUnitArray": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/outputUnit" }
+    },
+    "flag": {
+      "properties": {
+        "valid": { "type": "boolean" }
+      },
+      "required": [ "valid" ]
+    },
+    "basic": { "$ref": "#/$defs/outputUnit" },
+    "detailed": { "$ref": "#/$defs/outputUnit" },
+    "verbose": { "$ref": "#/$defs/outputUnit" }
+  }
+}
diff --git a/json/test-schema.json b/json/test-schema.json
index 5d25031..8339316 100644
--- a/json/test-schema.json
+++ b/json/test-schema.json
@@ -1,5 +1,6 @@
 {
     "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://json-schema.org/tests/test-schema",
     "description": "A schema for files contained within this suite",
 
     "type": "array",
diff --git a/json/tests/draft-next/dynamicRef.json b/json/tests/draft-next/dynamicRef.json
index 7c360e3..a4a7c44 100644
--- a/json/tests/draft-next/dynamicRef.json
+++ b/json/tests/draft-next/dynamicRef.json
@@ -488,13 +488,18 @@
     {
         "description": "$dynamicAnchor inside propertyDependencies",
         "schema": {
-            "$schema": "https://json-schema.org/draft/2020-12/schema",
-            "$id": "http://localhost:1234/draft2020-12/dynamicanchor-in-propertydependencies.json",
+            "$schema": "https://json-schema.org/draft/next/schema",
+            "$id": "http://localhost:1234/draft-next/dynamicanchor-in-propertydependencies.json",
             "$defs": {
                 "inner": {
                     "$id": "inner",
                     "$dynamicAnchor": "foo",
                     "type": "object",
+                    "properties": {
+                        "expectedTypes": {
+                            "type": "string"
+                        }
+                    },
                     "additionalProperties": {
                         "$dynamicRef": "#foo"
                     }
@@ -557,20 +562,6 @@
                     "anotherProperty": "a string"
                 },
                 "valid": false
-            },
-            {
-                "description": "expected missing - additional property as an object is valid",
-                "data": {
-                    "anotherProperty": {}
-                },
-                "valid": true
-            },
-            {
-                "description": "expected missing - additional property as not object is invalid",
-                "data": {
-                    "anotherProperty": 42
-                },
-                "valid": false
             }
         ]
     }
diff --git a/json/tests/draft-next/unevaluatedProperties.json b/json/tests/draft-next/unevaluatedProperties.json
index b6df08a..59c0e4e 100644
--- a/json/tests/draft-next/unevaluatedProperties.json
+++ b/json/tests/draft-next/unevaluatedProperties.json
@@ -1475,6 +1475,11 @@
         "description": "unevaluatedProperties can see inside propertyDependencies",
         "schema": {
             "$schema": "https://json-schema.org/draft/next/schema",
+            "properties": {
+                "foo": {
+                    "type": "string"
+                }
+            },
             "propertyDependencies": {
                 "foo": {
                     "foo1": { 
diff --git a/json/tox.ini b/json/tox.ini
index ec180a9..7ca9de9 100644
--- a/json/tox.ini
+++ b/json/tox.ini
@@ -5,5 +5,5 @@ skipsdist = True
 
 [testenv:sanity]
 # used just for validating the structure of the test case files themselves
-deps = jsonschema==4.6.1
+deps = jsonschema==4.17.3
 commands = {envpython} bin/jsonschema_suite check