path: root/json/bin/jsonschema_suite

#! /usr/bin/env python3
from pathlib import Path
import argparse
import errno
import json
import os
import random
import shutil
import sys
import textwrap
import unittest
import warnings

try:
    import jsonschema.validators
except ImportError:
    jsonschema = None
    VALIDATORS = {}
else:
    VALIDATORS = {
        "draft3": jsonschema.validators.Draft3Validator,
        "draft4": jsonschema.validators.Draft4Validator,
        "draft6": jsonschema.validators.Draft6Validator,
        "draft7": jsonschema.validators.Draft7Validator,
        "draft2019-09": jsonschema.validators.Draft201909Validator,
        "draft2020-12": jsonschema.validators.Draft202012Validator,
        "latest": jsonschema.validators.Draft202012Validator,
    }


ROOT_DIR = Path(__file__).parent.parent
SUITE_ROOT_DIR = ROOT_DIR / "tests"
REMOTES_DIR = ROOT_DIR / "remotes"

TESTSUITE_SCHEMA = json.loads((ROOT_DIR / "test-schema.json").read_text())


def files(paths):
    """
    Each test file in the provided paths, as an array of test cases.
    """
    for path in paths:
        yield path, json.loads(path.read_text())


def cases(paths):
    """
    Each test case within each file in the provided paths.
    """
    for _, test_file in files(paths):
        yield from test_file


def tests(paths):
    """
    Each individual test within all cases within the provided paths.
    """
    for case in cases(paths):
        for test in case["tests"]:
            test["schema"] = case["schema"]
            yield test


def collect(root_dir):
    """
    All of the test file paths within the given root directory, recursively.
    """
    return root_dir.glob("**/*.json")


class SanityTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        print(f"Looking for tests in {SUITE_ROOT_DIR}")
        print(f"Looking for remotes in {REMOTES_DIR}")

        cls.test_files = list(collect(SUITE_ROOT_DIR))
        assert cls.test_files, "Didn't find the test files!"
        print(f"Found {len(cls.test_files)} test files")

        cls.remote_files = list(collect(REMOTES_DIR))
        assert cls.remote_files, "Didn't find the remote files!"
        print(f"Found {len(cls.remote_files)} remote files")

    def assertUnique(self, iterable):
        """
        Assert that the elements of an iterable are unique.
        """

        seen, duplicated = set(), set()
        for each in iterable:
            if each in seen:
                duplicated.add(each)
            seen.add(each)
        self.assertFalse(duplicated, "Elements are not unique.")

    def assertFollowsDescriptionStyle(self, description):
        """
        Instead of saying "test that X frobs" or "X should frob" use "X frobs".

        See e.g. https://jml.io/pages/test-docstrings.html

        This test isn't comprehensive (it doesn't catch all the extra
        verbiage there), but it's just to catch whatever it manages to
        cover.
        """

        message = (
            "In descriptions, don't say 'Test that X frobs' or 'X should "
            "frob' or 'X should be valid'. Just say 'X frobs' or 'X is "
            "valid'. It's shorter, and the test suite is entirely about "
            "what *should* be already. "
            "See https://jml.io/pages/test-docstrings.html for help."
        )
        self.assertNotRegex(description, r"\bshould\b", message)
        self.assertNotRegex(description, r"(?i)\btest(s)? that\b", message)

    def test_all_test_files_are_valid_json(self):
        """
        All test files contain valid JSON.
        """
        for path in self.test_files:
            with self.subTest(path=path):
                try:
                    json.loads(path.read_text())
                except ValueError as error:
                    self.fail(f"{path} contains invalid JSON ({error})")

    def test_all_remote_files_are_valid_json(self):
        """
        All remote files contain valid JSON.
        """
        for path in self.remote_files:
            with self.subTest(path=path):
                try:
                    json.loads(path.read_text())
                except ValueError as error:
                    self.fail(f"{path} contains invalid JSON ({error})")

    def test_all_case_descriptions_have_reasonable_length(self):
        """
        All cases have reasonably long descriptions.
        """
        for case in cases(self.test_files):
            with self.subTest(description=case["description"]):
                self.assertLess(
                    len(case["description"]),
                    150,
                    "Description is too long (keep it to less than 150 chars)."
                )

    def test_all_test_descriptions_have_reasonable_length(self):
        """
        All tests have reasonably long descriptions.
        """
        for count, test in enumerate(tests(self.test_files)):
            with self.subTest(description=test["description"]):
                self.assertLess(
                    len(test["description"]),
                    70,
                    "Description is too long (keep it to less than 70 chars)."
                )
        print(f"Found {count} tests.")

    def test_all_case_descriptions_are_unique(self):
        """
        All cases have unique descriptions in their files.
        """
        for path, cases in files(self.test_files):
            with self.subTest(path=path):
                self.assertUnique(case["description"] for case in cases)

    def test_all_test_descriptions_are_unique(self):
        """
        All test cases have unique test descriptions in their tests.
        """
        for count, case in enumerate(cases(self.test_files)):
            with self.subTest(description=case["description"]):
                self.assertUnique(
                    test["description"] for test in case["tests"]
                )
        print(f"Found {count} test cases.")

    def test_case_descriptions_do_not_use_modal_verbs(self):
        for case in cases(self.test_files):
            with self.subTest(description=case["description"]):
                self.assertFollowsDescriptionStyle(case["description"])

    def test_test_descriptions_do_not_use_modal_verbs(self):
        for test in tests(self.test_files):
            with self.subTest(description=test["description"]):
                self.assertFollowsDescriptionStyle(test["description"])

    @unittest.skipIf(jsonschema is None, "Validation library not present!")
    def test_all_schemas_are_valid(self):
        """
        All schemas are valid under their metaschemas.
        """
        for version in SUITE_ROOT_DIR.iterdir():
            if not version.is_dir():
                continue

            Validator = VALIDATORS.get(version.name)
            if Validator is not None:
                test_files = collect(version)
                for case in cases(test_files):
                    with self.subTest(case=case):
                        try:
                            Validator.check_schema(case["schema"])
                        except jsonschema.SchemaError:
                            self.fail(
                                "Found an invalid schema."
                                "See the traceback for details on why."
                            )
            else:
                warnings.warn(f"No schema validator for {version.name}")

    @unittest.skipIf(jsonschema is None, "Validation library not present!")
    def test_suites_are_valid(self):
        """
        All test files are valid under test-schema.json.
        """
        Validator = jsonschema.validators.validator_for(TESTSUITE_SCHEMA)
        validator = Validator(TESTSUITE_SCHEMA)
        for path, cases in files(self.test_files):
            with self.subTest(path=path):
                try:
                    validator.validate(cases)
                except jsonschema.ValidationError as error:
                    self.fail(str(error))


def main(arguments):
    if arguments.command == "check":
        suite = unittest.TestLoader().loadTestsFromTestCase(SanityTests)
        result = unittest.TextTestRunner().run(suite)
        sys.exit(not result.wasSuccessful())
    elif arguments.command == "flatten":
        selected_cases = [case for case in cases(collect(arguments.version))]

        if arguments.randomize:
            random.shuffle(selected_cases)

        json.dump(selected_cases, sys.stdout, indent=4, sort_keys=True)
    elif arguments.command == "remotes":
        remotes = {}
        for path in collect(REMOTES_DIR):
            relative_path = os.path.relpath(path, REMOTES_DIR)
            remotes[relative_path] = json.loads(path.read_text())
        json.dump(remotes, sys.stdout, indent=4, sort_keys=True)
    elif arguments.command == "dump_remotes":
        if arguments.update:
            shutil.rmtree(arguments.out_dir, ignore_errors=True)

        try:
            shutil.copytree(REMOTES_DIR, arguments.out_dir)
        except FileExistsError:
            print(f"{arguments.out_dir} already exists. Aborting.")
            sys.exit(1)
    elif arguments.command == "serve":
        try:
            import flask
        except ImportError:
            print(textwrap.dedent("""
                The Flask library is required to serve the remote schemas.

                You can install it by running `pip install Flask`.

                Alternatively, see the `jsonschema_suite remotes` or
                `jsonschema_suite dump_remotes` commands to create static files
                that can be served with your own web server.
            """.strip("\n")))
            sys.exit(1)

        app = flask.Flask(__name__)

        @app.route("/<path:path>")
        def serve_path(path):
            return flask.send_from_directory(REMOTES_DIR, path)

        app.run(port=1234)


parser = argparse.ArgumentParser(
    description="JSON Schema Test Suite utilities",
)
subparsers = parser.add_subparsers(
    help="utility commands", dest="command", metavar="COMMAND"
)
subparsers.required = True

check = subparsers.add_parser("check", help="Sanity check the test suite.")

flatten = subparsers.add_parser(
    "flatten",
    help="Output a flattened file containing a selected version's test cases."
)
flatten.add_argument(
    "--randomize",
    action="store_true",
    help="Randomize the order of the outputted cases.",
)
flatten.add_argument(
    "version", help="The directory containing the version to output",
)

remotes = subparsers.add_parser(
    "remotes",
    help="Output the expected URLs and their associated schemas for remote "
         "ref tests as a JSON object."
)

dump_remotes = subparsers.add_parser(
    "dump_remotes", help="Dump the remote ref schemas into a file tree",
)
dump_remotes.add_argument(
    "--update",
    action="store_true",
    help="Update the remotes in an existing directory.",
)
dump_remotes.add_argument(
    "--out-dir",
    default=REMOTES_DIR,
    type=os.path.abspath,
    help="The output directory to create as the root of the file tree",
)

serve = subparsers.add_parser(
    "serve",
    help="Start a webserver to serve schemas used by remote ref tests."
)

if __name__ == "__main__":
    main(parser.parse_args())