From 23b363c55c755f648ae38b61f09253860bfc61d3 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Fri, 4 Dec 2015 17:54:52 +0000 Subject: Use jsonschema to validate the morphs Use jsonschema to do a first stage of validation of the morphologies. Example error messages: ERROR: strata/core.morph: core->build-depends: 'foo' is not of type 'array' ERROR: strata/core.morph: core: 'name' is a required property ERROR: strata/core.morph: core: Additional properties are not allowed ('extra-field' was unexpected) ERROR: strata/core.morph: core->chunks[0]->name: 1111 is not of type 'string' ERROR: strata/core.morph: core->chunks[0]: 'repo' is a required property on {'morph': 'strata/core/gdbm.morph', 'ref': 'e5faeaaf75ecfb705a9b643b3e4cb881ebb69f48', 'unpetrify-ref': 'gdbm-1.11', 'name': 'gdbm'} ERROR: strata/core.morph: core->chunks[0]: Additional properties are not allowed ('extra-field' was unexpected) on {'repo': 'upstream:gdbm-tarball', 'extra-field': None, 'name': 'gdbm', 'morph': 'strata/core/gdbm.morph', 'ref': 'e5faeaaf75ecfb705a9b643b3e4cb881ebb69f48', 'unpetrify-ref': 'gdbm-1.11'} Change-Id: If9fd488e16db35130d074492a93754a447ea98e1 --- morphlib/artifact_tests.py | 3 +- morphlib/artifactresolver_tests.py | 17 +- morphlib/cachekeycomputer_tests.py | 3 +- morphlib/defaults.py | 10 +- morphlib/definitions_repo.py | 4 +- morphlib/localartifactcache_tests.py | 3 +- morphlib/morphloader.py | 296 +++------------------------------- morphlib/morphloader_tests.py | 267 ++---------------------------- morphlib/plugins/deploy_plugin.py | 5 +- morphlib/plugins/diff_plugin.py | 8 +- morphlib/remoteartifactcache_tests.py | 3 +- morphlib/source_tests.py | 4 +- morphlib/sourceresolver.py | 4 +- morphlib/util.py | 44 ++++- 14 files changed, 120 insertions(+), 551 deletions(-) diff --git a/morphlib/artifact_tests.py b/morphlib/artifact_tests.py index 31e3ad7f..53185cf2 100644 --- a/morphlib/artifact_tests.py +++ b/morphlib/artifact_tests.py @@ -22,7 +22,8 @@ import morphlib class ArtifactTests(unittest.TestCase): def setUp(self): - loader = morphlib.morphloader.MorphologyLoader() + schemas = morphlib.util.read_schemas() + loader = morphlib.morphloader.MorphologyLoader(schemas=schemas) morph = loader.load_from_string( ''' name: chunk diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index 1ad9ba2d..a4924b37 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -45,7 +45,8 @@ def get_chunk_morphology(name, artifact_names=[]): text = yaml.dump({'name': name, 'kind': 'chunk'}, default_flow_style=False) - loader = morphlib.morphloader.MorphologyLoader() + schemas = morphlib.util.read_schemas() + loader = morphlib.morphloader.MorphologyLoader(schemas=schemas) morph = loader.load_from_string(text) return morph @@ -78,7 +79,8 @@ def get_stratum_morphology(name, chunks=[], build_depends=[]): "build-depends": build_depends_list}, default_flow_style=False) - loader = morphlib.morphloader.MorphologyLoader() + schemas = morphlib.util.read_schemas() + loader = morphlib.morphloader.MorphologyLoader(schemas=schemas) morph = loader.load_from_string(text) return morph @@ -248,7 +250,8 @@ class ArtifactResolverTests(unittest.TestCase): for dep in chunk_artifact.dependents)) def test_detection_of_mutual_dependency_between_two_strata(self): - loader = morphlib.morphloader.MorphologyLoader() + schemas = morphlib.util.read_schemas() + loader = morphlib.morphloader.MorphologyLoader(schemas=schemas) pool = morphlib.sourcepool.SourcePool() chunk = get_chunk_morphology('chunk1') @@ -290,8 +293,8 @@ class ArtifactResolverTests(unittest.TestCase): def test_handles_chunk_dependencies_out_of_invalid_order(self): pool = morphlib.sourcepool.SourcePool() - - loader = morphlib.morphloader.MorphologyLoader() + schemas = morphlib.util.read_schemas() + loader = morphlib.morphloader.MorphologyLoader(schemas=schemas) morph = loader.load_from_string( ''' name: stratum @@ -338,8 +341,8 @@ class ArtifactResolverTests(unittest.TestCase): def test_handles_invalid_chunk_dependencies(self): pool = morphlib.sourcepool.SourcePool() - - loader = morphlib.morphloader.MorphologyLoader() + schemas = morphlib.util.read_schemas() + loader = morphlib.morphloader.MorphologyLoader(schemas=schemas) morph = loader.load_from_string( ''' name: stratum diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index a2dff969..8153ee87 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -37,7 +37,8 @@ default_split_rules = { class CacheKeyComputerTests(unittest.TestCase): def setUp(self): - loader = morphlib.morphloader.MorphologyLoader() + schemas = morphlib.util.read_schemas() + loader = morphlib.morphloader.MorphologyLoader(schemas=schemas) self.source_pool = morphlib.sourcepool.SourcePool() for name, text in { 'chunk.morph': ''' diff --git a/morphlib/defaults.py b/morphlib/defaults.py index 86828baa..ee90956e 100644 --- a/morphlib/defaults.py +++ b/morphlib/defaults.py @@ -16,7 +16,6 @@ import cliapp -import jsonschema import yaml import os @@ -53,12 +52,9 @@ class Defaults(object): # It's OK to be empty, I guess. return build_systems, split_rules - try: - # It would be nice if this could give line numbers when it spotted - # errors. Seems tricky. - jsonschema.validate(data, self.schema) - except jsonschema.ValidationError as e: - raise cliapp.AppException('Invalid DEFAULTS file: %s' % e.message) + error = morphlib.util.validate_json(data, self.schema, 'DEFAULTS') + if error: + raise cliapp.AppException(error) build_system_data = data.get('build-systems', {}) for name, commands in build_system_data.items(): diff --git a/morphlib/definitions_repo.py b/morphlib/definitions_repo.py index 9fc4e734..f76cc401 100644 --- a/morphlib/definitions_repo.py +++ b/morphlib/definitions_repo.py @@ -235,8 +235,10 @@ class DefinitionsRepo(gitdir.GitDirectory): defaults = morphlib.defaults.Defaults(version, text=defaults_text) + schemas = morphlib.util.read_schemas(version) loader = morphlib.morphloader.MorphologyLoader( - predefined_build_systems=defaults.build_systems()) + predefined_build_systems=defaults.build_systems(), + schemas=schemas) return loader diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py index 9483f9f6..f8a0d1ee 100644 --- a/morphlib/localartifactcache_tests.py +++ b/morphlib/localartifactcache_tests.py @@ -26,7 +26,8 @@ class LocalArtifactCacheTests(unittest.TestCase): def setUp(self): self.tempfs = fs.tempfs.TempFS() - loader = morphlib.morphloader.MorphologyLoader() + schemas = morphlib.util.read_schemas() + loader = morphlib.morphloader.MorphologyLoader(schemas=schemas) morph = loader.load_from_string( ''' name: chunk diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index c0344123..e114b7dd 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -14,9 +14,7 @@ # # =*= License: GPL-2 =*= - import collections -import warnings import yaml import morphlib @@ -58,27 +56,6 @@ class MissingFieldError(MorphologyValidationError): 'Missing field %s from morphology %s' % (field, morphology_name)) -class InvalidFieldError(MorphologyValidationError): - - def __init__(self, field, morphology_name): - self.field = field - self.morphology_name = morphology_name - self.msg = ( - 'Field %s not allowed in morphology %s' % (field, morphology_name)) - - -class InvalidTypeError(MorphologyValidationError): - - def __init__(self, field, expected, actual, morphology_name): - self.field = field - self.expected = expected - self.actual = actual - self.morphology_name = morphology_name - self.msg = ( - 'Field %s expected type %s, got %s in morphology %s' % - (field, expected, actual, morphology_name)) - - class UnknownArchitectureError(MorphologyValidationError): def __init__(self, arch, morph_filename): @@ -101,14 +78,6 @@ class NoStratumBuildDependenciesError(MorphologyValidationError): (stratum_name, morph_filename)) -class EmptyStratumError(MorphologyValidationError): - - def __init__(self, stratum_name, morph_filename): - self.msg = ( - 'Stratum %s has no chunks in %s' % - (stratum_name, morph_filename)) - - class DuplicateChunkError(MorphologyValidationError): def __init__(self, stratum_name, chunk_name): @@ -119,27 +88,6 @@ class DuplicateChunkError(MorphologyValidationError): 'in stratum %(stratum_name)s' % locals()) -class EmptyRefError(MorphologyValidationError): - - def __init__(self, ref_location, morph_filename): - self.ref_location = ref_location - self.morph_filename = morph_filename - MorphologyValidationError.__init__( - self, 'Empty ref found for %(ref_location)s '\ - 'in %(morph_filename)s' % locals()) - - -class ChunkSpecRefNotStringError(MorphologyValidationError): - - def __init__(self, ref_value, chunk_name, stratum_name): - self.ref_value = ref_value - self.chunk_name = chunk_name - self.stratum_name = stratum_name - MorphologyValidationError.__init__( - self, 'Ref %(ref_value)s for %(chunk_name)s '\ - 'in stratum %(stratum_name)s is not a string' % locals()) - - class ChunkSpecConflictingFieldsError(MorphologyValidationError): def __init__(self, fields, chunk_name, stratum_name): @@ -164,17 +112,6 @@ class ChunkSpecNoBuildInstructionsError(MorphologyValidationError): 'and setting "morph: " in the stratum.' % locals()) -class SystemStrataNotListError(MorphologyValidationError): - - def __init__(self, system_name, strata_type): - self.system_name = system_name - self.strata_type = strata_type - typename = strata_type.__name__ - MorphologyValidationError.__init__( - self, 'System %(system_name)s has the wrong type for its strata: '\ - '%(typename)s, expected list' % locals()) - - class DuplicateStratumError(MorphologyValidationError): def __init__(self, system_name, stratum_name): @@ -185,23 +122,6 @@ class DuplicateStratumError(MorphologyValidationError): 'in system %(system_name)s' % locals()) -class SystemStratumSpecsNotMappingError(MorphologyValidationError): - - def __init__(self, system_name, strata): - self.system_name = system_name - self.strata = strata - MorphologyValidationError.__init__( - self, 'System %(system_name)s has stratum specs '\ - 'that are not mappings.' % locals()) - - -class EmptySystemError(MorphologyValidationError): - - def __init__(self, system_name): - MorphologyValidationError.__init__( - self, 'System %(system_name)s has no strata.' % locals()) - - class DependsOnSelfError(MorphologyValidationError): def __init__(self, name, filename): @@ -210,16 +130,6 @@ class DependsOnSelfError(MorphologyValidationError): MorphologyValidationError.__init__(self, msg) -class MultipleValidationErrors(MorphologyValidationError): - - def __init__(self, name, errors): - self.name = name - self.errors = errors - self.msg = 'Multiple errors when validating %(name)s:' - for error in errors: - self.msg += ('\n' + str(error)) - - class DuplicateDeploymentNameError(MorphologyValidationError): def __init__(self, cluster_filename, duplicates): @@ -320,24 +230,6 @@ class MorphologyLoader(object): '''Load morphologies from disk, or save them back to disk.''' - _required_fields = { - 'chunk': [ - 'name', - ], - 'stratum': [ - 'name', - ], - 'system': [ - 'name', - 'arch', - 'strata', - ], - 'cluster': [ - 'name', - 'systems', - ], - } - _static_defaults = { 'chunk': { 'description': '', @@ -380,9 +272,9 @@ class MorphologyLoader(object): }, } - def __init__(self, - predefined_build_systems={}): + def __init__(self, predefined_build_systems={}, schemas={}): self._predefined_build_systems = predefined_build_systems.copy() + self._schemas = schemas if 'manual' not in self._predefined_build_systems: self._predefined_build_systems['manual'] = \ @@ -441,78 +333,60 @@ class MorphologyLoader(object): '''Validate a morphology.''' # Validate that the kind field is there. - self._require_field('kind', morph) + if 'kind' not in morph: + raise MissingFieldError('kind', morph.filename) # The rest of the validation is dependent on the kind. kind = morph['kind'] if kind not in ('system', 'stratum', 'chunk', 'cluster'): raise UnknownKindError(morph['kind'], morph.filename) - required = ['kind'] + self._required_fields[kind] - allowed = self._static_defaults[kind].keys() - self._require_fields(required, morph) - self._deny_unknown_fields(required + allowed, morph) + error = morphlib.util.validate_json( + dict(morph), self._schemas[kind], morph.filename) + if error: + raise MorphologyValidationError(error) getattr(self, '_validate_%s' % kind)(morph) + @classmethod + def get_subsystem_names(cls, system): # pragma: no cover + for subsystem in system.get('subsystems', []): + for name in subsystem['deploy'].iterkeys(): + yield name + for name in cls.get_subsystem_names(subsystem): + yield name + def _validate_cluster(self, morph): + # Deployment names must be unique within a cluster deployments = collections.Counter() for system in morph['systems']: deployments.update(system['deploy'].iterkeys()) if 'subsystems' in system: - deployments.update(self._get_subsystem_names(system)) + deployments.update(self.get_subsystem_names(system)) duplicates = set(deployment for deployment, count in deployments.iteritems() if count > 1) if duplicates: raise DuplicateDeploymentNameError(morph.filename, duplicates) - def _get_subsystem_names(self, system): # pragma: no cover - for subsystem in system.get('subsystems', []): - for name in subsystem['deploy'].iterkeys(): - yield name - for name in self._get_subsystem_names(subsystem): - yield name - def _validate_system(self, morph): - # A system must contain at least one stratum - strata = morph['strata'] - if (not isinstance(strata, collections.Iterable) - or isinstance(strata, collections.Mapping)): - - raise SystemStrataNotListError(morph['name'], - type(strata)) - - if not strata: - raise EmptySystemError(morph['name']) - - if not all(isinstance(o, collections.Mapping) for o in strata): - raise SystemStratumSpecsNotMappingError(morph['name'], strata) + # Architecture name must be known. + if morph['arch'] not in morphlib.valid_archs: + raise UnknownArchitectureError(morph['arch'], morph.filename) # All stratum names should be unique within a system. names = set() + strata = morph['strata'] for spec in strata: name = spec['morph'] if name in names: raise DuplicateStratumError(morph['name'], name) names.add(name) - # Architecture name must be known. - if morph['arch'] not in morphlib.valid_archs: - raise UnknownArchitectureError(morph['arch'], morph.filename) - def _validate_stratum(self, morph): - # Require at least one chunk. - if len(morph.get('chunks', [])) == 0: - raise EmptyStratumError(morph['name'], morph.filename) - # Require build-dependencies for the stratum itself, unless # it has chunks built in bootstrap mode. if 'build-depends' in morph: - if not isinstance(morph['build-depends'], list): - raise InvalidTypeError( - 'build-depends', list, type(morph['build-depends']), - morph['name']) for dep in morph['build-depends']: if dep['morph'] == morph.filename: raise DependsOnSelfError(morph['name'], morph.filename) @@ -535,23 +409,6 @@ class MorphologyLoader(object): # Check each reference to a chunk. for spec in morph['chunks']: chunk_name = spec['name'] - - # All chunk refs must be strings. - if 'ref' in spec: - ref = spec['ref'] - if ref == None: - raise EmptyRefError(spec['name'], morph.filename) - elif not isinstance(ref, basestring): - raise ChunkSpecRefNotStringError( - ref, spec['name'], morph.filename) - - # The build-depends field must be a list. - if 'build-depends' in spec: - if not isinstance(spec['build-depends'], list): - raise InvalidTypeError( - '%s.build-depends' % chunk_name, list, - type(spec['build-depends']), morph['name']) - # Either 'morph' or 'build-system' must be specified. if 'morph' in spec and 'build-system' in spec: raise ChunkSpecConflictingFieldsError( @@ -560,113 +417,8 @@ class MorphologyLoader(object): raise ChunkSpecNoBuildInstructionsError( chunk_name, morph.filename) - @classmethod - def _validate_chunk(cls, morphology): - errors = [] - - if 'products' in morphology: - cls._validate_products(morphology['name'], - morphology['products'], errors) - - for key in MorphologyDumper.keyorder: - if key.endswith('-commands') and key in morphology: - cls._validate_commands(morphology['name'], key, - morphology[key], errors) - - if len(errors) == 1: - raise errors[0] - elif errors: - raise MultipleValidationErrors(morphology['name'], errors) - - @classmethod - def _validate_commands(cls, morphology_name, key, commands, errors): - if commands is None: - return - - for cmd_index, cmd in enumerate(commands): # pragma: no cover - if not isinstance(cmd, basestring): - e = InvalidTypeError('%s[%d]' % (key, cmd_index), - str, type(cmd), morphology_name) - errors.append(e) - - @classmethod - def _validate_products(cls, morphology_name, products, errors): - '''Validate the products field is of the correct type.''' - if (not isinstance(products, collections.Iterable) - or isinstance(products, collections.Mapping)): - raise InvalidTypeError('products', list, - type(products), morphology_name) - - for spec_index, spec in enumerate(products): - - if not isinstance(spec, collections.Mapping): - e = InvalidTypeError('products[%d]' % spec_index, - dict, type(spec), morphology_name) - errors.append(e) - continue - - cls._validate_products_spec_fields_exist(morphology_name, - spec_index, spec, errors) - - if 'include' in spec: - cls._validate_products_specs_include( - morphology_name, spec_index, spec['include'], errors) - - product_spec_required_fields = ('artifact', 'include') - @classmethod - def _validate_products_spec_fields_exist( - cls, morphology_name, spec_index, spec, errors): - - given_fields = sorted(spec.iterkeys()) - missing = (field for field in cls.product_spec_required_fields - if field not in given_fields) - for field in missing: - e = MissingFieldError('products[%d].%s' % (spec_index, field), - morphology_name) - errors.append(e) - unexpected = (field for field in given_fields - if field not in cls.product_spec_required_fields) - for field in unexpected: - e = InvalidFieldError('products[%d].%s' % (spec_index, field), - morphology_name) - errors.append(e) - - @classmethod - def _validate_products_specs_include(cls, morphology_name, spec_index, - include_patterns, errors): - '''Validate that products' include field is a list of strings.''' - # Allow include to be most iterables, but not a mapping - # or a string, since iter of a mapping is just the keys, - # and the iter of a string is a 1 character length string, - # which would also validate as an iterable of strings. - if (not isinstance(include_patterns, collections.Iterable) - or isinstance(include_patterns, collections.Mapping) - or isinstance(include_patterns, basestring)): - - e = InvalidTypeError('products[%d].include' % spec_index, list, - type(include_patterns), morphology_name) - errors.append(e) - else: - for pattern_index, pattern in enumerate(include_patterns): - pattern_path = ('products[%d].include[%d]' % - (spec_index, pattern_index)) - if not isinstance(pattern, basestring): - e = InvalidTypeError(pattern_path, str, - type(pattern), morphology_name) - errors.append(e) - - def _require_field(self, field, morphology): - if field not in morphology: - raise MissingFieldError(field, morphology.filename) - - def _require_fields(self, fields, morphology): - for field in fields: - self._require_field(field, morphology) - - def _deny_unknown_fields(self, allowed, morphology): - for field in morphology: - if field not in allowed: - raise InvalidFieldError(field, morphology.filename) + def _validate_chunk(self, morph): + pass def set_defaults(self, morphology): '''Set all missing fields in the morpholoy to their defaults. diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index db22264f..1eac3fbe 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -21,6 +21,7 @@ import shutil import tempfile import unittest import warnings +import yaml import morphlib @@ -48,7 +49,8 @@ def stratum_template(name): class MorphologyLoaderTests(unittest.TestCase): def setUp(self): - self.loader = morphlib.morphloader.MorphologyLoader() + schemas = morphlib.util.read_schemas() + self.loader = morphlib.morphloader.MorphologyLoader(schemas=schemas) self.tempdir = tempfile.mkdtemp() self.filename = os.path.join(self.tempdir, 'foo.morph') @@ -83,166 +85,10 @@ build-system: manual self.assertRaises( morphlib.morphloader.MissingFieldError, self.loader.validate, m) - def test_fails_to_validate_chunk_with_no_fields(self): - m = morphlib.morphology.Morphology({ - 'kind': 'chunk', - }) - self.assertRaises( - morphlib.morphloader.MissingFieldError, self.loader.validate, m) - - def test_fails_to_validate_chunk_with_invalid_field(self): - m = morphlib.morphology.Morphology({ - 'kind': 'chunk', - 'name': 'foo', - 'invalid': 'field', - }) + def test_fails_to_validate_morphology_not_compliant_with_schema(self): self.assertRaises( - morphlib.morphloader.InvalidFieldError, self.loader.validate, m) - - def test_validate_requires_products_list(self): - m = morphlib.morphology.Morphology( - kind='chunk', - name='foo', - products={ - 'foo-runtime': ['.'], - 'foo-devel': ['.'], - }) - with self.assertRaises(morphlib.morphloader.InvalidTypeError) as cm: - self.loader.validate(m) - e = cm.exception - self.assertEqual(e.field, 'products') - self.assertEqual(e.expected, list) - self.assertEqual(e.actual, dict) - self.assertEqual(e.morphology_name, 'foo') - - def test_validate_requires_products_list_of_mappings(self): - m = morphlib.morphology.Morphology( - kind='chunk', - name='foo', - products=[ - 'foo-runtime', - ]) - with self.assertRaises(morphlib.morphloader.InvalidTypeError) as cm: - self.loader.validate(m) - e = cm.exception - self.assertEqual(e.field, 'products[0]') - self.assertEqual(e.expected, dict) - self.assertEqual(e.actual, str) - self.assertEqual(e.morphology_name, 'foo') - - def test_validate_requires_products_list_required_fields(self): - m = morphlib.morphology.Morphology( - kind='chunk', - name='foo', - products=[ - { - 'factiart': 'foo-runtime', - 'cludein': [], - } - ]) - with self.assertRaises(morphlib.morphloader.MultipleValidationErrors) \ - as cm: - self.loader.validate(m) - exs = cm.exception.errors - self.assertEqual(type(exs[0]), morphlib.morphloader.MissingFieldError) - self.assertEqual(exs[0].field, 'products[0].artifact') - self.assertEqual(type(exs[1]), morphlib.morphloader.MissingFieldError) - self.assertEqual(exs[1].field, 'products[0].include') - self.assertEqual(type(exs[2]), morphlib.morphloader.InvalidFieldError) - self.assertEqual(exs[2].field, 'products[0].cludein') - self.assertEqual(type(exs[3]), morphlib.morphloader.InvalidFieldError) - self.assertEqual(exs[3].field, 'products[0].factiart') - - def test_validate_requires_products_list_include_is_list(self): - m = morphlib.morphology.Morphology( - kind='chunk', - name='foo', - products=[ - { - 'artifact': 'foo-runtime', - 'include': '.*', - } - ]) - with self.assertRaises(morphlib.morphloader.InvalidTypeError) as cm: - self.loader.validate(m) - ex = cm.exception - self.assertEqual(ex.field, 'products[0].include') - self.assertEqual(ex.expected, list) - self.assertEqual(ex.actual, str) - self.assertEqual(ex.morphology_name, 'foo') - - def test_validate_requires_products_list_include_is_list_of_strings(self): - m = morphlib.morphology.Morphology( - kind='chunk', - name='foo', - products=[ - { - 'artifact': 'foo-runtime', - 'include': [ - 123, - ] - } - ]) - with self.assertRaises(morphlib.morphloader.InvalidTypeError) as cm: - self.loader.validate(m) - ex = cm.exception - self.assertEqual(ex.field, 'products[0].include[0]') - self.assertEqual(ex.expected, str) - self.assertEqual(ex.actual, int) - self.assertEqual(ex.morphology_name, 'foo') - - - def test_fails_to_validate_stratum_with_no_fields(self): - m = morphlib.morphology.Morphology({ - 'kind': 'stratum', - }) - self.assertRaises( - morphlib.morphloader.MissingFieldError, self.loader.validate, m) - - def test_fails_to_validate_stratum_with_invalid_field(self): - m = morphlib.morphology.Morphology({ - 'kind': 'stratum', - 'name': 'foo', - 'invalid': 'field', - }) - self.assertRaises( - morphlib.morphloader.InvalidFieldError, self.loader.validate, m) - - def test_validate_requires_chunk_refs_in_stratum_to_be_strings(self): - m = morphlib.morphology.Morphology({ - 'kind': 'stratum', - 'name': 'foo', - 'build-depends': [], - 'chunks': [ - { - 'name': 'chunk', - 'repo': 'test:repo', - 'ref': 1, - 'build-depends': [] - } - ] - }) - with self.assertRaises( - morphlib.morphloader.ChunkSpecRefNotStringError): - self.loader.validate(m) - - def test_fails_to_validate_stratum_with_empty_refs_for_a_chunk(self): - m = morphlib.morphology.Morphology({ - 'kind': 'stratum', - 'name': 'foo', - 'build-depends': [], - 'chunks' : [ - { - 'name': 'chunk', - 'repo': 'test:repo', - 'ref': None, - 'build-depends': [] - } - ] - }) - with self.assertRaises( - morphlib.morphloader.EmptyRefError): - self.loader.validate(m) + morphlib.morphloader.MorphologyValidationError, + self.loader.load_from_string, 'kind: chunk', 'test') def test_fails_to_validate_stratum_which_build_depends_on_self(self): text = '''\ @@ -258,25 +104,6 @@ chunks: morphlib.morphloader.DependsOnSelfError, self.loader.load_from_string, text, 'strata/bad-stratum.morph') - def test_fails_to_validate_system_with_no_fields(self): - m = morphlib.morphology.Morphology({ - 'kind': 'system', - }) - self.assertRaises( - morphlib.morphloader.MissingFieldError, self.loader.validate, m) - - def test_fails_to_validate_system_with_invalid_field(self): - m = morphlib.morphology.Morphology( - kind="system", - name="foo", - arch="blah", - strata=[ - {'morph': 'bar'}, - ], - invalid='field') - self.assertRaises( - morphlib.morphloader.InvalidFieldError, self.loader.validate, m) - def test_fails_to_validate_morphology_with_unknown_kind(self): m = morphlib.morphology.Morphology({ 'kind': 'invalid', @@ -289,17 +116,13 @@ chunks: { "kind": "system", "name": "foo", - "arch": "x86-64", + "arch": "x86_64", "strata": [ { "morph": "stratum", - "repo": "test1", - "ref": "ref" }, { "morph": "stratum", - "repo": "test2", - "ref": "ref" } ] }) @@ -355,20 +178,6 @@ chunks: m['chunks'][0]['build-mode'] = 'bootstrap' self.loader.validate(m) - def test_validate_stratum_build_deps_are_list(self): - m = stratum_template("stratum-invalid-bdeps") - m['build-depends'] = 0.1 - self.assertRaises( - morphlib.morphloader.InvalidTypeError, - self.loader.validate, m) - - def test_validate_chunk_build_deps_are_list(self): - m = stratum_template("stratum-invalid-bdeps") - m['chunks'][0]['build-depends'] = 0.1 - self.assertRaises( - morphlib.morphloader.InvalidTypeError, - self.loader.validate, m) - def test_validate_chunk_has_build_instructions(self): m = stratum_template("stratum-no-build-instructions") del m['chunks'][0]['build-system'] @@ -383,67 +192,20 @@ chunks: morphlib.morphloader.ChunkSpecConflictingFieldsError, self.loader.validate, m) - def test_validate_requires_chunks_in_strata(self): - m = stratum_template("stratum-no-chunks") - del m['chunks'] - self.assertRaises( - morphlib.morphloader.EmptyStratumError, - self.loader.validate, m) - - def test_validate_requires_strata_in_system(self): - m = morphlib.morphology.Morphology( - name='system', - kind='system', - arch='testarch') - self.assertRaises( - morphlib.morphloader.MissingFieldError, - self.loader.validate, m) - - def test_validate_requires_list_of_strata_in_system(self): - for v in (None, {}): - m = morphlib.morphology.Morphology( - name='system', - kind='system', - arch='testarch', - strata=v) - with self.assertRaises( - morphlib.morphloader.SystemStrataNotListError) as cm: - - self.loader.validate(m) - self.assertEqual(cm.exception.strata_type, type(v)) - - def test_validate_requires_non_empty_strata_in_system(self): - m = morphlib.morphology.Morphology( - name='system', - kind='system', - arch='testarch', - strata=[]) - self.assertRaises( - morphlib.morphloader.EmptySystemError, - self.loader.validate, m) - - def test_validate_requires_stratum_specs_in_system(self): - m = morphlib.morphology.Morphology( - name='system', - kind='system', - arch='testarch', - strata=["foo"]) - with self.assertRaises( - morphlib.morphloader.SystemStratumSpecsNotMappingError) as cm: - - self.loader.validate(m) - self.assertEqual(cm.exception.strata, ["foo"]) - def test_validate_requires_unique_deployment_names_in_cluster(self): - subsystem = [{'morph': 'baz', 'deploy': {'foobar': None}}] + subsystem = [{'morph': 'baz', + 'deploy': { + 'foobar': { 'type': 'foo', 'location': 'bar'}}}] m = morphlib.morphology.Morphology( name='cluster', kind='cluster', systems=[{'morph': 'foo', - 'deploy': {'deployment': {}}, + 'deploy': + {'deployment': {'type': 'foo', 'location': 'bar'}}, 'subsystems': subsystem}, {'morph': 'bar', - 'deploy': {'deployment': {}}, + 'deploy': + {'deployment': {'type': 'foo', 'location': 'bar'}}, 'subsystems': subsystem}]) with self.assertRaises( morphlib.morphloader.DuplicateDeploymentNameError) as cm: @@ -537,7 +299,6 @@ build-system: manual 'name': 'foo', }) self.loader.set_defaults(m) - self.loader.validate(m) self.assertEqual( dict(m), { diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 18ea8d81..e925597e 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -340,11 +340,10 @@ class DeployPlugin(cliapp.Plugin): cluster_filename = morphlib.util.sanitise_morphology_path(args[0]) cluster_filename = definitions_repo.relative_path(cluster_filename) - loader = morphlib.morphloader.MorphologyLoader() + loader = definitions_repo.get_morphology_loader() cluster_text = definitions_repo.read_file(cluster_filename) cluster_morphology = loader.load_from_string(cluster_text, filename=cluster_filename) - if cluster_morphology['kind'] != 'cluster': raise cliapp.AppException( "Error: morph deployment commands are only supported for " @@ -357,7 +356,7 @@ class DeployPlugin(cliapp.Plugin): for system in cluster_morphology['systems']: all_deployments.update(system['deploy'].iterkeys()) if 'subsystems' in system: - all_subsystems.update(loader._get_subsystem_names(system)) + all_subsystems.update(loader.get_subsystem_names(system)) for item in args[1:]: if not item in all_deployments: break diff --git a/morphlib/plugins/diff_plugin.py b/morphlib/plugins/diff_plugin.py index de4ca0b9..5a59c95d 100644 --- a/morphlib/plugins/diff_plugin.py +++ b/morphlib/plugins/diff_plugin.py @@ -16,6 +16,7 @@ import cliapp +import morphlib from morphlib.buildcommand import BuildCommand from morphlib.cmdline_parse_utils import (definition_lists_synopsis, parse_definition_lists) @@ -23,6 +24,8 @@ from morphlib.morphologyfinder import MorphologyFinder from morphlib.morphloader import MorphologyLoader from morphlib.morphset import MorphologySet +from morphlib.definitions_version import check_version_file + class DiffPlugin(cliapp.Plugin): @@ -99,9 +102,12 @@ class DiffPlugin(cliapp.Plugin): def get_systems((reponame, ref, definitions)): 'Convert a definition path list into a list of systems' - ml = MorphologyLoader() repo = self.bc.repo_cache.get_updated_repo(reponame, ref=ref) mf = MorphologyFinder(gitdir=repo, ref=ref) + version_text = mf.read_file('VERSION') + definitons_version = check_version_file(version_text) + schemas = morphlib.util.read_schemas(definitons_version) + ml = MorphologyLoader(schemas=schemas) # We may have been given an empty set of definitions as input, in # which case we instead use every we find. if not definitions: diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py index 18bef13f..571f1d61 100644 --- a/morphlib/remoteartifactcache_tests.py +++ b/morphlib/remoteartifactcache_tests.py @@ -23,7 +23,8 @@ import morphlib class RemoteArtifactCacheTests(unittest.TestCase): def setUp(self): - loader = morphlib.morphloader.MorphologyLoader() + schemas = morphlib.util.read_schemas() + loader = morphlib.morphloader.MorphologyLoader(schemas=schemas) morph = loader.load_from_string( ''' name: chunk diff --git a/morphlib/source_tests.py b/morphlib/source_tests.py index 4f6006e9..6bf09806 100644 --- a/morphlib/source_tests.py +++ b/morphlib/source_tests.py @@ -30,7 +30,9 @@ class SourceTests(unittest.TestCase): self.original_ref = 'original/ref' self.sha1 = 'CAFEF00D' self.tree = 'F000000D' - loader = morphlib.morphloader.MorphologyLoader() + + schemas = morphlib.util.read_schemas() + loader = morphlib.morphloader.MorphologyLoader(schemas=schemas) self.morphology = loader.load_from_string(self.morphology_text) self.filename = 'foo.morph' self.source, = morphlib.source.make_sources(self.repo_name, diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 5d04ece9..2f6dc35b 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -408,8 +408,10 @@ class SourceResolver(object): self._get_defaults( definitions_checkout_dir, definitions_version) + schemas = morphlib.util.read_schemas(definitions_version) morph_loader = morphlib.morphloader.MorphologyLoader( - predefined_build_systems=predefined_build_systems) + predefined_build_systems=predefined_build_systems, + schemas=schemas) # First, process the system and its stratum morphologies. These # will all live in the same Git repository, and will point to diff --git a/morphlib/util.py b/morphlib/util.py index 0c3291c2..857a833d 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -25,6 +25,8 @@ import subprocess import textwrap import tempfile import sys +import yaml +import jsonschema import fs.osfs @@ -780,7 +782,47 @@ class ProgressBar(object): sys.stderr.flush() -def schemas_directory(version): # pragma: no cover +def schemas_directory(version=None): # pragma: no cover '''Returns a path to the schemas/ subdirectory of the 'morphlib' module.''' + if not version: + version = morphlib.definitions_version.SUPPORTED_VERSIONS[-1] code_dir = os.path.dirname(morphlib.__file__) return os.path.join(code_dir, 'schemas', str(version)) + +def read_schemas(version=None): # pragma: no cover + schemas = {} + schemas_dir = schemas_directory(version) + for f in os.listdir(schemas_dir): + kind, ext = os.path.splitext(f) + if ext == '.json-schema': + with open(os.path.join(schemas_dir, f)) as ff: + schemas[kind] = yaml.load(ff) + return schemas + +def validate_json(json, schema, filename): # pragma: no cover + + def path_to_string(path): + basename, _ = os.path.splitext( + os.path.basename(filename)) + if basename != filename: + path_str = '{filename}: {basename}'.format(filename=filename, + basename=basename) + else: + path_str = '{filename}'.format(filename=filename) + for p in path: + if isinstance(p, basestring): + path_str += '->{subpath}'.format(subpath=p) + else: + path_str += '[{index}]'.format(index=p) + return path_str + + try: + jsonschema.validate(json, schema) + except jsonschema.ValidationError as e: + path_str = path_to_string(e.path) + instance = '' + if isinstance(e.instance, dict) and e.instance != json: + instance = ' on %s' % e.instance + return '{path}: {message}{instance}'.format(path=path_str, + message=e.message, + instance=instance) -- cgit v1.2.1