diff options
Diffstat (limited to 'morphlib/morphloader.py')
| -rw-r--r-- | morphlib/morphloader.py | 789 |
1 files changed, 789 insertions, 0 deletions
diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py new file mode 100644 index 00000000..8289b01e --- /dev/null +++ b/morphlib/morphloader.py @@ -0,0 +1,789 @@ +# Copyright (C) 2013-2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import collections +import logging +import warnings +import yaml + +import morphlib + + +class MorphologyObsoleteFieldWarning(UserWarning): + + def __init__(self, morphology, spec, field): + self.kind = morphology['kind'] + self.morphology_name = morphology.get('name', '<unknown>') + self.stratum_name = spec.get('alias', spec['morph']) + self.field = field + + def __str__(self): + format_string = ('%(kind)s morphology %(morphology_name)s refers to ' + 'stratum %(stratum_name)s with the %(field)s field. ' + 'Defaulting to null.') + return format_string % self.__dict__ + + +class MorphologySyntaxError(morphlib.Error): + pass + + +class MorphologyNotYamlError(MorphologySyntaxError): + + def __init__(self, morphology, errmsg): + self.msg = 'Syntax error in morphology %s:\n%s' % (morphology, errmsg) + + +class NotADictionaryError(MorphologySyntaxError): + + def __init__(self, morph_filename): + self.msg = 'Not a dictionary: morphology %s' % morph_filename + + +class MorphologyValidationError(morphlib.Error): + pass + + +class UnknownKindError(MorphologyValidationError): + + def __init__(self, kind, morph_filename): + self.msg = ( + 'Unknown kind %s in morphology %s' % (kind, morph_filename)) + + +class MissingFieldError(MorphologyValidationError): + + def __init__(self, field, morphology_name): + self.field = field + self.morphology_name = morphology_name + self.msg = ( + 'Missing field %s from morphology %s' % (field, morphology_name)) + + +class InvalidFieldError(MorphologyValidationError): + + def __init__(self, field, morphology_name): + self.field = field + self.morphology_name = morphology_name + self.msg = ( + 'Field %s not allowed in morphology %s' % (field, morphology_name)) + + +class InvalidTypeError(MorphologyValidationError): + + def __init__(self, field, expected, actual, morphology_name): + self.field = field + self.expected = expected + self.actual = actual + self.morphology_name = morphology_name + self.msg = ( + 'Field %s expected type %s, got %s in morphology %s' % + (field, expected, actual, morphology_name)) + + +class ObsoleteFieldsError(MorphologyValidationError): + + def __init__(self, fields, morph_filename): + self.msg = ( + 'Morphology %s uses obsolete fields: %s' % + (morph_filename, ' '.join(fields))) + + +class UnknownArchitectureError(MorphologyValidationError): + + def __init__(self, arch, morph_filename): + self.msg = ('Unknown architecture %s in morphology %s' + % (arch, morph_filename)) + + +class NoBuildDependenciesError(MorphologyValidationError): + + def __init__(self, stratum_name, chunk_name, morph_filename): + self.msg = ( + 'Stratum %s has no build dependencies for chunk %s in %s' % + (stratum_name, chunk_name, morph_filename)) + + +class NoStratumBuildDependenciesError(MorphologyValidationError): + + def __init__(self, stratum_name, morph_filename): + self.msg = ( + 'Stratum %s has no build dependencies in %s' % + (stratum_name, morph_filename)) + + +class EmptyStratumError(MorphologyValidationError): + + def __init__(self, stratum_name, morph_filename): + self.msg = ( + 'Stratum %s has no chunks in %s' % + (stratum_name, morph_filename)) + + +class DuplicateChunkError(MorphologyValidationError): + + def __init__(self, stratum_name, chunk_name): + self.stratum_name = stratum_name + self.chunk_name = chunk_name + MorphologyValidationError.__init__( + self, 'Duplicate chunk %(chunk_name)s '\ + 'in stratum %(stratum_name)s' % locals()) + + +class EmptyRefError(MorphologyValidationError): + + def __init__(self, ref_location, morph_filename): + self.ref_location = ref_location + self.morph_filename = morph_filename + MorphologyValidationError.__init__( + self, 'Empty ref found for %(ref_location)s '\ + 'in %(morph_filename)s' % locals()) + + +class ChunkSpecRefNotStringError(MorphologyValidationError): + + def __init__(self, ref_value, chunk_name, stratum_name): + self.ref_value = ref_value + self.chunk_name = chunk_name + self.stratum_name = stratum_name + MorphologyValidationError.__init__( + self, 'Ref %(ref_value)s for %(chunk_name)s '\ + 'in stratum %(stratum_name)s is not a string' % locals()) + + +class SystemStrataNotListError(MorphologyValidationError): + + def __init__(self, system_name, strata_type): + self.system_name = system_name + self.strata_type = strata_type + typename = strata_type.__name__ + MorphologyValidationError.__init__( + self, 'System %(system_name)s has the wrong type for its strata: '\ + '%(typename)s, expected list' % locals()) + + +class DuplicateStratumError(MorphologyValidationError): + + def __init__(self, system_name, stratum_name): + self.system_name = system_name + self.stratum_name = stratum_name + MorphologyValidationError.__init__( + self, 'Duplicate stratum %(stratum_name)s '\ + 'in system %(system_name)s' % locals()) + + +class SystemStratumSpecsNotMappingError(MorphologyValidationError): + + def __init__(self, system_name, strata): + self.system_name = system_name + self.strata = strata + MorphologyValidationError.__init__( + self, 'System %(system_name)s has stratum specs '\ + 'that are not mappings.' % locals()) + + +class EmptySystemError(MorphologyValidationError): + + def __init__(self, system_name): + MorphologyValidationError.__init__( + self, 'System %(system_name)s has no strata.' % locals()) + + +class MultipleValidationErrors(MorphologyValidationError): + + def __init__(self, name, errors): + self.name = name + self.errors = errors + self.msg = 'Multiple errors when validating %(name)s:' + for error in errors: + self.msg += ('\n' + str(error)) + + +class DuplicateDeploymentNameError(MorphologyValidationError): + + def __init__(self, cluster_filename, duplicates): + self.duplicates = duplicates + self.cluster_filename = cluster_filename + morphlib.Error.__init__(self, + 'Cluster %s contains the following duplicate deployment names:%s' + % (cluster_filename, '\n ' + '\n '.join(duplicates))) + + +class MorphologyDumper(yaml.SafeDumper): + keyorder = ( + 'name', + 'kind', + 'description', + 'arch', + 'strata', + 'configuration-extensions', + 'morph', + 'repo', + 'ref', + 'unpetrify-ref', + 'build-depends', + 'build-mode', + 'artifacts', + 'max-jobs', + 'products', + 'chunks', + 'build-system', + 'pre-configure-commands', + 'configure-commands', + 'post-configure-commands', + 'pre-build-commands', + 'build-commands', + 'post-build-commands', + 'pre-install-commands', + 'install-commands', + 'post-install-commands', + 'artifact', + 'include', + 'systems', + 'deploy-defaults', + 'deploy', + 'type', + 'location', + ) + + @classmethod + def _iter_in_global_order(cls, mapping): + for key in cls.keyorder: + if key in mapping: + yield key, mapping[key] + for key in sorted(mapping.iterkeys()): + if key not in cls.keyorder: + yield key, mapping[key] + + @classmethod + def _represent_dict(cls, dumper, mapping): + return dumper.represent_mapping('tag:yaml.org,2002:map', + cls._iter_in_global_order(mapping)) + + @classmethod + def _represent_str(cls, dumper, orig_data): + fallback_representer = yaml.representer.SafeRepresenter.represent_str + try: + data = unicode(orig_data, 'ascii') + if data.count('\n') == 0: + return fallback_representer(dumper, orig_data) + except UnicodeDecodeError: + try: + data = unicode(orig_data, 'utf-8') + if data.count('\n') == 0: + return fallback_representer(dumper, orig_data) + except UnicodeDecodeError: + return fallback_representer(dumper, orig_data) + return dumper.represent_scalar(u'tag:yaml.org,2002:str', + data, style='|') + + @classmethod + def _represent_unicode(cls, dumper, data): + if data.count('\n') == 0: + return yaml.representer.SafeRepresenter.represent_unicode(dumper, + data) + return dumper.represent_scalar(u'tag:yaml.org,2002:str', + data, style='|') + + def __init__(self, *args, **kwargs): + yaml.SafeDumper.__init__(self, *args, **kwargs) + self.add_representer(dict, self._represent_dict) + self.add_representer(str, self._represent_str) + self.add_representer(unicode, self._represent_unicode) + + +class MorphologyLoader(object): + + '''Load morphologies from disk, or save them back to disk.''' + + _required_fields = { + 'chunk': [ + 'name', + ], + 'stratum': [ + 'name', + ], + 'system': [ + 'name', + 'arch', + 'strata', + ], + 'cluster': [ + 'name', + 'systems', + ], + } + + _obsolete_fields = { + 'system': [ + 'system-kind', + 'disk-size', + ], + } + + _static_defaults = { + 'chunk': { + 'description': '', + 'pre-configure-commands': None, + 'configure-commands': None, + 'post-configure-commands': None, + 'pre-build-commands': None, + 'build-commands': None, + 'post-build-commands': None, + 'pre-test-commands': None, + 'test-commands': None, + 'post-test-commands': None, + 'pre-install-commands': None, + 'install-commands': None, + 'post-install-commands': None, + 'devices': [], + 'products': [], + 'max-jobs': None, + 'build-system': 'manual', + 'build-mode': 'staging', + 'prefix': '/usr', + 'system-integration': [], + }, + 'stratum': { + 'chunks': [], + 'description': '', + 'build-depends': [], + 'products': [], + }, + 'system': { + 'description': '', + 'arch': None, + 'configuration-extensions': [], + }, + 'cluster': { + 'description': '', + }, + } + + def parse_morphology_text(self, text, morph_filename): + '''Parse a textual morphology. + + The text may be a string, or an open file handle. + + Return the new Morphology object, or raise an error indicating + the problem. This method does minimal validation: a syntactically + correct morphology is fine, even if none of the fields are + valid. It also does not set any default values for any of the + fields. See validate and set_defaults. + + ''' + + try: + obj = yaml.safe_load(text) + except yaml.error.YAMLError as e: + raise MorphologyNotYamlError(morph_filename, e) + + if not isinstance(obj, dict): + raise NotADictionaryError(morph_filename) + + return morphlib.morphology.Morphology(obj) + + def load_from_string(self, string, filename='string'): + '''Load a morphology from a string. + + Return the Morphology object. + + ''' + + m = self.parse_morphology_text(string, filename) + m.filename = filename + self.validate(m) + self.set_commands(m) + self.set_defaults(m) + return m + + def load_from_file(self, filename): + '''Load a morphology from a named file. + + Return the Morphology object. + + ''' + + with open(filename) as f: + text = f.read() + return self.load_from_string(text, filename=filename) + + def save_to_string(self, morphology): + '''Return normalised textual form of morphology.''' + + return yaml.dump(morphology.data, Dumper=MorphologyDumper, + default_flow_style=False) + + def save_to_file(self, filename, morphology): + '''Save a morphology object to a named file.''' + + text = self.save_to_string(morphology) + with morphlib.savefile.SaveFile(filename, 'w') as f: + f.write(text) + + def validate(self, morph): + '''Validate a morphology.''' + + # Validate that the kind field is there. + self._require_field('kind', morph) + + # The rest of the validation is dependent on the kind. + kind = morph['kind'] + if kind not in ('system', 'stratum', 'chunk', 'cluster'): + raise UnknownKindError(morph['kind'], morph.filename) + + required = ['kind'] + self._required_fields[kind] + obsolete = self._obsolete_fields.get(kind, []) + allowed = self._static_defaults[kind].keys() + self._require_fields(required, morph) + self._deny_obsolete_fields(obsolete, morph) + self._deny_unknown_fields(required + allowed, morph) + + getattr(self, '_validate_%s' % kind)(morph) + + def _validate_cluster(self, morph): + # Deployment names must be unique within a cluster + deployments = collections.Counter() + for system in morph['systems']: + deployments.update(system['deploy'].iterkeys()) + if 'subsystems' in system: + deployments.update(self._get_subsystem_names(system)) + duplicates = set(deployment for deployment, count + in deployments.iteritems() if count > 1) + if duplicates: + raise DuplicateDeploymentNameError(morph.filename, duplicates) + + def _get_subsystem_names(self, system): # pragma: no cover + for subsystem in system.get('subsystems', []): + for name in subsystem['deploy'].iterkeys(): + yield name + for name in self._get_subsystem_names(subsystem): + yield name + + def _validate_system(self, morph): + # A system must contain at least one stratum + strata = morph['strata'] + if (not isinstance(strata, collections.Iterable) + or isinstance(strata, collections.Mapping)): + + raise SystemStrataNotListError(morph['name'], + type(strata)) + + if not strata: + raise EmptySystemError(morph['name']) + + if not all(isinstance(o, collections.Mapping) for o in strata): + raise SystemStratumSpecsNotMappingError(morph['name'], strata) + + # All stratum names should be unique within a system. + names = set() + for spec in strata: + name = spec.get('alias', spec['morph']) + if name in names: + raise DuplicateStratumError(morph['name'], name) + names.add(name) + + # Validate stratum spec fields + self._validate_stratum_specs_fields(morph, 'strata') + + # We allow the ARMv7 little-endian architecture to be specified + # as armv7 and armv7l. Normalise. + if morph['arch'] == 'armv7': + morph['arch'] = 'armv7l' + + # Architecture name must be known. + if morph['arch'] not in morphlib.valid_archs: + raise UnknownArchitectureError(morph['arch'], morph.filename) + + def _validate_stratum(self, morph): + # Require at least one chunk. + if len(morph.get('chunks', [])) == 0: + raise EmptyStratumError(morph['name'], morph.filename) + + # All chunk names must be unique within a stratum. + names = set() + for spec in morph['chunks']: + name = spec.get('alias', spec['name']) + if name in names: + raise DuplicateChunkError(morph['name'], name) + names.add(name) + + # All chunk refs must be strings. + for spec in morph['chunks']: + if 'ref' in spec: + ref = spec['ref'] + if ref == None: + raise EmptyRefError( + spec.get('alias', spec['name']), morph.filename) + elif not isinstance(ref, basestring): + raise ChunkSpecRefNotStringError( + ref, spec.get('alias', spec['name']), morph.filename) + + # Require build-dependencies for the stratum itself, unless + # it has chunks built in bootstrap mode. + if 'build-depends' in morph: + if not isinstance(morph['build-depends'], list): + raise InvalidTypeError( + 'build-depends', list, type(morph['build-depends']), + morph['name']) + else: + for spec in morph['chunks']: + if spec.get('build-mode') in ['bootstrap', 'test']: + break + else: + raise NoStratumBuildDependenciesError( + morph['name'], morph.filename) + + # Validate build-dependencies if specified + self._validate_stratum_specs_fields(morph, 'build-depends') + + # Require build-dependencies for each chunk. + for spec in morph['chunks']: + chunk_name = spec.get('alias', spec['name']) + if 'build-depends' in spec: + if not isinstance(spec['build-depends'], list): + raise InvalidTypeError( + '%s.build-depends' % chunk_name, list, + type(spec['build-depends']), morph['name']) + else: + raise NoBuildDependenciesError( + morph['name'], chunk_name, morph.filename) + + @classmethod + def _validate_chunk(cls, morphology): + errors = [] + + if 'products' in morphology: + cls._validate_products(morphology['name'], + morphology['products'], errors) + + if len(errors) == 1: + raise errors[0] + elif errors: + raise MultipleValidationErrors(morphology['name'], errors) + + @classmethod + def _validate_products(cls, morphology_name, products, errors): + '''Validate the products field is of the correct type.''' + if (not isinstance(products, collections.Iterable) + or isinstance(products, collections.Mapping)): + raise InvalidTypeError('products', list, + type(products), morphology_name) + + for spec_index, spec in enumerate(products): + + if not isinstance(spec, collections.Mapping): + e = InvalidTypeError('products[%d]' % spec_index, + dict, type(spec), morphology_name) + errors.append(e) + continue + + cls._validate_products_spec_fields_exist(morphology_name, + spec_index, spec, errors) + + if 'include' in spec: + cls._validate_products_specs_include( + morphology_name, spec_index, spec['include'], errors) + + product_spec_required_fields = ('artifact', 'include') + @classmethod + def _validate_products_spec_fields_exist( + cls, morphology_name, spec_index, spec, errors): + + given_fields = sorted(spec.iterkeys()) + missing = (field for field in cls.product_spec_required_fields + if field not in given_fields) + for field in missing: + e = MissingFieldError('products[%d].%s' % (spec_index, field), + morphology_name) + errors.append(e) + unexpected = (field for field in given_fields + if field not in cls.product_spec_required_fields) + for field in unexpected: + e = InvalidFieldError('products[%d].%s' % (spec_index, field), + morphology_name) + errors.append(e) + + @classmethod + def _validate_products_specs_include(cls, morphology_name, spec_index, + include_patterns, errors): + '''Validate that products' include field is a list of strings.''' + # Allow include to be most iterables, but not a mapping + # or a string, since iter of a mapping is just the keys, + # and the iter of a string is a 1 character length string, + # which would also validate as an iterable of strings. + if (not isinstance(include_patterns, collections.Iterable) + or isinstance(include_patterns, collections.Mapping) + or isinstance(include_patterns, basestring)): + + e = InvalidTypeError('products[%d].include' % spec_index, list, + type(include_patterns), morphology_name) + errors.append(e) + else: + for pattern_index, pattern in enumerate(include_patterns): + pattern_path = ('products[%d].include[%d]' % + (spec_index, pattern_index)) + if not isinstance(pattern, basestring): + e = InvalidTypeError(pattern_path, str, + type(pattern), morphology_name) + errors.append(e) + + @classmethod + def _warn_obsolete_field(cls, morphology, spec, field): + warnings.warn(MorphologyObsoleteFieldWarning(morphology, spec, field), + stacklevel=2) + + @classmethod + def _validate_stratum_specs_fields(cls, morphology, specs_field): + for spec in morphology.get(specs_field, None) or []: + for obsolete_field in ('repo', 'ref'): + if obsolete_field in spec: + cls._warn_obsolete_field(morphology, spec, obsolete_field) + + def _require_field(self, field, morphology): + if field not in morphology: + raise MissingFieldError(field, morphology.filename) + + def _require_fields(self, fields, morphology): + for field in fields: + self._require_field(field, morphology) + + def _deny_obsolete_fields(self, fields, morphology): + obsolete_ones = [x for x in morphology if x in fields] + if obsolete_ones: + raise ObsoleteFieldsError(obsolete_ones, morphology.filename) + + def _deny_unknown_fields(self, allowed, morphology): + for field in morphology: + if field not in allowed: + raise InvalidFieldError(field, morphology.filename) + + def set_defaults(self, morphology): + '''Set all missing fields in the morpholoy to their defaults. + + The morphology is assumed to be valid. + + ''' + + kind = morphology['kind'] + defaults = self._static_defaults[kind] + for key in defaults: + if key not in morphology: + morphology[key] = defaults[key] + + getattr(self, '_set_%s_defaults' % kind)(morphology) + + def unset_defaults(self, morphology): + '''If a field is equal to its default, delete it. + + The morphology is assumed to be valid. + + ''' + + kind = morphology['kind'] + defaults = self._static_defaults[kind] + for key in defaults: + if key in morphology and morphology[key] == defaults[key]: + del morphology[key] + + getattr(self, '_unset_%s_defaults' % kind)(morphology) + + @classmethod + def _set_stratum_specs_defaults(cls, morphology, specs_field): + for spec in morphology.get(specs_field, None) or []: + for obsolete_field in ('repo', 'ref'): + if obsolete_field in spec: + del spec[obsolete_field] + + @classmethod + def _unset_stratum_specs_defaults(cls, morphology, specs_field): + for spec in morphology.get(specs_field, []): + for obsolete_field in ('repo', 'ref'): + if obsolete_field in spec: + del spec[obsolete_field] + + def _set_cluster_defaults(self, morph): + for system in morph.get('systems', []): + if 'deploy-defaults' not in system: + system['deploy-defaults'] = {} + if 'deploy' not in system: + system['deploy'] = {} + + def _unset_cluster_defaults(self, morph): + for system in morph.get('systems', []): + if 'deploy-defaults' in system and system['deploy-defaults'] == {}: + del system['deploy-defaults'] + if 'deploy' in system and system['deploy'] == {}: + del system['deploy'] + + def _set_system_defaults(self, morph): + self._set_stratum_specs_defaults(morph, 'strata') + + def _unset_system_defaults(self, morph): + self._unset_stratum_specs_defaults(morph, 'strata') + + def _set_stratum_defaults(self, morph): + for spec in morph['chunks']: + if 'repo' not in spec: + spec['repo'] = spec['name'] + if 'build-mode' not in spec: + spec['build-mode'] = \ + self._static_defaults['chunk']['build-mode'] + if 'prefix' not in spec: + spec['prefix'] = \ + self._static_defaults['chunk']['prefix'] + self._set_stratum_specs_defaults(morph, 'build-depends') + + def _unset_stratum_defaults(self, morph): + for spec in morph['chunks']: + if 'repo' in spec and spec['repo'] == spec['name']: + del spec['repo'] + if 'build-mode' in spec and spec['build-mode'] == \ + self._static_defaults['chunk']['build-mode']: + del spec['build-mode'] + if 'prefix' in spec and spec['prefix'] == \ + self._static_defaults['chunk']['prefix']: + del spec['prefix'] + self._unset_stratum_specs_defaults(morph, 'strata') + + def _set_chunk_defaults(self, morph): + if morph['max-jobs'] is not None: + morph['max-jobs'] = int(morph['max-jobs']) + + def _unset_chunk_defaults(self, morph): # pragma: no cover + for key in self._static_defaults['chunk']: + if key not in morph: continue + if 'commands' not in key: continue + attr = key.replace('-', '_') + default_bs = self._static_defaults['chunk']['build-system'] + bs = morphlib.buildsystem.lookup_build_system( + morph.get('build-system', default_bs)) + default_value = getattr(bs, attr) + if morph[key] == default_value: + del morph[key] + + def set_commands(self, morph): + if morph['kind'] == 'chunk': + for key in self._static_defaults['chunk']: + if 'commands' not in key: continue + if key not in morph: + attr = '_'.join(key.split('-')) + default = self._static_defaults['chunk']['build-system'] + bs = morphlib.buildsystem.lookup_build_system( + morph.get('build-system', default)) + morph[key] = getattr(bs, attr) |