path: root/morphlib/morphloader_tests.py

# Copyright (C) 2013-2015  Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# =*= License: GPL-2 =*=


import contextlib
import os
import shutil
import tempfile
import unittest
import warnings
import yaml
import textwrap

import morphlib


def stratum_template(name):
    '''Returns a valid example stratum, with one chunk reference.'''
    m = morphlib.morphology.Morphology(yaml.load('''
    name: name
    kind: stratum
    build-depends:
    - morph: foo
    chunks:
    - name: chunk
      repo: test:repo
      ref: sha1
      build-system: manual
    '''))
    return m


class MorphologyLoaderTests(unittest.TestCase):

    def setUp(self):
        schemas = morphlib.util.read_schemas()
        self.loader = morphlib.morphloader.MorphologyLoader(schemas=schemas)
        self.tempdir = tempfile.mkdtemp()
        self.filename = os.path.join(self.tempdir, 'foo.morph')

    def tearDown(self):
        shutil.rmtree(self.tempdir)

    def test_parses_yaml_from_string(self):
        string = '''
        name: foo
        kind: chunk
        build-system: manual
        '''
        morph = self.loader.load_from_string(string, 'test')
        self.assertEqual(morph['kind'], 'chunk')
        self.assertEqual(morph['name'], 'foo')
        self.assertEqual(morph['build-system'], 'manual')

    def test_fails_to_parse_utter_garbage(self):
        self.assertRaises(
            morphlib.morphloader.MorphologySyntaxError,
            self.loader.load_from_string, ',,,', 'test')

    def test_fails_to_parse_non_dict(self):
        self.assertRaises(
            morphlib.morphloader.NotADictionaryError,
            self.loader.load_from_string, '- item1\n- item2\n', 'test')

    def test_fails_to_validate_dict_without_kind(self):
        m = morphlib.morphology.Morphology(
            invalid='field'
        )
        self.assertRaises(
            morphlib.morphloader.MissingFieldError, self.loader.validate, m)

    def test_fails_to_validate_morphology_not_compliant_with_schema(self):
        self.assertRaises(
            morphlib.morphloader.MorphologyValidationError,
            self.loader.load_from_string, 'kind: chunk', 'test')

    def test_fails_to_validate_stratum_with_a_missing_path(self):
        m = morphlib.morphology.Morphology({
            'kind': 'stratum',
            'name': 'foo',
            'build-depends': [],
            'chunks': [
                {
                    'name': 'chunk',
                    'repo': 'test:repo',
                    'ref': 'master',
                    'build-system': 'manual',
                    'build-depends': [],
                    'extra-sources':
                        [
                            {
                                'repo': 'foo',
                                'path': 'somepath',
                                'extra-sources':
                                    [
                                        {
                                            'repo': 'bar',
                                            'ref': 'master'
                                        }
                                    ]
                            }
                        ]
                }
            ]
        })
        self.assertRaises(
            morphlib.morphloader.MissingFieldError, self.loader.validate, m)

    def test_fails_to_validate_stratum_with_invalid_path(self):
        m = morphlib.morphology.Morphology({
            'kind': 'stratum',
            'name': 'foo',
            'build-depends': [],
            'chunks': [
                {
                    'name': 'chunk',
                    'repo': 'test:repo',
                    'ref': 'master',
                    'build-system': 'manual',
                    'build-depends': [],
                    'extra-sources':
                        [
                            {
                                'repo': 'foo',
                                'path': '../foo'
                            }
                        ]
                }
            ]
        })
        self.assertRaises(
            morphlib.morphloader.InvalidPathError, self.loader.validate, m)

    def test_fails_to_validate_stratum_which_build_depends_on_self(self):
        text = '''
        name: bad-stratum
        kind: stratum
        build-depends:
        - morph: strata/bad-stratum.morph
        chunks:
        - name: chunk
          repo: test:repo
          ref: foo
        '''
        self.assertRaises(
            morphlib.morphloader.DependsOnSelfError,
            self.loader.load_from_string, text, 'strata/bad-stratum.morph')

    def test_fails_to_validate_morphology_with_unknown_kind(self):
        m = morphlib.morphology.Morphology(
            kind='invalid'
        )
        self.assertRaises(
            morphlib.morphloader.UnknownKindError, self.loader.validate, m)

    def test_validate_requires_unique_stratum_names_within_a_system(self):
        m = morphlib.morphology.Morphology(yaml.load('''
        kind: system
        name: foo
        arch: x86_64
        strata:
        - morph: stratum
        - morph: stratum
        '''))
        self.assertRaises(morphlib.morphloader.DuplicateStratumError,
                          self.loader.validate, m)

    def test_validate_requires_unique_chunk_names_within_a_stratum(self):
        m = morphlib.morphology.Morphology(yaml.load('''
        kind: stratum
        name: foo
        build-depends:
        - morph: bar
        chunks:
        - name: chunk
          repo: test1
          ref: ref
        - name: chunk
          repo: test2
          ref: ref
        '''))
        self.assertRaises(morphlib.morphloader.DuplicateChunkError,
                          self.loader.validate, m)

    def test_validate_requires_a_valid_architecture(self):
        text = '''
        kind: system
        name: foo
        arch: blah
        strata:
        - morph: bar
        '''
        self.assertRaises(
            morphlib.morphloader.UnknownArchitectureError,
            self.loader.load_from_string, text)

    def test_validate_requires_build_deps_or_bootstrap_mode_for_strata(self):
        m = stratum_template("stratum-no-bdeps-no-bootstrap")

        self.loader.validate(m)

        del m['build-depends']
        self.assertRaises(
            morphlib.morphloader.NoStratumBuildDependenciesError,
            self.loader.validate, m)

        m['chunks'][0]['build-mode'] = 'bootstrap'
        self.loader.validate(m)

    def test_validate_chunk_has_build_instructions(self):
        m = stratum_template("stratum-no-build-instructions")
        del m['chunks'][0]['build-system']
        self.assertRaises(
            morphlib.morphloader.ChunkSpecNoBuildInstructionsError,
            self.loader.validate, m)

    def test_validate_chunk_conflicting_build_instructions(self):
        m = stratum_template("stratum-conflicting-build-instructions")
        m['chunks'][0]['morph'] = 'conflicting-information'
        self.assertRaises(
            morphlib.morphloader.ChunkSpecConflictingFieldsError,
            self.loader.validate, m)

    def test_validate_requires_unique_deployment_names_in_cluster(self):
        subsystem = [{'morph': 'baz',
                      'deploy': {
                        'foobar': { 'type': 'foo', 'location': 'bar'}}}]
        m = morphlib.morphology.Morphology(
            name='cluster',
            kind='cluster',
            systems=[{'morph': 'foo',
                      'deploy':
                          {'deployment': {'type': 'foo', 'location': 'bar'}},
                      'subsystems': subsystem},
                     {'morph': 'bar',
                      'deploy':
                          {'deployment': {'type': 'foo', 'location': 'bar'}},
                      'subsystems': subsystem}])
        with self.assertRaises(
                morphlib.morphloader.DuplicateDeploymentNameError) as cm:
            self.loader.validate(m)
        ex = cm.exception
        self.assertIn('foobar', ex.duplicates)
        self.assertIn('deployment', ex.duplicates)

    def test_loads_yaml_from_string(self):
        string = '''
        name: foo
        kind: chunk
        build-system: manual
        '''
        morph = self.loader.load_from_string(string)
        self.assertEqual(morph['kind'], 'chunk')
        self.assertEqual(morph['name'], 'foo')
        self.assertEqual(morph['build-system'], 'manual')

    def test_loads_json_from_string(self):
        string = '''
        name: foo
        kind: chunk
        build-system: manual
        '''
        morph = self.loader.load_from_string(string)
        self.assertEqual(morph['kind'], 'chunk')
        self.assertEqual(morph['name'], 'foo')
        self.assertEqual(morph['build-system'], 'manual')

    def test_loads_from_file(self):
        with open(self.filename, 'w') as f:
            f.write('''
            name: foo
            kind: chunk
            build-system: manual
            ''')
        morph = self.loader.load_from_file(self.filename)
        self.assertEqual(morph['kind'], 'chunk')
        self.assertEqual(morph['name'], 'foo')
        self.assertEqual(morph['build-system'], 'manual')

    def test_saves_to_string(self):
        morph = morphlib.morphology.Morphology(yaml.load('''
        name: foo
        kind: chunk
        build-system: manual
        '''))
        text = self.loader.save_to_string(morph)

        # The following verifies that the YAML is written in a normalised
        # fashion.
        self.assertEqual(text, textwrap.dedent('''\
        name: foo
        kind: chunk
        build-system: manual
        '''))

    def test_saves_to_file(self):
        morph = morphlib.morphology.Morphology(yaml.load('''
        name: foo
        kind: chunk
        build-system: manual
        '''))
        self.loader.save_to_file(self.filename, morph)

        with open(self.filename) as f:
            text = f.read()

        # The following verifies that the YAML is written in a normalised
        # fashion.
        self.assertEqual(text, textwrap.dedent('''\
        name: foo
        kind: chunk
        build-system: manual
        '''))

    def test_validate_does_not_set_defaults(self):
        m = morphlib.morphology.Morphology(yaml.load('''
        kind: chunk
        name: foo
        '''))
        self.loader.validate(m)
        self.assertEqual(sorted(m.keys()), sorted(['kind', 'name']))

    def test_sets_defaults_for_chunks(self):
        m = morphlib.morphology.Morphology(yaml.load('''
        kind: chunk
        name: foo
        '''))
        self.loader.set_defaults(m)
        self.assertEqual(dict(m), yaml.load('''
        kind: chunk
        name: foo
        description: ''
        build-system: manual
        build-mode: staging

        configure-commands:
        pre-configure-commands:
        post-configure-commands:

        build-commands:
        pre-build-commands:
        post-build-commands:

        test-commands:
        pre-test-commands:
        post-test-commands:

        install-commands:
        pre-install-commands:
        post-install-commands:

        strip-commands:
        pre-strip-commands:
        post-strip-commands:

        extra-sources: []
        products: []
        system-integration: []
        devices: []
        max-jobs:
        prefix: /usr'''))

    def test_sets_defaults_for_strata(self):
        m = morphlib.morphology.Morphology(yaml.load('''
        kind: stratum
        name: foo
        chunks:
        - name: bar
          repo: bar
          ref: bar
          morph: bar
          build-mode: bootstrap
          build-depends: []
        '''))
        self.loader.set_defaults(m)
        self.loader.validate(m)
        self.assertEqual(dict(m), yaml.load('''
        kind: stratum
        name: foo
        description: ''
        build-depends: []
        chunks:
        - name: bar
          repo: bar
          ref: bar
          morph: bar
          build-mode: bootstrap
          build-depends: []
          extra-sources: []
          prefix: '/usr'
        products: []
        '''))

    def test_sets_defaults_for_system(self):
        m = morphlib.morphology.Morphology(yaml.load('''
        kind: system
        name: foo
        arch: testarch
        strata:
        - morph: bar
        '''))
        self.loader.set_defaults(m)
        self.assertEqual(dict(m), yaml.load('''
        kind: system
        name: foo
        description: ''
        arch: testarch
        strata:
        - morph: bar
        configuration-extensions: []
        '''))

    def test_sets_defaults_for_cluster(self):
        m = morphlib.morphology.Morphology(yaml.load('''
        name: foo
        kind: cluster
        systems:
        - morph: foo
        - morph: bar
        '''))
        self.loader.set_defaults(m)
        self.loader.validate(m)
        self.assertEqual(m['systems'], yaml.load('''
        - morph: foo
          deploy-defaults: {}
          deploy: {}
        - morph: bar
          deploy-defaults: {}
          deploy: {}
        '''))

    def test_sets_stratum_chunks_name_from_repo(self):
        m = morphlib.morphology.Morphology(yaml.load('''
        name: foo
        kind: stratum
        chunks:
        - name: le-chunk
          ref: ref
          build-system: manual
          build-depends: []
        '''))
        self.loader.set_defaults(m)
        self.loader.validate(m)
        self.assertEqual(m['chunks'][0]['repo'], 'le-chunk')

    def test_convertes_max_jobs_to_an_integer(self):
        m = morphlib.morphology.Morphology(yaml.load('''
        name: foo
        kind: chunk
        max-jobs: "42"
        '''))
        self.loader.set_defaults(m)
        self.assertEqual(m['max-jobs'], 42)

    def test_parses_simple_cluster_morph(self):
        string = '''
        name: foo
        kind: cluster
        systems:
        - morph: bar
          deploy: {}
        '''
        m = self.loader.load_from_string(string, 'test')
        self.loader.set_defaults(m)
        self.loader.validate(m)
        self.assertEqual(m['name'], 'foo')
        self.assertEqual(m['kind'], 'cluster')
        self.assertEqual(m['systems'][0]['morph'], 'bar')

    @contextlib.contextmanager
    def catch_warnings(*warning_classes):
        with warnings.catch_warnings(record=True) as caught_warnings:
            warnings.resetwarnings()
            for warning_class in warning_classes:
                warnings.simplefilter("always", warning_class)
            yield caught_warnings

    def test_unordered_asciibetically_after_ordered(self):
        # We only get morphologies with arbitrary keys in clusters
        m = morphlib.morphology.Morphology(yaml.load('''
        name: foo
        kind: cluster
        systems:
          morph: system-name
          repo: test:morphs
          ref: master
          deploy:
            deployment-foo:
              type: tarball
              location: /tmp/path.tar
              HOSTNAME: aasdf
        '''))
        s = self.loader.save_to_string(m)
        # root field order
        self.assertLess(s.find('name'), s.find('kind'))
        self.assertLess(s.find('kind'), s.find('systems'))
        # systems field order
        self.assertLess(s.find('morph'), s.find('repo'))
        self.assertLess(s.find('repo'), s.find('ref'))
        self.assertLess(s.find('ref'), s.find('deploy'))
        # deployment keys field order
        self.assertLess(s.find('type'), s.find('location'))
        self.assertLess(s.find('location'), s.find('HOSTNAME'))

    def test_multi_line_round_trip(self):
        s = ('name: foo\n'
             'kind: system\n'
             'description: |\n'
             '  1 2 3\n'
             '  4 5 6\n'
             '  7 8 9\n'
             'arch: x86_64\n'
             'strata:\n'
             '- name: le-chunk\n'
             '  morph: le-chunk\n'
             'configuration-extensions: []\n')
        m = self.loader.load_from_string(s, 'string')
        self.assertEqual(s, self.loader.save_to_string(m))

    def test_smoketest_multi_line_unicode(self):
        m = morphlib.morphology.Morphology(
            name=u'foo',
            description=u'1 2 3\n4 5 6\n7 8 9\n',
        )
        s = self.loader.save_to_string(m)

    def test_smoketest_multi_line_unicode_encoded(self):
        m = morphlib.morphology.Morphology(
            name=u'foo \u263A'.encode('utf-8'),
            description=u'1 \u263A\n2 \u263A\n3 \u263A\n'.encode('utf-8'),
        )
        s = self.loader.save_to_string(m)

    def test_smoketest_binary_garbage(self):
        m = morphlib.morphology.Morphology(
            description='\x92',
        )
        s = self.loader.save_to_string(m)


    def test_unknown_build_system(self):
        m = morphlib.morphology.Morphology(yaml.load('''
        kind: chunk
        name: foo
        build-system: monkeyscientist
        '''))
        with self.assertRaises(morphlib.morphloader.UnknownBuildSystemError):
            s = self.loader.set_commands(m)