From ce2de6bd60db874f26502350a75de10faa1bf220 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 1 Oct 2014 15:12:06 +0000 Subject: Fix and integrate distbuild unit tests --- distbuild/serialise.py | 139 ++++++++++++++++--------------------------- distbuild/serialise_tests.py | 101 +++++++++++++++++++------------ setup.py | 2 +- without-test-modules | 19 ++++++ 4 files changed, 133 insertions(+), 128 deletions(-) diff --git a/distbuild/serialise.py b/distbuild/serialise.py index d410b6cf..0a60b0c2 100644 --- a/distbuild/serialise.py +++ b/distbuild/serialise.py @@ -22,11 +22,6 @@ import morphlib import logging -morphology_attributes = [ - 'needs_artifact_metadata_cached', -] - - def serialise_artifact(artifact): '''Serialise an Artifact object and its dependencies into string form.''' @@ -34,24 +29,27 @@ def serialise_artifact(artifact): result = {} for key in morphology.keys(): result[key] = morphology[key] - for x in morphology_attributes: - result['__%s' % x] = getattr(morphology, x) return result def encode_source(source): source_dic = { + 'name': source.name, 'repo': None, 'repo_name': source.repo_name, 'original_ref': source.original_ref, 'sha1': source.sha1, 'tree': source.tree, - 'morphology': encode_morphology(source.morphology), + 'morphology': str(id(source.morphology)), 'filename': source.filename, # dict keys are converted to strings by json # so we encode the artifact ids as strings 'artifact_ids': [str(id(artifact)) for (_, artifact) in source.artifacts.iteritems()], + 'cache_id': source.cache_id, + 'cache_key': source.cache_key, + 'dependencies': [str(id(d)) + for d in source.dependencies], } if source.morphology['kind'] == 'chunk': @@ -59,68 +57,42 @@ def serialise_artifact(artifact): source_dic['prefix'] = source.prefix return source_dic - def encode_artifact(a, artifacts, source_id): - if artifact.source.morphology['kind'] == 'system': + def encode_artifact(a): + if artifact.source.morphology['kind'] == 'system': # pragma: no cover arch = artifact.source.morphology['arch'] else: arch = artifact.arch return { - 'source_id': source_id, + 'source_id': id(a.source), 'name': a.name, - 'cache_id': a.cache_id, - 'cache_key': a.cache_key, - 'dependencies': [str(id(artifacts[id(d)])) - for d in a.dependencies], 'arch': arch } - visited = set() - def traverse(a): - visited.add(a) - for dep in a.dependencies: - if dep in visited: - continue - for ret in traverse(dep): - yield ret - yield a - - - artifacts = {} encoded_artifacts = {} encoded_sources = {} + encoded_morphologies = {} - for a in traverse(artifact): + for a in artifact.walk(): if id(a.source) not in encoded_sources: - if a.source.morphology['kind'] == 'chunk': - for (_, sa) in a.source.artifacts.iteritems(): - if id(sa) not in artifacts: - artifacts[id(sa)] = sa - encoded_artifacts[id(sa)] = encode_artifact(sa, - artifacts, id(a.source)) - else: - # We create separate sources for strata and systems, - # this is a bit of a hack, but needed to allow - # us to build strata and systems independently - - s = a.source - t = morphlib.source.Source(s.repo_name, s.original_ref, - s.sha1, s.tree, s.morphology, s.filename) - - t.artifacts = {a.name: a} - a.source = t - + for (_, sa) in a.source.artifacts.iteritems(): + if id(sa) not in encoded_artifacts: + encoded_artifacts[id(sa)] = encode_artifact(sa) + encoded_morphologies[id(a.source.morphology)] = encode_morphology(a.source.morphology) encoded_sources[id(a.source)] = encode_source(a.source) - if id(a) not in artifacts: - artifacts[id(a)] = a - encoded_artifacts[id(a)] = encode_artifact(a, artifacts, - id(a.source)) - - encoded_artifacts['_root'] = str(id(artifact)) + if id(a) not in encoded_artifacts: # pragma: no cover + encoded_artifacts[id(a)] = encode_artifact(a) return json.dumps({'sources': encoded_sources, - 'artifacts': encoded_artifacts}) + 'artifacts': encoded_artifacts, + 'morphologies': encoded_morphologies, + 'root_artifact': str(id(artifact)), + 'default_split_rules': { + 'chunk': morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES, + 'stratum': morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES, + }, + }) def deserialise_artifact(encoded): @@ -141,38 +113,25 @@ def deserialise_artifact(encoded): ''' - class FakeMorphology(dict): - - def get_commands(self, which): - '''Get commands to run from a morphology or build system''' - if self[which] is None: - attr = '_'.join(which.split('-')) - bs = morphlib.buildsystem.lookup_build_system( - self['build-system']) - return getattr(bs, attr) - else: - return self[which] - - morphology = FakeMorphology(le_dict) - for x in morphology_attributes: - setattr(morphology, x, le_dict['__%s' % x]) - del morphology['__%s' % x] - return morphology - - def decode_source(le_dict): + return morphlib.morphology.Morphology(le_dict) + + def decode_source(le_dict, morphology, split_rules): '''Convert a dict into a Source object.''' - morphology = decode_morphology(le_dict['morphology']) - source = morphlib.source.Source(le_dict['repo_name'], + source = morphlib.source.Source(le_dict['name'], + le_dict['repo_name'], le_dict['original_ref'], le_dict['sha1'], le_dict['tree'], morphology, - le_dict['filename']) + le_dict['filename'], + split_rules) if morphology['kind'] == 'chunk': source.build_mode = le_dict['build_mode'] source.prefix = le_dict['prefix'] + source.cache_id = le_dict['cache_id'] + source.cache_key = le_dict['cache_key'] return source def decode_artifact(artifact_dict, source): @@ -183,8 +142,6 @@ def deserialise_artifact(encoded): ''' artifact = morphlib.artifact.Artifact(source, artifact_dict['name']) - artifact.cache_id = artifact_dict['cache_id'] - artifact.cache_key = artifact_dict['cache_key'] artifact.arch = artifact_dict['arch'] artifact.source = source @@ -193,18 +150,22 @@ def deserialise_artifact(encoded): le_dicts = json.loads(encoded) artifacts_dict = le_dicts['artifacts'] sources_dict = le_dicts['sources'] + morphologies_dict = le_dicts['morphologies'] + root_artifact = le_dicts['root_artifact'] - artifact_ids = ([artifacts_dict['_root']] + - filter(lambda k: k != '_root', artifacts_dict.keys())) - - source_ids = [sid for sid in sources_dict.keys()] + artifact_ids = ([root_artifact] + artifacts_dict.keys()) artifacts = {} sources = {} + morphologies = {id: decode_morphology(d) + for (id, d) in morphologies_dict.iteritems()} - for source_id in source_ids: - source_dict = sources_dict[source_id] - sources[source_id] = decode_source(source_dict) + for source_id, source_dict in sources_dict.iteritems(): + morphology = morphologies[source_dict['morphology']] + kind = morphology['kind'] + ruler = getattr(morphlib.artifactsplitrule, 'unify_%s_matches' % kind) + rules = ruler(morphology, le_dicts['default_split_rules'][kind]) + sources[source_id] = decode_source(source_dict, morphology, rules) # clear the source artifacts that get automatically generated # we want to add the ones that were sent to us @@ -222,9 +183,9 @@ def deserialise_artifact(encoded): sources[source_id].artifacts[key] = artifacts[artifact_id] # now add the dependencies - for artifact_id in artifact_ids: - artifact = artifacts[artifact_id] - artifact.dependencies = [artifacts[aid] for aid in - artifacts_dict[artifact_id]['dependencies']] + for source_id, source_dict in sources_dict.iteritems(): + source = sources[source_id] + source.dependencies = [artifacts[aid] + for aid in source_dict['dependencies']] - return artifacts[artifacts_dict['_root']] + return artifacts[root_artifact] diff --git a/distbuild/serialise_tests.py b/distbuild/serialise_tests.py index 2ad3a384..70973346 100644 --- a/distbuild/serialise_tests.py +++ b/distbuild/serialise_tests.py @@ -23,13 +23,22 @@ import distbuild class MockMorphology(object): - def __init__(self, name): + def __init__(self, name, kind): self.dict = { 'name': '%s.morphology.name' % name, - 'kind': '%s.morphology.kind' % name, + 'kind': kind, + 'chunks': [], + 'products': [ + { + 'artifact': name, + 'include': [r'.*'], + }, + ], } - self.needs_staging_area = None - self.needs_artifact_metadata_cached = None + + @property + def needs_artifact_metadata_cached(self): + return self.dict['kind'] == 'stratum' def keys(self): return self.dict.keys() @@ -40,36 +49,56 @@ class MockMorphology(object): class MockSource(object): - def __init__(self, name): + build_mode = 'staging' + prefix = '/usr' + def __init__(self, name, kind): + self.name = name self.repo = None self.repo_name = '%s.source.repo_name' % name self.original_ref = '%s.source.original_ref' % name self.sha1 = '%s.source.sha1' % name self.tree = '%s.source.tree' % name - self.morphology = MockMorphology(name) + self.morphology = MockMorphology(name, kind) self.filename = '%s.source.filename' % name - - -class MockArtifact(object): - - def __init__(self, name): - self.source = MockSource(name) - self.name = name + self.dependencies = [] self.cache_id = { 'blip': '%s.blip' % name, 'integer': 42, } self.cache_key = '%s.cache_key' % name - self.dependencies = [] + self.artifacts = {} + + +class MockArtifact(object): + + arch = 'testarch' + + def __init__(self, name, kind): + self.source = MockSource(name, kind) + self.source.artifacts = {name: self} + self.name = name + + def walk(self): # pragma: no cover + done = set() + + def depth_first(a): + if a not in done: + done.add(a) + for dep in a.source.dependencies: + for ret in depth_first(dep): + yield ret + yield a + + return list(depth_first(self)) class SerialisationTests(unittest.TestCase): def setUp(self): - self.art1 = MockArtifact('name1') - self.art2 = MockArtifact('name2') - self.art3 = MockArtifact('name3') - self.art4 = MockArtifact('name4') + self.art1 = MockArtifact('name1', 'stratum') + self.art2 = MockArtifact('name2', 'chunk') + self.art3 = MockArtifact('name3', 'chunk') + self.art4 = MockArtifact('name4', 'chunk') def assertEqualMorphologies(self, a, b): self.assertEqual(sorted(a.keys()), sorted(b.keys())) @@ -77,11 +106,8 @@ class SerialisationTests(unittest.TestCase): a_values = [a[k] for k in keys] b_values = [b[k] for k in keys] self.assertEqual(a_values, b_values) - self.assertEqual(a.needs_staging_area, b.needs_staging_area) self.assertEqual(a.needs_artifact_metadata_cached, b.needs_artifact_metadata_cached) - self.assertEqual(a.needs_staging_area, - b.needs_staging_area) def assertEqualSources(self, a, b): self.assertEqual(a.repo, b.repo) @@ -95,30 +121,29 @@ class SerialisationTests(unittest.TestCase): def assertEqualArtifacts(self, a, b): self.assertEqualSources(a.source, b.source) self.assertEqual(a.name, b.name) - self.assertEqual(a.cache_id, b.cache_id) - self.assertEqual(a.cache_key, b.cache_key) - self.assertEqual(len(a.dependencies), len(b.dependencies)) - for i in range(len(a.dependencies)): - self.assertEqualArtifacts(a.dependencies[i], b.dependencies[i]) + self.assertEqual(a.source.cache_id, b.source.cache_id) + self.assertEqual(a.source.cache_key, b.source.cache_key) + self.assertEqual(len(a.source.dependencies), + len(b.source.dependencies)) + for i in range(len(a.source.dependencies)): + self.assertEqualArtifacts(a.source.dependencies[i], + b.source.dependencies[i]) def verify_round_trip(self, artifact): encoded = distbuild.serialise_artifact(artifact) decoded = distbuild.deserialise_artifact(encoded) self.assertEqualArtifacts(artifact, decoded) - def key(a): - return a.cache_key - objs = {} queue = [decoded] while queue: obj = queue.pop() - k = key(obj) + k = obj.source.cache_key if k in objs: self.assertTrue(obj is objs[k]) else: objs[k] = obj - queue.extend(obj.dependencies) + queue.extend(obj.source.dependencies) def test_returns_string(self): encoded = distbuild.serialise_artifact(self.art1) @@ -128,21 +153,21 @@ class SerialisationTests(unittest.TestCase): self.verify_round_trip(self.art1) def test_works_with_single_dependency(self): - self.art1.dependencies = [self.art2] + self.art1.source.dependencies = [self.art2] self.verify_round_trip(self.art1) def test_works_with_two_dependencies(self): - self.art1.dependencies = [self.art2, self.art3] + self.art1.source.dependencies = [self.art2, self.art3] self.verify_round_trip(self.art1) def test_works_with_two_levels_of_dependencies(self): - self.art2.dependencies = [self.art4] - self.art1.dependencies = [self.art2, self.art3] + self.art2.source.dependencies = [self.art4] + self.art1.source.dependencies = [self.art2, self.art3] self.verify_round_trip(self.art1) def test_works_with_dag(self): - self.art2.dependencies = [self.art4] - self.art3.dependencies = [self.art4] - self.art1.dependencies = [self.art2, self.art3] + self.art2.source.dependencies = [self.art4] + self.art3.source.dependencies = [self.art4] + self.art1.source.dependencies = [self.art2, self.art3] self.verify_round_trip(self.art1) diff --git a/setup.py b/setup.py index 99c18ed4..8125788f 100644 --- a/setup.py +++ b/setup.py @@ -128,7 +128,7 @@ class Check(Command): def run(self): subprocess.check_call(['python', '-m', 'CoverageTestRunner', '--ignore-missing-from=without-test-modules', - 'morphlib']) + 'morphlib', 'distbuild']) os.remove('.coverage') diff --git a/without-test-modules b/without-test-modules index a3aedc45..55e5291d 100644 --- a/without-test-modules +++ b/without-test-modules @@ -31,5 +31,24 @@ morphlib/plugins/print_architecture_plugin.py morphlib/plugins/add_binary_plugin.py morphlib/plugins/push_pull_plugin.py morphlib/plugins/distbuild_plugin.py +distbuild/__init__.py +distbuild/build_controller.py +distbuild/connection_machine.py +distbuild/distbuild_socket.py +distbuild/eventsrc.py +distbuild/helper_router.py +distbuild/idgen.py +distbuild/initiator.py +distbuild/initiator_connection.py +distbuild/jm.py +distbuild/json_router.py +distbuild/mainloop.py +distbuild/protocol.py +distbuild/proxy_event_source.py +distbuild/sockbuf.py +distbuild/socketsrc.py +distbuild/sockserv.py +distbuild/timer_event_source.py +distbuild/worker_build_scheduler.py # Not unit tested, since it needs a full system branch morphlib/buildbranch.py -- cgit v1.2.1