diff options
| -rw-r--r-- | morphlib/builder_tests.py | 89 | ||||
| -rw-r--r-- | morphlib/remoteartifactcache.py | 12 | ||||
| -rw-r--r-- | morphlib/remoteartifactcache_tests.py | 269 | ||||
| -rw-r--r-- | morphlib/testutils.py | 56 | ||||
| -rw-r--r-- | without-test-modules | 3 |
5 files changed, 198 insertions, 231 deletions
diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py index 0cc90819..0067022f 100644 --- a/morphlib/builder_tests.py +++ b/morphlib/builder_tests.py @@ -15,12 +15,12 @@ import json -import os import StringIO import unittest import morphlib import morphlib.gitdir_tests +import morphlib.testutils class FakeBuildSystem(object): @@ -78,64 +78,6 @@ class FakeBuildEnv(object): } -class FakeFileHandle(object): - - def __init__(self, cache, key): - self._string = "" - self._cache = cache - self._key = key - - def __enter__(self): - return self - - def _writeback(self): - self._cache._cached[self._key] = self._string - - def __exit__(self, type, value, traceback): - self._writeback() - - def close(self): - self._writeback() - - def write(self, string): - self._string += string - - -class FakeArtifactCache(object): - - def __init__(self): - self._cached = {} - - def put(self, artifact): - return FakeFileHandle(self, (artifact.cache_key, artifact.name)) - - def put_artifact_metadata(self, artifact, name): - return FakeFileHandle(self, (artifact.cache_key, artifact.name, name)) - - def put_source_metadata(self, source, cachekey, name): - return FakeFileHandle(self, (cachekey, name)) - - def get(self, artifact): - return StringIO.StringIO( - self._cached[(artifact.cache_key, artifact.name)]) - - def get_artifact_metadata(self, artifact, name): - return StringIO.StringIO( - self._cached[(artifact.cache_key, artifact.name, name)]) - - def get_source_metadata(self, source, cachekey, name): - return StringIO.StringIO(self._cached[(cachekey, name)]) - - def has(self, artifact): - return (artifact.cache_key, artifact.name) in self._cached - - def has_artifact_metadata(self, artifact, name): - return (artifact.cache_key, artifact.name, name) in self._cached - - def has_source_metadata(self, source, cachekey, name): - return (cachekey, name) in self._cached - - class BuilderBaseTests(unittest.TestCase): def fake_runcmd(self, argv, *args, **kwargs): @@ -151,7 +93,7 @@ class BuilderBaseTests(unittest.TestCase): self.commands_run = [] self.app = FakeApp(self.fake_runcmd) self.staging_area = FakeStagingArea(self.fake_runcmd, FakeBuildEnv()) - self.artifact_cache = FakeArtifactCache() + self.artifact_cache = morphlib.testutils.FakeLocalArtifactCache() self.artifact = FakeArtifact('le-artifact') self.repo_cache = None self.build_env = FakeBuildEnv() @@ -187,33 +129,6 @@ class BuilderBaseTests(unittest.TestCase): self.assertEqual(sorted(events), sorted(meta['build-times'].keys())) - def test_downloads_depends(self): - lac = FakeArtifactCache() - rac = FakeArtifactCache() - afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] - for a in afacts: - fh = rac.put(a) - fh.write(a.name) - fh.close() - morphlib.builder.download_depends(afacts, lac, rac) - self.assertTrue(all(lac.has(a) for a in afacts)) - - def test_downloads_depends_metadata(self): - lac = FakeArtifactCache() - rac = FakeArtifactCache() - afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] - for a in afacts: - fh = rac.put(a) - fh.write(a.name) - fh.close() - fh = rac.put_artifact_metadata(a, 'meta') - fh.write('metadata') - fh.close() - morphlib.builder.download_depends(afacts, lac, rac, ('meta',)) - self.assertTrue(all(lac.has(a) for a in afacts)) - self.assertTrue(all(lac.has_artifact_metadata(a, 'meta') - for a in afacts)) - class ChunkBuilderTests(unittest.TestCase): diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py index 45933d10..c5bf9a48 100644 --- a/morphlib/remoteartifactcache.py +++ b/morphlib/remoteartifactcache.py @@ -22,7 +22,7 @@ import urllib2 import urlparse -class HeadRequest(urllib2.Request): # pragma: no cover +class HeadRequest(urllib2.Request): def get_method(self): return 'HEAD' @@ -51,7 +51,7 @@ class RemoteArtifactCache(object): filename = '%s.%s' % (cachekey, name) return self._has_file(filename) - def _has_file(self, filename): # pragma: no cover + def _has_file(self, filename): url = self._request_url(filename) logging.debug('RemoteArtifactCache._has_file: url=%s' % url) request = HeadRequest(url) @@ -61,7 +61,7 @@ class RemoteArtifactCache(object): except (urllib2.HTTPError, urllib2.URLError): return False - def _request_url(self, filename): # pragma: no cover + def _request_url(self, filename): server_url = self.server_url if not server_url.endswith('/'): server_url += '/' @@ -79,7 +79,7 @@ class RemoteArtifactCache(object): try: remote_file = urllib2.urlopen(remote_url) shutil.copyfileobj(remote_file, local_file) - except (urllib.HTTPError, urllib.URLError) as e: + except (urllib2.HTTPError, urllib2.URLError) as e: logging.debug(str(e)) raise GetError(self, remote_filename, e) @@ -113,7 +113,9 @@ class RemoteArtifactCache(object): to_fetch.append( (artifact.basename(), lac.put(artifact))) - if artifact.source.morphology.needs_artifact_metadata_cached: + needs_artifact_meta = \ + artifact.source.morphology.needs_artifact_metadata_cached + if needs_artifact_meta: # pragma: no cover if not lac.has_artifact_metadata(artifact, 'meta'): to_fetch.append(( artifact.metadata_basename(artifact, 'meta'), diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py index 788882c2..f6d19ae7 100644 --- a/morphlib/remoteartifactcache_tests.py +++ b/morphlib/remoteartifactcache_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,151 +14,142 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import StringIO +import BaseHTTPServer +import contextlib +import os +import socket +import SocketServer +import threading import unittest -import urllib2 +import urlparse import morphlib +import morphlib.testutils + + +def artifact_files_for_chunk_source(source): + '''Determine what filenames would exist for all artifacts of one source.''' + + existing_files = set(['%s.meta' % source.cache_key]) + for a in source.artifacts.itervalues(): + existing_files.add(a.basename()) + existing_files.add(a.metadata_basename('meta')) + return existing_files + + + +@contextlib.contextmanager +def http_server(request_handler): + server = SocketServer.TCPServer(('', 0), request_handler) + thread = threading.Thread(target=server.serve_forever) + thread.setDaemon(True) + thread.start() + + url = 'http://%s:%s' % (socket.gethostname(), server.server_address[1]) + yield url + + server.shutdown() + + +def artifact_cache_server(valid_filenames): + '''Run a basic artifact cache server. + + It implements the 'artifacts' request only. Dummy content will be returned + for any file listed in 'valid_filenames'. Anything else will get a 404 + error. + + ''' + + class Handler(BaseHTTPServer.BaseHTTPRequestHandler): + dummy_content = 'I am an artifact.\n' + + def log_message(self, *args, **kwargs): + # This is overridden to avoid dumping logs on stdout. + pass + + def artifacts(self, query, send_body=True): + '''Return a cached artifact, or 404.''' + filename = urlparse.parse_qs(query)['filename'][0] + + if filename in valid_filenames: + self.send_response(200) + self.end_headers() + if send_body: + self.wfile.write(self.dummy_content) + else: + self.send_response(404) + + def do_GET(self, send_body=True): + parts = urlparse.urlsplit(self.path) + if parts.path == '/1.0/artifacts': + self.artifacts(parts.query, send_body=send_body) + else: + self.send_response(404) + + def do_HEAD(self): + return self.do_GET(send_body=False) + + return http_server(Handler) class RemoteArtifactCacheTests(unittest.TestCase): + def fake_status_cb(*args, **kwargs): + pass + + def test_artifacts_are_cached(self): + '''Exercise fetching of artifacts that are cached.''' + + chunk = morphlib.testutils.example_chunk() + valid_files = artifact_files_for_chunk_source(chunk) + + with artifact_cache_server(valid_files) as url: + rac = morphlib.remoteartifactcache.RemoteArtifactCache(url) + an_artifact = chunk.artifacts.values()[0] + + self.assertTrue( + rac.has(an_artifact)) + + self.assertTrue( + rac.has_artifact_metadata(an_artifact, 'meta')) + + self.assertTrue( + rac.has_source_metadata(chunk, chunk.cache_key, 'meta')) + + lac = morphlib.testutils.FakeLocalArtifactCache() + self.assertFalse(lac.has(an_artifact)) - def setUp(self): - loader = morphlib.morphloader.MorphologyLoader() - morph = loader.load_from_string( - ''' - name: chunk - kind: chunk - products: - - artifact: chunk-runtime - include: - - usr/bin - - usr/sbin - - usr/lib - - usr/libexec - - artifact: chunk-devel - include: - - usr/include - - artifact: chunk-doc - include: - - usr/share/doc - ''') - sources = morphlib.source.make_sources('repo', 'original/ref', - 'chunk.morph', 'sha1', - 'tree', morph) - self.source, = sources - self.runtime_artifact = morphlib.artifact.Artifact( - self.source, 'chunk-runtime') - self.runtime_artifact.cache_key = 'CHUNK' - self.devel_artifact = morphlib.artifact.Artifact( - self.source, 'chunk-devel') - self.devel_artifact.cache_key = 'CHUNK' - self.doc_artifact = morphlib.artifact.Artifact( - self.source, 'chunk-doc') - self.doc_artifact.cache_key = 'CHUNK' - - self.existing_files = set([ - self.runtime_artifact.basename(), - self.devel_artifact.basename(), - self.runtime_artifact.metadata_basename('meta'), - '%s.%s' % (self.runtime_artifact.cache_key, 'meta'), - ]) - - self.server_url = 'http://foo.bar:8080' - self.cache = morphlib.remoteartifactcache.RemoteArtifactCache( - self.server_url) - self.cache._has_file = self._has_file - self.cache._get_file = self._get_file - - def _has_file(self, filename): - return filename in self.existing_files - - def _get_file(self, filename): - if filename in self.existing_files: - return StringIO.StringIO('%s' % filename) - else: - raise urllib2.URLError('foo') - - def test_sets_server_url(self): - self.assertEqual(self.cache.server_url, self.server_url) - - def test_has_existing_artifact(self): - self.assertTrue(self.cache.has(self.runtime_artifact)) - - def test_has_a_different_existing_artifact(self): - self.assertTrue(self.cache.has(self.devel_artifact)) - - def test_does_not_have_a_non_existent_artifact(self): - self.assertFalse(self.cache.has(self.doc_artifact)) - - def test_has_existing_artifact_metadata(self): - self.assertTrue(self.cache.has_artifact_metadata( - self.runtime_artifact, 'meta')) - - def test_does_not_have_non_existent_artifact_metadata(self): - self.assertFalse(self.cache.has_artifact_metadata( - self.runtime_artifact, 'non-existent-meta')) - - def test_has_existing_source_metadata(self): - self.assertTrue(self.cache.has_source_metadata( - self.runtime_artifact.source, - self.runtime_artifact.cache_key, - 'meta')) - - def test_does_not_have_non_existent_source_metadata(self): - self.assertFalse(self.cache.has_source_metadata( - self.runtime_artifact.source, - self.runtime_artifact.cache_key, - 'non-existent-meta')) - - def test_get_existing_artifact(self): - handle = self.cache.get(self.runtime_artifact) - data = handle.read() - self.assertEqual(data, self.runtime_artifact.basename()) - - def test_get_a_different_existing_artifact(self): - handle = self.cache.get(self.devel_artifact) - data = handle.read() - self.assertEqual(data, self.devel_artifact.basename()) - - def test_fails_to_get_a_non_existent_artifact(self): - self.assertRaises(morphlib.remoteartifactcache.GetError, - self.cache.get, self.doc_artifact, - log=lambda *args: None) - - def test_get_existing_artifact_metadata(self): - handle = self.cache.get_artifact_metadata( - self.runtime_artifact, 'meta') - data = handle.read() - self.assertEqual( - data, '%s.%s' % (self.runtime_artifact.basename(), 'meta')) - - def test_fails_to_get_non_existent_artifact_metadata(self): - self.assertRaises( - morphlib.remoteartifactcache.GetArtifactMetadataError, - self.cache.get_artifact_metadata, - self.runtime_artifact, - 'non-existent-meta', - log=lambda *args: None) - - def test_get_existing_source_metadata(self): - handle = self.cache.get_source_metadata( - self.runtime_artifact.source, - self.runtime_artifact.cache_key, - 'meta') - data = handle.read() - self.assertEqual( - data, '%s.%s' % (self.runtime_artifact.cache_key, 'meta')) - - def test_fails_to_get_non_existent_source_metadata(self): - self.assertRaises( - morphlib.remoteartifactcache.GetSourceMetadataError, - self.cache.get_source_metadata, - self.runtime_artifact.source, - self.runtime_artifact.cache_key, - 'non-existent-meta') + rac.get_artifacts( + chunk.artifacts.values(), lac, self.fake_status_cb) + self.assertTrue(lac.has(an_artifact)) + + def test_artifacts_are_not_cached(self): + '''Exercise trying to fetch artifacts that the cache doesn't have.''' + + chunk = morphlib.testutils.example_chunk() + valid_files = [] + + with artifact_cache_server(valid_files) as url: + rac = morphlib.remoteartifactcache.RemoteArtifactCache(url) + an_artifact = chunk.artifacts.values()[0] + + self.assertFalse( + rac.has(an_artifact)) + + self.assertFalse( + rac.has_artifact_metadata(an_artifact, 'non-existent-meta')) + + self.assertFalse( + rac.has_source_metadata(chunk, chunk.cache_key, 'meta')) + + lac = morphlib.testutils.FakeLocalArtifactCache() + self.assertRaises( + morphlib.remoteartifactcache.GetError, rac.get_artifacts, + chunk.artifacts.values(), lac, self.fake_status_cb) def test_escapes_pluses_in_request_urls(self): - returned_url = self.cache._request_url('gtk+') - correct_url = '%s/1.0/artifacts?filename=gtk%%2B' % self.server_url + rac = morphlib.remoteartifactcache.RemoteArtifactCache( + 'http://example.com') + + returned_url = rac._request_url('gtk+') + correct_url = 'http://example.com/1.0/artifacts?filename=gtk%2B' self.assertEqual(returned_url, correct_url) diff --git a/morphlib/testutils.py b/morphlib/testutils.py new file mode 100644 index 00000000..3678fc50 --- /dev/null +++ b/morphlib/testutils.py @@ -0,0 +1,56 @@ +# Copyright (C) 2012-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''Collected utilities for use in automated tests of Morph.''' + + +import fs + +import morphlib + + +def example_chunk(): + '''Returns a chunk source with some artifacts.''' + + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( + ''' + name: test + kind: chunk + products: + - artifact: chunk-runtime + include: + - usr/bin + - usr/sbin + - usr/lib + - usr/libexec + - artifact: chunk-devel + include: + - usr/include + - artifact: chunk-doc + include: + - usr/share/doc + ''') + + source = next(morphlib.source.make_sources( + 'repo', 'original/ref', 'chunk.morph', 'sha1', 'tree', morph)) + source.cache_key = 'CHUNK' + return source + + +class FakeLocalArtifactCache(morphlib.localartifactcache.LocalArtifactCache): + def __init__(self): + super(FakeLocalArtifactCache, self).__init__(fs.tempfs.TempFS()) diff --git a/without-test-modules b/without-test-modules index 530deb4f..66f25371 100644 --- a/without-test-modules +++ b/without-test-modules @@ -56,3 +56,6 @@ morphlib/buildbranch.py # Requires rather a lot of fake data in order to be unit tested; better to # leave it to the functional tests. morphlib/sourceresolver.py + +# These are part of the test suite +morphlib/testutils.py |