diff options
author | Daniel Silverstone <daniel.silverstone@codethink.co.uk> | 2012-09-11 15:22:14 +0100 |
---|---|---|
committer | Daniel Silverstone <daniel.silverstone@codethink.co.uk> | 2012-09-11 15:22:14 +0100 |
commit | 6c73c9b3642795b4f601ac3aee325a673e9b139e (patch) | |
tree | 3301b1bfc6edc4ac90c931db33a27728e24d1dbe | |
parent | e9e9d759805305af540c5890e626f2b91a70a2c7 (diff) | |
parent | 5870c3581ac10c14d68337fc875000ead522e99d (diff) | |
download | morph-cache-server-6c73c9b3642795b4f601ac3aee325a673e9b139e.tar.gz |
Merge branch 'danielsilverstone/updates'
-rw-r--r-- | .gitignore | 1 | ||||
-rwxr-xr-x | morph-cache-server | 103 | ||||
-rw-r--r-- | morphcacheserver/repocache.py | 47 |
3 files changed, 137 insertions, 14 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/morph-cache-server b/morph-cache-server index 3f72c18..b4f8fa1 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -20,6 +20,7 @@ import cliapp import logging import os import urllib +import shutil from bottle import Bottle, request, response, run, static_file @@ -30,12 +31,17 @@ defaults = { 'repo-dir': '/var/cache/morph-cache-server/gits', 'bundle-dir': '/var/cache/morph-cache-server/bundles', 'artifact-dir': '/var/cache/morph-cache-server/artifacts', + 'port': 8080, } class MorphCacheServer(cliapp.Application): def add_settings(self): + self.settings.integer(['port'], + 'port to listen on', + metavar='PORTNUM', + default=defaults['port']) self.settings.string(['repo-dir'], 'path to the repository cache directory', metavar='PATH', @@ -48,13 +54,101 @@ class MorphCacheServer(cliapp.Application): 'path to the artifact cache directory', metavar='PATH', default=defaults['artifact-dir']) + self.settings.boolean(['direct-mode'], + 'cache directories are directly managed') + self.settings.boolean(['enable-writes'], + 'enable the write methods (fetch and delete)') def process_args(self, args): app = Bottle() repo_cache = RepoCache(self, self.settings['repo-dir'], - self.settings['bundle-dir']) + self.settings['bundle-dir'], + self.settings['direct-mode']) + + def writable(prefix): + """Selectively enable bottle prefixes. + + prefix -- The path prefix we are enabling + + If the runtime configuration setting --enable-writes is provided + then we return the app.get() decorator for the given path prefix + otherwise we return a lambda which passes the function through + undecorated. + + This has the effect of being a runtime-enablable @app.get(...) + + """ + if self.settings['enable-writes']: + return app.get(prefix) + return lambda fn: fn + + @writable('/list') + def list(): + response.set_header('Cache-Control', 'no-cache') + results = {} + files = {} + results["files"] = files + for artifactdir, __, filenames in \ + os.walk(self.settings['artifact-dir']): + fsstinfo = os.statvfs(artifactdir) + results["freespace"] = fsstinfo.f_bsize * fsstinfo.f_bavail + for fname in filenames: + if not fname.startswith(".dl."): + try: + stinfo = os.stat("%s/%s" % (artifactdir, fname)) + files[fname] = { + "atime": stinfo.st_atime, + "size": stinfo.st_size, + "used": stinfo.st_blocks * 512, + } + except Exception, e: + print(e) + return results + + @writable('/fetch') + def fetch(): + host = self._unescape_parameter(request.query.host) + artifact = self._unescape_parameter(request.query.artifact) + try: + response.set_header('Cache-Control', 'no-cache') + in_fh = urllib.urlopen("http://%s/artifacts?basename=%s" % + (host, urllib.quote(artifact))) + tmpname = "%s/.dl.%s" % ( + self.settings['artifact-dir'], + artifact) + localtmp = open(tmpname, "w") + shutil.copyfileobj(in_fh, localtmp) + localtmp.close() + in_fh.close() + artifilename = "%s/%s" % (self.settings['artifact-dir'], + artifact) + os.rename(tmpname, artifilename) + stinfo = os.stat(artifilename) + ret = {} + ret[artifact] = { + "size": stinfo.st_size, + "used": stinfo.st_blocks * 512 + } + return ret + + except Exception, e: + response.status = 500 + logging.debug('%s' % e) + + @writable('/delete') + def delete(): + artifact = self._unescape_parameter(request.query.artifact) + try: + os.unlink('%s/%s' % (self.settings['artifact-dir'], + artifact)) + return { "status": 0, "reason": "success" } + except OSError, ose: + return { "status": ose.errno, "reason": ose.strerror } + except Exception, e: + response.status = 500 + logging.debug('%s' % e) @app.get('/sha1s') def sha1(): @@ -62,11 +156,12 @@ class MorphCacheServer(cliapp.Application): ref = self._unescape_parameter(request.query.ref) try: response.set_header('Cache-Control', 'no-cache') - sha1 = repo_cache.resolve_ref(repo, ref) + sha1, tree = repo_cache.resolve_ref(repo, ref) return { 'repo': '%s' % repo, 'ref': '%s' % ref, - 'sha1': '%s' % sha1 + 'sha1': '%s' % sha1, + 'tree': '%s' % tree } except Exception, e: response.status = 404 @@ -124,7 +219,7 @@ class MorphCacheServer(cliapp.Application): root = Bottle() root.mount(app, '/1.0') - run(root, host='0.0.0.0', port=8080, reloader=True) + run(root, host='0.0.0.0', port=self.settings['port'], reloader=True) def _unescape_parameter(self, param): return urllib.unquote(param) diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index 7061508..b55692f 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -17,6 +17,7 @@ import cliapp import os import string +import urlparse class RepositoryNotFoundError(cliapp.AppException): @@ -44,32 +45,51 @@ class UnresolvedNamedReferenceError(cliapp.AppException): class RepoCache(object): - def __init__(self, app, repo_cache_dir, bundle_cache_dir): + def __init__(self, app, repo_cache_dir, bundle_cache_dir, direct_mode): self.app = app self.repo_cache_dir = repo_cache_dir self.bundle_cache_dir = bundle_cache_dir + self.direct_mode = direct_mode def resolve_ref(self, repo_url, ref): quoted_url = self._quote_url(repo_url) repo_dir = os.path.join(self.repo_cache_dir, quoted_url) if not os.path.exists(repo_dir): - raise RepositoryNotFoundError(repo_url) + repo_dir = "%s.git" % repo_dir + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) try: refs = self._show_ref(repo_dir, ref).split('\n') - refs = [x.split() for x in refs if 'origin' in x] - return refs[0][0] + if self.direct_mode: + refs = [x.split() for x in refs] + else: + refs = [x.split() for x in refs if 'origin' in x] + return refs[0][0], self._tree_from_commit(repo_dir, refs[0][0]) + except cliapp.AppException: pass + if not self._is_valid_sha1(ref): raise InvalidReferenceError(repo_url, ref) try: - return self._rev_list(ref).strip() + sha = self._rev_list(ref).strip() + return sha, self._tree_from_commit(repo_dir, sha) except: raise InvalidReferenceError(repo_url, ref) + def _tree_from_commit(self, repo_dir, commitsha): + commit_info = self.app.runcmd(['git', 'log', '-1', + '--format=format:%T', commitsha], + cwd=repo_dir) + return commit_info.strip() + def cat_file(self, repo_url, ref, filename): quoted_url = self._quote_url(repo_url) repo_dir = os.path.join(self.repo_cache_dir, quoted_url) + if not os.path.exists(repo_dir): + repo_dir = "%s.git" % repo_dir + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) if not self._is_valid_sha1(ref): raise UnresolvedNamedReferenceError(repo_url, ref) if not os.path.exists(repo_dir): @@ -84,6 +104,10 @@ class RepoCache(object): def ls_tree(self, repo_url, ref, path): quoted_url = self._quote_url(repo_url) repo_dir = os.path.join(self.repo_cache_dir, quoted_url) + if not os.path.exists(repo_dir): + repo_dir = "%s.git" % repo_dir + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) if not self._is_valid_sha1(ref): raise UnresolvedNamedReferenceError(repo_url, ref) if not os.path.exists(repo_dir): @@ -108,13 +132,16 @@ class RepoCache(object): return data def get_bundle_filename(self, repo_url): - quoted_url = self._quote_url(repo_url) + quoted_url = self._quote_url(repo_url, True) return os.path.join(self.bundle_cache_dir, '%s.bndl' % quoted_url) - def _quote_url(self, url): - valid_chars = string.digits + string.letters + '%_' - transl = lambda x: x if x in valid_chars else '_' - return ''.join([transl(x) for x in url]) + def _quote_url(self, url, always_indirect=False): + if self.direct_mode and not always_indirect: + return urlparse.urlparse(url)[2] + else: + valid_chars = string.digits + string.letters + '%_' + transl = lambda x: x if x in valid_chars else '_' + return ''.join([transl(x) for x in url]) def _show_ref(self, repo_dir, ref): return self.app.runcmd(['git', 'show-ref', ref], cwd=repo_dir) |