From 9ba282863b41bea8d9fe990a9aba92677e2f4501 Mon Sep 17 00:00:00 2001
From: Jannis Pohlmann <jannis.pohlmann@codethink.co.uk>
Date: Tue, 17 Apr 2012 15:29:51 +0100
Subject: Initial import implementing /files and /sha1s.

---
 README                        | 21 ++++++++++
 morph-cache-server            | 83 ++++++++++++++++++++++++++++++++++++++
 morphcacheserver/__init__.py  | 17 ++++++++
 morphcacheserver/repocache.py | 92 +++++++++++++++++++++++++++++++++++++++++++
 setup.py                      | 46 ++++++++++++++++++++++
 5 files changed, 259 insertions(+)
 create mode 100644 README
 create mode 100755 morph-cache-server
 create mode 100644 morphcacheserver/__init__.py
 create mode 100644 morphcacheserver/repocache.py
 create mode 100644 setup.py

diff --git a/README b/README
new file mode 100644
index 00000000..a57e9a06
--- /dev/null
+++ b/README
@@ -0,0 +1,21 @@
+README for morph-cache-server
+=============================
+
+Legalese
+--------
+
+Copyright (C) 2012  Codethink Limited
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
diff --git a/morph-cache-server b/morph-cache-server
new file mode 100755
index 00000000..777e6276
--- /dev/null
+++ b/morph-cache-server
@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2012  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+import logging
+import urllib
+
+from bottle import Bottle, request, response, run
+from morphcacheserver.repocache import RepoCache
+
+
+defaults = {
+    'repo-dir': '/var/cache/morph-cache-server/gits',
+}
+
+
+class MorphCacheServer(cliapp.Application):
+
+    def add_settings(self):
+        self.settings.string(['repo-dir'],
+                             'path to the repository cache directory',
+                             metavar='PATH',
+                             default=defaults['repo-dir'])
+
+    def process_args(self, args):
+        app = Bottle()
+
+        repo_cache = RepoCache(self, self.settings['repo-dir'])
+        
+        @app.get('/sha1s')
+        def sha1():
+            repo = self._unescape_parameter(request.query.repo)
+            ref = self._unescape_parameter(request.query.ref)
+            try:
+                sha1 = repo_cache.resolve_ref(repo, ref)
+                return {
+                    'repo': '%s' % repo,
+                    'ref': '%s' % ref,
+                    'sha1': '%s' % sha1
+                }
+            except Exception, e:
+                response.status = 404
+                logging.debug('%s' % e)
+        
+        @app.get('/files')
+        def file():
+            repo = self._unescape_parameter(request.query.repo)
+            ref = self._unescape_parameter(request.query.ref)
+            filename = self._unescape_parameter(request.query.filename)
+            try:
+                content = repo_cache.cat_file(repo, ref, filename)
+                response.set_header('Content-Type', 'application/octet-stream')
+                return content
+            except Exception, e:
+                response.status = 404
+                logging.debug('%s' % e)
+        
+        root = Bottle()
+        root.mount(app, '/1.0')
+        
+        run(root, host='0.0.0.0', port=8080, reloader=True)
+
+    def _unescape_parameter(self, param):
+        return urllib.unquote(param)
+
+
+if __name__ == '__main__':
+    MorphCacheServer().run()
diff --git a/morphcacheserver/__init__.py b/morphcacheserver/__init__.py
new file mode 100644
index 00000000..9ad5a305
--- /dev/null
+++ b/morphcacheserver/__init__.py
@@ -0,0 +1,17 @@
+# Copyright (C) 2012  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import repocache
diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
new file mode 100644
index 00000000..49b82001
--- /dev/null
+++ b/morphcacheserver/repocache.py
@@ -0,0 +1,92 @@
+# Copyright (C) 2012  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+import os
+import string
+
+
+class InvalidReferenceError(cliapp.AppException):
+    
+    def __init__(self, repo, ref):
+        cliapp.AppException.__init__(
+                self, 'Ref %s is an invalid reference for repo %s' %
+                (ref, repo))
+
+
+class UnresolvedNamedReferenceError(cliapp.AppException):
+
+    def __init__(self, repo, ref):
+        cliapp.AppException.__init__(
+                self, 'Ref %s is not a SHA1 ref for repo %s' %
+                (ref, repo))
+
+
+class RepoCache(object):
+    
+    def __init__(self, app, dirname):
+        self.app = app
+        self.dirname = dirname
+
+    def resolve_ref(self, repo_url, ref):
+        quoted_url = self._quote_url(repo_url)
+        repo_dir = os.path.join(self.dirname, quoted_url)
+        try:
+            refs = self._show_ref(repo_dir, ref).split('\n')
+            refs = [x.split() for x in refs if 'origin' in x]
+            return refs[0][0]
+        except cliapp.AppException:
+            pass
+        if not self._is_valid_sha1(ref):
+            raise InvalidReferenceError(repo_url, ref)
+        try:
+            return self._rev_list(ref).strip()
+        except:
+            raise InvalidReferenceError(repo_url, ref)
+
+    def cat_file(self, repo_url, ref, filename):
+        quoted_url = self._quote_url(repo_url)
+        repo_dir = os.path.join(self.dirname, quoted_url)
+
+        if not self._is_valid_sha1(ref):
+            raise UnresolvedNamedReferenceError(repo_url, ref)
+        try:
+            sha1 = self._rev_list(repo_dir, ref).strip()
+        except:
+            raise InvalidReferenceError(repo_url, ref)
+
+        return self._cat_file(repo_dir, sha1, filename)
+        
+    def _quote_url(self, url):
+        valid_chars = string.digits + string.letters + '%_'
+        transl = lambda x: x if x in valid_chars else '_'
+        return ''.join([transl(x) for x in url])
+
+    def _show_ref(self, repo_dir, ref):
+        return self.app.runcmd(['git', 'show-ref', ref], cwd=repo_dir)
+
+    def _rev_list(self, repo_dir, ref):
+        return self.app.runcmd(
+                ['git', 'rev-list', '--no-walk', ref], cwd=repo_dir)
+
+    def _cat_file(self, repo_dir, sha1, filename):
+        return self.app.runcmd(
+                ['git', 'cat-file', 'blob', '%s:%s' % (sha1, filename)],
+                cwd=repo_dir)
+
+    def _is_valid_sha1(self, ref):
+        valid_chars = 'abcdefABCDEF0123456789'
+        return len(ref) == 40 and all([x in valid_chars for x in ref])
diff --git a/setup.py b/setup.py
new file mode 100644
index 00000000..e861f392
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,46 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2012  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+from distutils.core import setup
+
+
+setup(name='morph-cache-server',
+      description='FIXME',
+      long_description='''\
+FIXME
+''',
+      classifiers=[
+        'Development Status :: 2 - Pre-Alpha',
+        'Environment :: Console',
+        'Environment :: Web Environment',
+        'Intended Audience :: Developers',
+        'Intended Audience :: System Administrators',
+        'License :: OSI Approved :: GNU General Public License (GPL)',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python',
+        'Topic :: Software Development :: Build Tools',
+        'Topic :: Software Development :: Embedded Systems',
+        'Topic :: System :: Archiving :: Packaging',
+        'Topic :: System :: Software Distribution',
+      ],
+      author='Jannis Pohlmann',
+      author_email='jannis.pohlmann@codethink.co.uk',
+      url='http://www.baserock.org/',
+      scripts=['morph-cache-server'],
+      packages=['morphcacheserver'],
+     )
-- 
cgit v1.2.1


From 63927c35611bf56a1fce03750e953ec5250fb282 Mon Sep 17 00:00:00 2001
From: Jannis Pohlmann <jannis.pohlmann@codethink.co.uk>
Date: Wed, 18 Apr 2012 13:55:32 +0100
Subject: Raise a RepositoryNotFoundError if a repo does not exist in the
 cache.

---
 morph-cache-server            |  2 +-
 morphcacheserver/repocache.py | 12 +++++++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/morph-cache-server b/morph-cache-server
index 777e6276..b13241ba 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -41,7 +41,7 @@ class MorphCacheServer(cliapp.Application):
         app = Bottle()
 
         repo_cache = RepoCache(self, self.settings['repo-dir'])
-        
+
         @app.get('/sha1s')
         def sha1():
             repo = self._unescape_parameter(request.query.repo)
diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index 49b82001..1b6862cb 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -19,6 +19,13 @@ import os
 import string
 
 
+class RepositoryNotFoundError(cliapp.AppException):
+
+    def __init__(self, repo):
+        cliapp.AppException.__init__(
+                self, 'Repository %s does not exist in the cache' % repo)
+
+
 class InvalidReferenceError(cliapp.AppException):
     
     def __init__(self, repo, ref):
@@ -44,6 +51,8 @@ class RepoCache(object):
     def resolve_ref(self, repo_url, ref):
         quoted_url = self._quote_url(repo_url)
         repo_dir = os.path.join(self.dirname, quoted_url)
+        if not os.path.exists(repo_dir):
+            raise RepositoryNotFoundError(repo_url)
         try:
             refs = self._show_ref(repo_dir, ref).split('\n')
             refs = [x.split() for x in refs if 'origin' in x]
@@ -60,9 +69,10 @@ class RepoCache(object):
     def cat_file(self, repo_url, ref, filename):
         quoted_url = self._quote_url(repo_url)
         repo_dir = os.path.join(self.dirname, quoted_url)
-
         if not self._is_valid_sha1(ref):
             raise UnresolvedNamedReferenceError(repo_url, ref)
+        if not os.path.exists(repo_dir):
+            raise RepositoryNotFoundError(repo_url)
         try:
             sha1 = self._rev_list(repo_dir, ref).strip()
         except:
-- 
cgit v1.2.1


From 8f3b7be2e1130a94cf4de5e9e1b5a18d382efff8 Mon Sep 17 00:00:00 2001
From: Jannis Pohlmann <jannis.pohlmann@codethink.co.uk>
Date: Wed, 18 Apr 2012 13:57:36 +0100
Subject: Use "Cache-Control: no-cache" to avoid caching of /sha1s results.

Resolving a ref may result in a different SHA1 between any two requests,
so we simply should never allow the results to be cached by an HTTP
cache.
---
 morph-cache-server | 1 +
 1 file changed, 1 insertion(+)

diff --git a/morph-cache-server b/morph-cache-server
index b13241ba..60c31053 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -47,6 +47,7 @@ class MorphCacheServer(cliapp.Application):
             repo = self._unescape_parameter(request.query.repo)
             ref = self._unescape_parameter(request.query.ref)
             try:
+                response.set_header('Cache-Control', 'no-cache')
                 sha1 = repo_cache.resolve_ref(repo, ref)
                 return {
                     'repo': '%s' % repo,
-- 
cgit v1.2.1


From 6b762c363224860833ea20c9ba8109c0c210d419 Mon Sep 17 00:00:00 2001
From: Jannis Pohlmann <jannis.pohlmann@codethink.co.uk>
Date: Wed, 18 Apr 2012 17:14:11 +0100
Subject: Add untested support for bundles.

---
 morph-cache-server            | 18 ++++++++++++++++--
 morphcacheserver/repocache.py | 13 +++++++++----
 2 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/morph-cache-server b/morph-cache-server
index 60c31053..5554481c 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -20,12 +20,14 @@ import cliapp
 import logging
 import urllib
 
-from bottle import Bottle, request, response, run
+from bottle import Bottle, request, response, run, static_file
+
 from morphcacheserver.repocache import RepoCache
 
 
 defaults = {
     'repo-dir': '/var/cache/morph-cache-server/gits',
+    'bundle-dir': '/var/cache/morph-cache-server/bundles',
 }
 
 
@@ -36,11 +38,17 @@ class MorphCacheServer(cliapp.Application):
                              'path to the repository cache directory',
                              metavar='PATH',
                              default=defaults['repo-dir'])
+        self.settings.string(['bundle-dir'],
+                             'path to the bundle cache directory',
+                             metavar='PATH',
+                             default=defaults['bundle-dir'])
 
     def process_args(self, args):
         app = Bottle()
 
-        repo_cache = RepoCache(self, self.settings['repo-dir'])
+        repo_cache = RepoCache(self,
+                               self.settings['repo-dir'],
+                               self.settings['bundles'])
 
         @app.get('/sha1s')
         def sha1():
@@ -70,6 +78,12 @@ class MorphCacheServer(cliapp.Application):
             except Exception, e:
                 response.status = 404
                 logging.debug('%s' % e)
+
+        @app.get('/bundles')
+        def bundle():
+            repo = self._unescape_parameter(request.query.repo)
+            filename = repo_cache.get_bundle_filename(repo)
+            return static_file(filename, download=True)
         
         root = Bottle()
         root.mount(app, '/1.0')
diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index 1b6862cb..e6ab6401 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -44,13 +44,14 @@ class UnresolvedNamedReferenceError(cliapp.AppException):
 
 class RepoCache(object):
     
-    def __init__(self, app, dirname):
+    def __init__(self, app, repo_cache_dir, bundle_cache_dir):
         self.app = app
-        self.dirname = dirname
+        self.repo_cache_dir = repo_cache_dir
+        self.bundle_cache_dir = bundle_cache_dir
 
     def resolve_ref(self, repo_url, ref):
         quoted_url = self._quote_url(repo_url)
-        repo_dir = os.path.join(self.dirname, quoted_url)
+        repo_dir = os.path.join(self.repo_cache_dir, quoted_url)
         if not os.path.exists(repo_dir):
             raise RepositoryNotFoundError(repo_url)
         try:
@@ -68,7 +69,7 @@ class RepoCache(object):
 
     def cat_file(self, repo_url, ref, filename):
         quoted_url = self._quote_url(repo_url)
-        repo_dir = os.path.join(self.dirname, quoted_url)
+        repo_dir = os.path.join(self.repo_cache_dir, quoted_url)
         if not self._is_valid_sha1(ref):
             raise UnresolvedNamedReferenceError(repo_url, ref)
         if not os.path.exists(repo_dir):
@@ -79,6 +80,10 @@ class RepoCache(object):
             raise InvalidReferenceError(repo_url, ref)
 
         return self._cat_file(repo_dir, sha1, filename)
+
+    def get_bundle_filename(self, repo_url):
+        quoted_url = self._quote_url(repo_url)
+        return os.path.join(self.bundle_dir, '%s.bndl' % quoted_url)
         
     def _quote_url(self, url):
         valid_chars = string.digits + string.letters + '%_'
-- 
cgit v1.2.1


From a58f373cec3b3bd4420478c99dde6e7fa2cba31b Mon Sep 17 00:00:00 2001
From: Jannis Pohlmann <jannis.pohlmann@codethink.co.uk>
Date: Wed, 18 Apr 2012 17:36:00 +0100
Subject: Fix various small issues preventing bundles from working.

---
 morph-cache-server            | 7 +++++--
 morphcacheserver/repocache.py | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/morph-cache-server b/morph-cache-server
index 5554481c..57713e25 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -18,6 +18,7 @@
 
 import cliapp
 import logging
+import os
 import urllib
 
 from bottle import Bottle, request, response, run, static_file
@@ -48,7 +49,7 @@ class MorphCacheServer(cliapp.Application):
 
         repo_cache = RepoCache(self,
                                self.settings['repo-dir'],
-                               self.settings['bundles'])
+                               self.settings['bundle-dir'])
 
         @app.get('/sha1s')
         def sha1():
@@ -83,7 +84,9 @@ class MorphCacheServer(cliapp.Application):
         def bundle():
             repo = self._unescape_parameter(request.query.repo)
             filename = repo_cache.get_bundle_filename(repo)
-            return static_file(filename, download=True)
+            dirname = os.path.dirname(filename)
+            basename = os.path.basename(filename)
+            return static_file(basename, root=dirname, download=True)
         
         root = Bottle()
         root.mount(app, '/1.0')
diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index e6ab6401..3bb348ff 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -83,7 +83,7 @@ class RepoCache(object):
 
     def get_bundle_filename(self, repo_url):
         quoted_url = self._quote_url(repo_url)
-        return os.path.join(self.bundle_dir, '%s.bndl' % quoted_url)
+        return os.path.join(self.bundle_cache_dir, '%s.bndl' % quoted_url)
         
     def _quote_url(self, url):
         valid_chars = string.digits + string.letters + '%_'
-- 
cgit v1.2.1


From d751e8da557741eb21d0856293e80d57474721b7 Mon Sep 17 00:00:00 2001
From: Jannis Pohlmann <jannis.pohlmann@codethink.co.uk>
Date: Wed, 18 Apr 2012 18:00:30 +0100
Subject: Add support for /artifacts.

---
 morph-cache-server | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/morph-cache-server b/morph-cache-server
index 57713e25..4639a33a 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -29,6 +29,7 @@ from morphcacheserver.repocache import RepoCache
 defaults = {
     'repo-dir': '/var/cache/morph-cache-server/gits',
     'bundle-dir': '/var/cache/morph-cache-server/bundles',
+    'artifact-dir': '/var/cache/morph-cache-server/artifacts',
 }
 
 
@@ -43,6 +44,10 @@ class MorphCacheServer(cliapp.Application):
                              'path to the bundle cache directory',
                              metavar='PATH',
                              default=defaults['bundle-dir'])
+        self.settings.string(['artifact-dir'],
+                             'path to the artifact cache directory',
+                             metavar='PATH',
+                             default=defaults['artifact-dir'])
 
     def process_args(self, args):
         app = Bottle()
@@ -87,6 +92,18 @@ class MorphCacheServer(cliapp.Application):
             dirname = os.path.dirname(filename)
             basename = os.path.basename(filename)
             return static_file(basename, root=dirname, download=True)
+
+        @app.get('/artifacts')
+        def artifact():
+            basename = self._unescape_parameter(request.query.filename)
+            filename = os.path.join(self.settings['artifact-dir'], basename)
+            if os.path.exists(filename):
+                return static_file(basename,
+                                   root=self.settings['artifact-dir'],
+                                   download=True)
+            else:
+                response.status = 404
+                logging.debug('artifact %s does not exist' % basename)
         
         root = Bottle()
         root.mount(app, '/1.0')
-- 
cgit v1.2.1


From f1fba299bd07510346082ef985ef08a494dca9d9 Mon Sep 17 00:00:00 2001
From: Jannis Pohlmann <jannis.pohlmann@codethink.co.uk>
Date: Wed, 18 Apr 2012 18:02:44 +0100
Subject: Use the desired artifact filename as the download filename.

---
 morph-cache-server | 1 +
 1 file changed, 1 insertion(+)

diff --git a/morph-cache-server b/morph-cache-server
index 4639a33a..7618f5b9 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -100,6 +100,7 @@ class MorphCacheServer(cliapp.Application):
             if os.path.exists(filename):
                 return static_file(basename,
                                    root=self.settings['artifact-dir'],
+                                   filename=basename,
                                    download=True)
             else:
                 response.status = 404
-- 
cgit v1.2.1


From eb4c1530c57b6ae200643b259e6ed95904951681 Mon Sep 17 00:00:00 2001
From: Jannis Pohlmann <jannis.pohlmann@codethink.co.uk>
Date: Fri, 20 Apr 2012 15:46:53 +0100
Subject: Add /trees which serves the contents of a git tree using ls-tree.

/trees queries take repo URI, a SHA1 ref and an optional path parameter.
The result is a JSON dictionary of the form

    {
       "repo": "<repo URI>",
       "ref": "<SHA1 ref>",
       "tree": {
            "filename1": {
               "mode": "100644",
               "kind": "blob",
               "sha1": "FOOBARBAZ"
            },
            ...
        }
    }
---
 morph-cache-server            | 16 ++++++++++++++++
 morphcacheserver/repocache.py | 29 +++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/morph-cache-server b/morph-cache-server
index 7618f5b9..86a1fe26 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -85,6 +85,22 @@ class MorphCacheServer(cliapp.Application):
                 response.status = 404
                 logging.debug('%s' % e)
 
+        @app.get('/trees')
+        def tree():
+            repo = self._unescape_parameter(request.query.repo)
+            ref = self._unescape_parameter(request.query.ref)
+            path = self._unescape_parameter(request.query.path)
+            try:
+                tree = repo_cache.ls_tree(repo, ref, path)
+                return {
+                    'repo': '%s' % repo,
+                    'ref': '%s' % ref,
+                    'tree': tree,
+                }
+            except Exception, e:
+                response.status = 404
+                logging.debug('%s' % e)
+
         @app.get('/bundles')
         def bundle():
             repo = self._unescape_parameter(request.query.repo)
diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index 3bb348ff..7061508d 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -81,6 +81,32 @@ class RepoCache(object):
 
         return self._cat_file(repo_dir, sha1, filename)
 
+    def ls_tree(self, repo_url, ref, path):
+        quoted_url = self._quote_url(repo_url)
+        repo_dir = os.path.join(self.repo_cache_dir, quoted_url)
+        if not self._is_valid_sha1(ref):
+            raise UnresolvedNamedReferenceError(repo_url, ref)
+        if not os.path.exists(repo_dir):
+            raise RepositoryNotFoundError(repo_url)
+
+        try:
+            sha1 = self._rev_list(repo_dir, ref).strip()
+        except:
+            raise InvalidReferenceError(repo_url, ref)
+
+        lines = self._ls_tree(repo_dir, sha1, path).strip()
+        lines = lines.splitlines()
+        data = {}
+        for line in lines:
+            elements = line.split()
+            basename = elements[3]
+            data[basename] = {
+                'mode': elements[0],
+                'kind': elements[1],
+                'sha1': elements[2],
+            }
+        return data
+
     def get_bundle_filename(self, repo_url):
         quoted_url = self._quote_url(repo_url)
         return os.path.join(self.bundle_cache_dir, '%s.bndl' % quoted_url)
@@ -102,6 +128,9 @@ class RepoCache(object):
                 ['git', 'cat-file', 'blob', '%s:%s' % (sha1, filename)],
                 cwd=repo_dir)
 
+    def _ls_tree(self, repo_dir, sha1, path):
+        return self.app.runcmd(['git', 'ls-tree', sha1, path], cwd=repo_dir)
+
     def _is_valid_sha1(self, ref):
         valid_chars = 'abcdefABCDEF0123456789'
         return len(ref) == 40 and all([x in valid_chars for x in ref])
-- 
cgit v1.2.1


From e9e9d759805305af540c5890e626f2b91a70a2c7 Mon Sep 17 00:00:00 2001
From: Jannis Pohlmann <jannis.pohlmann@codethink.co.uk>
Date: Tue, 1 May 2012 11:41:59 +0100
Subject: Revert "Use the desired artifact filename as the download filename."

This reverts commit f1fba299bd07510346082ef985ef08a494dca9d9.
---
 morph-cache-server | 1 -
 1 file changed, 1 deletion(-)

diff --git a/morph-cache-server b/morph-cache-server
index 86a1fe26..3f72c186 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -116,7 +116,6 @@ class MorphCacheServer(cliapp.Application):
             if os.path.exists(filename):
                 return static_file(basename,
                                    root=self.settings['artifact-dir'],
-                                   filename=basename,
                                    download=True)
             else:
                 response.status = 404
-- 
cgit v1.2.1


From b53860b0aa27e5c004adc45552e7fead71de09e7 Mon Sep 17 00:00:00 2001
From: Daniel Silverstone <daniel.silverstone@codethink.co.uk>
Date: Mon, 3 Sep 2012 15:30:10 +0100
Subject: Add a .gitignore to ignore *.pyc

To reduce the noise when I run 'git status' this gitignore will mean
that git won't notify me of repocache.pyc and __init__.pyc
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 .gitignore

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..0d20b648
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+*.pyc
-- 
cgit v1.2.1


From cd00de30a0f4d2d422053692948ea9986960c43f Mon Sep 17 00:00:00 2001
From: Daniel Silverstone <daniel.silverstone@codethink.co.uk>
Date: Tue, 4 Sep 2012 10:49:35 +0100
Subject: A direct-mode for git cache access

Direct-mode, when enabled, causes morph-cache-server to assume a more
Trove-like structure for the repositories, rather than the morph-cache
structure which it was originally written for.  This means that for
the workers, we can use the original code and for Trove, the direct mode.
---
 morph-cache-server            |  5 ++++-
 morphcacheserver/repocache.py | 34 ++++++++++++++++++++++++++--------
 2 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/morph-cache-server b/morph-cache-server
index 3f72c186..bb84915a 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -48,13 +48,16 @@ class MorphCacheServer(cliapp.Application):
                              'path to the artifact cache directory',
                              metavar='PATH',
                              default=defaults['artifact-dir'])
+        self.settings.boolean(['direct-mode'],
+                              'cache directories are directly managed')
 
     def process_args(self, args):
         app = Bottle()
 
         repo_cache = RepoCache(self,
                                self.settings['repo-dir'],
-                               self.settings['bundle-dir'])
+                               self.settings['bundle-dir'],
+                               self.settings['direct-mode'])
 
         @app.get('/sha1s')
         def sha1():
diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index 7061508d..c226ef40 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -17,6 +17,7 @@
 import cliapp
 import os
 import string
+import urlparse
 
 
 class RepositoryNotFoundError(cliapp.AppException):
@@ -44,19 +45,25 @@ class UnresolvedNamedReferenceError(cliapp.AppException):
 
 class RepoCache(object):
     
-    def __init__(self, app, repo_cache_dir, bundle_cache_dir):
+    def __init__(self, app, repo_cache_dir, bundle_cache_dir, direct_mode):
         self.app = app
         self.repo_cache_dir = repo_cache_dir
         self.bundle_cache_dir = bundle_cache_dir
+        self.direct_mode = direct_mode
 
     def resolve_ref(self, repo_url, ref):
         quoted_url = self._quote_url(repo_url)
         repo_dir = os.path.join(self.repo_cache_dir, quoted_url)
         if not os.path.exists(repo_dir):
-            raise RepositoryNotFoundError(repo_url)
+            repo_dir = "%s.git" % repo_dir
+            if not os.path.exists(repo_dir):
+                raise RepositoryNotFoundError(repo_url)
         try:
             refs = self._show_ref(repo_dir, ref).split('\n')
-            refs = [x.split() for x in refs if 'origin' in x]
+            if self.direct_mode:
+                refs = [x.split() for x in refs]
+            else:
+                refs = [x.split() for x in refs if 'origin' in x]
             return refs[0][0]
         except cliapp.AppException:
             pass
@@ -70,6 +77,10 @@ class RepoCache(object):
     def cat_file(self, repo_url, ref, filename):
         quoted_url = self._quote_url(repo_url)
         repo_dir = os.path.join(self.repo_cache_dir, quoted_url)
+        if not os.path.exists(repo_dir):
+            repo_dir = "%s.git" % repo_dir
+            if not os.path.exists(repo_dir):
+                raise RepositoryNotFoundError(repo_url)
         if not self._is_valid_sha1(ref):
             raise UnresolvedNamedReferenceError(repo_url, ref)
         if not os.path.exists(repo_dir):
@@ -84,6 +95,10 @@ class RepoCache(object):
     def ls_tree(self, repo_url, ref, path):
         quoted_url = self._quote_url(repo_url)
         repo_dir = os.path.join(self.repo_cache_dir, quoted_url)
+        if not os.path.exists(repo_dir):
+            repo_dir = "%s.git" % repo_dir
+            if not os.path.exists(repo_dir):
+                raise RepositoryNotFoundError(repo_url)
         if not self._is_valid_sha1(ref):
             raise UnresolvedNamedReferenceError(repo_url, ref)
         if not os.path.exists(repo_dir):
@@ -108,13 +123,16 @@ class RepoCache(object):
         return data
 
     def get_bundle_filename(self, repo_url):
-        quoted_url = self._quote_url(repo_url)
+        quoted_url = self._quote_url(repo_url, True)
         return os.path.join(self.bundle_cache_dir, '%s.bndl' % quoted_url)
         
-    def _quote_url(self, url):
-        valid_chars = string.digits + string.letters + '%_'
-        transl = lambda x: x if x in valid_chars else '_'
-        return ''.join([transl(x) for x in url])
+    def _quote_url(self, url, always_indirect=False):
+        if self.direct_mode and not always_indirect:
+            return urlparse.urlparse(url)[2]
+        else:
+            valid_chars = string.digits + string.letters + '%_'
+            transl = lambda x: x if x in valid_chars else '_'
+            return ''.join([transl(x) for x in url])
 
     def _show_ref(self, repo_dir, ref):
         return self.app.runcmd(['git', 'show-ref', ref], cwd=repo_dir)
-- 
cgit v1.2.1


From 2c04007fc74d5971b12f351a4c2076e403386997 Mon Sep 17 00:00:00 2001
From: Daniel Silverstone <daniel.silverstone@codethink.co.uk>
Date: Wed, 5 Sep 2012 17:49:21 +0100
Subject: Return tree SHA1 when looking for ref resolution.

Morph now expects the tree SHA1 in addition when resolving references
using the cache server.  This is to better facilitate correct cache
key computation since commits can be made which have no tree changes
and thus nothing to usefully affect the build.  (For example the morph
branch and build features)
---
 morph-cache-server            |  5 +++--
 morphcacheserver/repocache.py | 13 +++++++++++--
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/morph-cache-server b/morph-cache-server
index bb84915a..3a121d49 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -65,11 +65,12 @@ class MorphCacheServer(cliapp.Application):
             ref = self._unescape_parameter(request.query.ref)
             try:
                 response.set_header('Cache-Control', 'no-cache')
-                sha1 = repo_cache.resolve_ref(repo, ref)
+                sha1, tree = repo_cache.resolve_ref(repo, ref)
                 return {
                     'repo': '%s' % repo,
                     'ref': '%s' % ref,
-                    'sha1': '%s' % sha1
+                    'sha1': '%s' % sha1,
+                    'tree': '%s' % tree
                 }
             except Exception, e:
                 response.status = 404
diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index c226ef40..b55692f2 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -64,16 +64,25 @@ class RepoCache(object):
                 refs = [x.split() for x in refs]
             else:
                 refs = [x.split() for x in refs if 'origin' in x]
-            return refs[0][0]
+            return refs[0][0], self._tree_from_commit(repo_dir, refs[0][0])
+
         except cliapp.AppException:
             pass
+
         if not self._is_valid_sha1(ref):
             raise InvalidReferenceError(repo_url, ref)
         try:
-            return self._rev_list(ref).strip()
+            sha = self._rev_list(ref).strip()
+            return sha, self._tree_from_commit(repo_dir, sha)
         except:
             raise InvalidReferenceError(repo_url, ref)
 
+    def _tree_from_commit(self, repo_dir, commitsha):
+        commit_info = self.app.runcmd(['git', 'log', '-1',
+                                       '--format=format:%T', commitsha],
+                                      cwd=repo_dir)
+        return commit_info.strip()
+
     def cat_file(self, repo_url, ref, filename):
         quoted_url = self._quote_url(repo_url)
         repo_dir = os.path.join(self.repo_cache_dir, quoted_url)
-- 
cgit v1.2.1


From c2998750dbb3d79b7455a079aa3f3d243715a15f Mon Sep 17 00:00:00 2001
From: Daniel Silverstone <daniel.silverstone@codethink.co.uk>
Date: Fri, 7 Sep 2012 13:26:38 +0100
Subject: Support running on a different port from 8080

In order to allow multiple morph-cache-server instances to run on
a single system, we need to support running on different ports.
---
 morph-cache-server | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/morph-cache-server b/morph-cache-server
index 3a121d49..827da10a 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -30,12 +30,17 @@ defaults = {
     'repo-dir': '/var/cache/morph-cache-server/gits',
     'bundle-dir': '/var/cache/morph-cache-server/bundles',
     'artifact-dir': '/var/cache/morph-cache-server/artifacts',
+    'port': 8080,
 }
 
 
 class MorphCacheServer(cliapp.Application):
 
     def add_settings(self):
+        self.settings.integer(['port'],
+                              'port to listen on',
+                              metavar='PORTNUM',
+                              default=defaults['port'])
         self.settings.string(['repo-dir'],
                              'path to the repository cache directory',
                              metavar='PATH',
@@ -128,7 +133,7 @@ class MorphCacheServer(cliapp.Application):
         root = Bottle()
         root.mount(app, '/1.0')
         
-        run(root, host='0.0.0.0', port=8080, reloader=True)
+        run(root, host='0.0.0.0', port=self.settings['port'], reloader=True)
 
     def _unescape_parameter(self, param):
         return urllib.unquote(param)
-- 
cgit v1.2.1


From 9c3279221262057c7eb8ebdcb29f366dc4de66d5 Mon Sep 17 00:00:00 2001
From: Daniel Silverstone <daniel.silverstone@codethink.co.uk>
Date: Fri, 7 Sep 2012 10:14:27 +0100
Subject: Add ability to have 'writable' cache servers.

Since we need to be able to update the cache from builders, this patch
introduces a --enable-writes argument to morph-cache-server and also adds a
@writable decorator to the class ready for marking particular paths which are
only available when --enable-writes is set.
---
 morph-cache-server | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/morph-cache-server b/morph-cache-server
index 827da10a..ba5f0b2a 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -55,6 +55,8 @@ class MorphCacheServer(cliapp.Application):
                              default=defaults['artifact-dir'])
         self.settings.boolean(['direct-mode'],
                               'cache directories are directly managed')
+        self.settings.boolean(['enable-writes'],
+                              'enable the write methods (fetch and delete)')
 
     def process_args(self, args):
         app = Bottle()
@@ -64,6 +66,23 @@ class MorphCacheServer(cliapp.Application):
                                self.settings['bundle-dir'],
                                self.settings['direct-mode'])
 
+        def writable(prefix):
+            """Selectively enable bottle prefixes.
+
+            prefix -- The path prefix we are enabling
+
+            If the runtime configuration setting --enable-writes is provided
+            then we return the app.get() decorator for the given path prefix
+            otherwise we return a lambda which passes the function through
+            undecorated.
+
+            This has the effect of being a runtime-enablable @app.get(...)
+
+            """
+            if self.settings['enable-writes']:
+                return app.get(prefix)
+            return lambda fn: fn
+
         @app.get('/sha1s')
         def sha1():
             repo = self._unescape_parameter(request.query.repo)
-- 
cgit v1.2.1


From 465e830d1d6d2c51425e2418b8e802a95145b6ee Mon Sep 17 00:00:00 2001
From: Daniel Silverstone <daniel.silverstone@codethink.co.uk>
Date: Fri, 7 Sep 2012 10:14:27 +0100
Subject: Add a /list method

When --enable-writes is set, we provide a /list target which produces
a JSON dictionary of information about the state of the artifact cache.

The dictionary is of the form:

{
    "freespace": NBYTES_OF_SPACE,
    "files": {
        "artifact-filename": {
            "atime": ATIME_AS_NUMBER,
            "size": NBYTES_SIZE_OF_FILE,
            "used": NBYTES_USED_ON_DISK
        },
        ...
    }
}

This allows a controller to decide which artifacts have not been requested in
some time and also how big artifacts are, not only in terms of their 'byte'
size, but also the space they consume on disk.  System images in particular may
differ in this respect since they should be sparsely stored.
---
 morph-cache-server | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/morph-cache-server b/morph-cache-server
index ba5f0b2a..b726c1e5 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -83,6 +83,28 @@ class MorphCacheServer(cliapp.Application):
                 return app.get(prefix)
             return lambda fn: fn
 
+        @writable('/list')
+        def list():
+            response.set_header('Cache-Control', 'no-cache')
+            results = {}
+            files = {}
+            results["files"] = files
+            for artifactdir, __, filenames in \
+                    os.walk(self.settings['artifact-dir']):
+                fsstinfo = os.statvfs(artifactdir)
+                results["freespace"] = fsstinfo.f_bsize * fsstinfo.f_bavail
+                for fname in filenames:
+                    try:
+                        stinfo = os.stat("%s/%s" % (artifactdir, fname))
+                        files[fname] = {
+                            "atime": stinfo.st_atime,
+                            "size": stinfo.st_size,
+                            "used": stinfo.st_blocks * 512,
+                            }
+                    except Exception, e:
+                        print(e)
+            return results
+
         @app.get('/sha1s')
         def sha1():
             repo = self._unescape_parameter(request.query.repo)
-- 
cgit v1.2.1


From 1a0e40d854d37f81e9cdaf8bb23e480790614d2a Mon Sep 17 00:00:00 2001
From: Daniel Silverstone <daniel.silverstone@codethink.co.uk>
Date: Fri, 7 Sep 2012 13:17:13 +0100
Subject: Support for fetching artifacts to the cache

Rather than pushing artifacts to the cache, this method allows the caller to
specify a host and artifact which the cache server will then fetch into its
local cache.  It takes the following arguments:

    host=hostname:port
    artifact=artifactname

This is transformed into a fetch to:

    http://hostname:port/artifacts?basename=artifactname

Which is then fetched into the cache under the given name.

The return from this is a JSON object of the form:

{
    "filename": artifactname,
    "size": NBYTES_SIZE_OF_FILE,
    "used": NBYTES_DISK_SPACE_USED
}
---
 morph-cache-server | 50 +++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 41 insertions(+), 9 deletions(-)

diff --git a/morph-cache-server b/morph-cache-server
index b726c1e5..286e56db 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -20,6 +20,7 @@ import cliapp
 import logging
 import os
 import urllib
+import shutil
 
 from bottle import Bottle, request, response, run, static_file
 
@@ -94,17 +95,48 @@ class MorphCacheServer(cliapp.Application):
                 fsstinfo = os.statvfs(artifactdir)
                 results["freespace"] = fsstinfo.f_bsize * fsstinfo.f_bavail
                 for fname in filenames:
-                    try:
-                        stinfo = os.stat("%s/%s" % (artifactdir, fname))
-                        files[fname] = {
-                            "atime": stinfo.st_atime,
-                            "size": stinfo.st_size,
-                            "used": stinfo.st_blocks * 512,
-                            }
-                    except Exception, e:
-                        print(e)
+                    if not fname.startswith(".dl."):
+                        try:
+                            stinfo = os.stat("%s/%s" % (artifactdir, fname))
+                            files[fname] = {
+                                "atime": stinfo.st_atime,
+                                "size": stinfo.st_size,
+                                "used": stinfo.st_blocks * 512,
+                                }
+                        except Exception, e:
+                            print(e)
             return results
 
+        @writable('/fetch')
+        def fetch():
+            host = self._unescape_parameter(request.query.host)
+            artifact = self._unescape_parameter(request.query.artifact)
+            try:
+                response.set_header('Cache-Control', 'no-cache')
+                in_fh = urllib.urlopen("http://%s/artifacts?basename=%s" %
+                                       (host, urllib.quote(artifact)))
+                tmpname = "%s/.dl.%s" % (
+                    self.settings['artifact-dir'],
+                    artifact)
+                localtmp = open(tmpname, "w")
+                shutil.copyfileobj(in_fh, localtmp)
+                localtmp.close()
+                in_fh.close()
+                artifilename = "%s/%s" % (self.settings['artifact-dir'],
+                                          artifact)
+                os.rename(tmpname, artifilename)
+                stinfo = os.stat(artifilename)
+                ret = {}
+                ret[artifact] = {
+                    "size": stinfo.st_size,
+                    "used": stinfo.st_blocks * 512
+                    }
+                return ret
+
+            except Exception, e:
+                response.status = 500
+                logging.debug('%s' % e)
+
         @app.get('/sha1s')
         def sha1():
             repo = self._unescape_parameter(request.query.repo)
-- 
cgit v1.2.1


From 5870c3581ac10c14d68337fc875000ead522e99d Mon Sep 17 00:00:00 2001
From: Daniel Silverstone <daniel.silverstone@codethink.co.uk>
Date: Fri, 7 Sep 2012 13:23:51 +0100
Subject: Add facility to delete artifacts

In order to allow the artifact cache to be cleaned up, this patch allows for a
/delete method which can remove artifacts from the cache.  It takes the
following arguments:

    artifact=artifactname

The artifact will be deleted and a JSON object returned in the form:

{
    "status": errno,
    "reason": strerror
}

Where errno is zero on success, 1 on EPERM, 2 on ENOENT etc. and reason is the
strerror of the errno, in case the architectures differ between caller and
cache.
---
 morph-cache-server | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/morph-cache-server b/morph-cache-server
index 286e56db..b4f8fa1a 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -137,6 +137,19 @@ class MorphCacheServer(cliapp.Application):
                 response.status = 500
                 logging.debug('%s' % e)
 
+        @writable('/delete')
+        def delete():
+            artifact = self._unescape_parameter(request.query.artifact)
+            try:
+                os.unlink('%s/%s' % (self.settings['artifact-dir'],
+                                     artifact))
+                return { "status": 0, "reason": "success" }
+            except OSError, ose:
+                return { "status": ose.errno, "reason": ose.strerror }
+            except Exception, e:
+                response.status = 500
+                logging.debug('%s' % e)
+
         @app.get('/sha1s')
         def sha1():
             repo = self._unescape_parameter(request.query.repo)
-- 
cgit v1.2.1


From 2ee9e745d46fd01f95cf598025ae10b88d1c051f Mon Sep 17 00:00:00 2001
From: Daniel Silverstone <daniel.silverstone@codethink.co.uk>
Date: Mon, 17 Sep 2012 13:49:21 +0100
Subject: Update /fetch API to latest definition

---
 morph-cache-server | 71 +++++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 51 insertions(+), 20 deletions(-)

diff --git a/morph-cache-server b/morph-cache-server
index b4f8fa1a..04a5710c 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -20,6 +20,7 @@ import cliapp
 import logging
 import os
 import urllib
+import urllib2
 import shutil
 
 from bottle import Bottle, request, response, run, static_file
@@ -59,6 +60,52 @@ class MorphCacheServer(cliapp.Application):
         self.settings.boolean(['enable-writes'],
                               'enable the write methods (fetch and delete)')
 
+    def _fetch_artifact(self, url, filename):
+        in_fh = None
+        try:
+            in_fh = urllib2.urlopen(url)
+            with open(filename, "w") as localtmp:
+                shutil.copyfileobj(in_fh, localtmp)
+            in_fh.close()
+        except Exception, e:
+            if in_fh is not None:
+                in_fh.close()
+            raise
+        else:
+            if in_fh is not None:
+                in_fh.close()
+        return os.stat(filename)
+
+    def _fetch_artifacts(self, server, cacheid, artifacts):
+        ret = {}
+        try:
+            for artifact in artifacts:
+                artifact_name = "%s.%s" % (cacheid, artifact)
+                tmpname = os.path.join(self.settings['artifact-dir'],
+                                       ".dl.%s" % artifact_name)
+                url = "http://%s/1.0/artifacts?filename=%s" % (
+                    server, urllib.quote(artifact_name))
+                stinfo = self._fetch_artifact(url, tmpname)
+                ret[artifact_name] = {
+                    "size": stinfo.st_size,
+                    "used": stinfo.st_blocks * 512,
+                    }
+        except Exception, e:
+            for artifact in ret.iterkeys():
+                os.unlink(os.path.join(self.settings['artifact-dir'],
+                                       ".dl.%s" % artifact))
+            raise
+
+        for artifact in ret.iterkeys():
+            tmpname = os.path.join(self.settings['artifact-dir'],
+                                   ".dl.%s" % artifact)
+            artifilename = os.path.join(self.settings['artifact-dir'],
+                                        artifact)
+            os.rename(tmpname, artifilename)
+
+        return ret
+
+
     def process_args(self, args):
         app = Bottle()
 
@@ -110,28 +157,12 @@ class MorphCacheServer(cliapp.Application):
         @writable('/fetch')
         def fetch():
             host = self._unescape_parameter(request.query.host)
-            artifact = self._unescape_parameter(request.query.artifact)
+            cacheid = self._unescape_parameter(request.query.cacheid)
+            artifacts = self._unescape_parameter(request.query.artifacts)
             try:
                 response.set_header('Cache-Control', 'no-cache')
-                in_fh = urllib.urlopen("http://%s/artifacts?basename=%s" %
-                                       (host, urllib.quote(artifact)))
-                tmpname = "%s/.dl.%s" % (
-                    self.settings['artifact-dir'],
-                    artifact)
-                localtmp = open(tmpname, "w")
-                shutil.copyfileobj(in_fh, localtmp)
-                localtmp.close()
-                in_fh.close()
-                artifilename = "%s/%s" % (self.settings['artifact-dir'],
-                                          artifact)
-                os.rename(tmpname, artifilename)
-                stinfo = os.stat(artifilename)
-                ret = {}
-                ret[artifact] = {
-                    "size": stinfo.st_size,
-                    "used": stinfo.st_blocks * 512
-                    }
-                return ret
+                artifacts = artifacts.split(",")
+                return self._fetch_artifacts(host, cacheid, artifacts)
 
             except Exception, e:
                 response.status = 500
-- 
cgit v1.2.1


From f7b8001175f492b41d68269ff5294fde347f8f4d Mon Sep 17 00:00:00 2001
From: Daniel Silverstone <daniel.silverstone@codethink.co.uk>
Date: Tue, 25 Sep 2012 11:53:55 +0100
Subject: Trim leading slashes from URI element during direct-mode. 
 rs=richardmaw

---
 morphcacheserver/repocache.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index b55692f2..b7d46c35 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -137,7 +137,10 @@ class RepoCache(object):
         
     def _quote_url(self, url, always_indirect=False):
         if self.direct_mode and not always_indirect:
-            return urlparse.urlparse(url)[2]
+            quoted_url = urlparse.urlparse(url)[2]
+            while quoted_url.startswith("/"):
+                quoted_url = quoted_url[1:]
+            return quoted_url
         else:
             valid_chars = string.digits + string.letters + '%_'
             transl = lambda x: x if x in valid_chars else '_'
-- 
cgit v1.2.1


From a1232696c7e57d98cb1bc3b093cde6f8c4eff5e6 Mon Sep 17 00:00:00 2001
From: Jannis Pohlmann <jannis.pohlmann@codethink.co.uk>
Date: Mon, 3 Dec 2012 14:05:48 +0000
Subject: Fix missing argument to rev_list(), breaking resolve_ref for SHA1s

This bug has been present since the initial commit to the cache
server. Due to the missing repo_dir argument to rev_list(), resolving
SHA1s rather than symbolic refs via /1.0/sha1s fails. This feature,
however, is absolutely required for morph to resolve petrified system
branches.
---
 morphcacheserver/repocache.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index b7d46c35..9675e04e 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -72,7 +72,7 @@ class RepoCache(object):
         if not self._is_valid_sha1(ref):
             raise InvalidReferenceError(repo_url, ref)
         try:
-            sha = self._rev_list(ref).strip()
+            sha = self._rev_list(repo_dir, ref).strip()
             return sha, self._tree_from_commit(repo_dir, sha)
         except:
             raise InvalidReferenceError(repo_url, ref)
-- 
cgit v1.2.1


From 90e9be0128cae0a93d481c0a5a1bcb9fed6e2ee2 Mon Sep 17 00:00:00 2001
From: Sam Thursfield <sam.thursfield@codethink.co.uk>
Date: Thu, 13 Dec 2012 17:37:47 +0000
Subject: Use 'git rev-parse --verify' to resolve refs

'git show-ref' returns multiple results where there are partial matches
for the given ref, which creates the possibility that we might resolve
a ref incorrectly. 'git rev-list' is also overkill for verifying that a
SHA1 is valid.
---
 morphcacheserver/repocache.py | 31 +++++++++----------------------
 1 file changed, 9 insertions(+), 22 deletions(-)

diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index 9675e04e..388436b0 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -59,24 +59,14 @@ class RepoCache(object):
             if not os.path.exists(repo_dir):
                 raise RepositoryNotFoundError(repo_url)
         try:
-            refs = self._show_ref(repo_dir, ref).split('\n')
-            if self.direct_mode:
-                refs = [x.split() for x in refs]
-            else:
-                refs = [x.split() for x in refs if 'origin' in x]
-            return refs[0][0], self._tree_from_commit(repo_dir, refs[0][0])
+            if not self.direct_mode and not refs.startswith('refs/origin/'):
+                ref = 'refs/origin/' + ref
+            sha1 = self._rev_parse(repo_dir, ref)
+            return sha1, self._tree_from_commit(repo_dir, sha1)
 
         except cliapp.AppException:
             pass
 
-        if not self._is_valid_sha1(ref):
-            raise InvalidReferenceError(repo_url, ref)
-        try:
-            sha = self._rev_list(repo_dir, ref).strip()
-            return sha, self._tree_from_commit(repo_dir, sha)
-        except:
-            raise InvalidReferenceError(repo_url, ref)
-
     def _tree_from_commit(self, repo_dir, commitsha):
         commit_info = self.app.runcmd(['git', 'log', '-1',
                                        '--format=format:%T', commitsha],
@@ -95,7 +85,7 @@ class RepoCache(object):
         if not os.path.exists(repo_dir):
             raise RepositoryNotFoundError(repo_url)
         try:
-            sha1 = self._rev_list(repo_dir, ref).strip()
+            sha1 = self._rev_parse(repo_dir, ref)
         except:
             raise InvalidReferenceError(repo_url, ref)
 
@@ -114,7 +104,7 @@ class RepoCache(object):
             raise RepositoryNotFoundError(repo_url)
 
         try:
-            sha1 = self._rev_list(repo_dir, ref).strip()
+            sha1 = self._rev_parse(repo_dir, ref)
         except:
             raise InvalidReferenceError(repo_url, ref)
 
@@ -146,12 +136,9 @@ class RepoCache(object):
             transl = lambda x: x if x in valid_chars else '_'
             return ''.join([transl(x) for x in url])
 
-    def _show_ref(self, repo_dir, ref):
-        return self.app.runcmd(['git', 'show-ref', ref], cwd=repo_dir)
-
-    def _rev_list(self, repo_dir, ref):
-        return self.app.runcmd(
-                ['git', 'rev-list', '--no-walk', ref], cwd=repo_dir)
+    def _rev_parse(self, repo_dir, ref):
+        return self.app.runcmd(['git', 'rev-parse', '--verify', ref],
+                               cwd=repo_dir)[0:40]
 
     def _cat_file(self, repo_dir, sha1, filename):
         return self.app.runcmd(
-- 
cgit v1.2.1


From dcb9b9ce10c918f66bc9d239e72a805847247be8 Mon Sep 17 00:00:00 2001
From: Sam Thursfield <sam.thursfield@codethink.co.uk>
Date: Thu, 13 Dec 2012 17:58:51 +0000
Subject: Log exceptions instead of ignoring them

---
 morphcacheserver/repocache.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index 388436b0..fa7a515b 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -65,7 +65,7 @@ class RepoCache(object):
             return sha1, self._tree_from_commit(repo_dir, sha1)
 
         except cliapp.AppException:
-            pass
+            raise
 
     def _tree_from_commit(self, repo_dir, commitsha):
         commit_info = self.app.runcmd(['git', 'log', '-1',
-- 
cgit v1.2.1


From 74df327db1727a26986cdf63294ae93cf3106081 Mon Sep 17 00:00:00 2001
From: Sam Thursfield <sam.thursfield@codethink.co.uk>
Date: Thu, 13 Dec 2012 19:01:59 +0000
Subject: Fix misspelled variable name

---
 morphcacheserver/repocache.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index fa7a515b..668d8fbb 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -59,7 +59,7 @@ class RepoCache(object):
             if not os.path.exists(repo_dir):
                 raise RepositoryNotFoundError(repo_url)
         try:
-            if not self.direct_mode and not refs.startswith('refs/origin/'):
+            if not self.direct_mode and not ref.startswith('refs/origin/'):
                 ref = 'refs/origin/' + ref
             sha1 = self._rev_parse(repo_dir, ref)
             return sha1, self._tree_from_commit(repo_dir, sha1)
-- 
cgit v1.2.1


From 22d1bfbc91a46134dd6c9410b86a3cb3ba250887 Mon Sep 17 00:00:00 2001
From: Jannis Pohlmann <jannis.pohlmann@codethink.co.uk>
Date: Wed, 2 Jan 2013 16:28:28 +0000
Subject: Handle batch sha1/file queries using POST requests

This commit adds support for resolving multiple refs into SHA1s and
requesting multiple files at once, each using a single POST request.

The (repo, ref) and (repo, ref, filename) parameters are passed to
the POST requests as JSON dictionaries in a list. The response to
both types of requests is a JSON list with the same dictionaries
again, to which a "sha1" and "data" field are added, respectively.

The file contents returned by "POST /1.0/files" requests are
base64-encoded and need to be decoded at the receiver's end. This
is because the contents may be binary or contain quotes and
therefore cause JSON syntax errors.
---
 morph-cache-server | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/morph-cache-server b/morph-cache-server
index 04a5710c..d3e42c62 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -16,7 +16,9 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
+import base64
 import cliapp
+import json
 import logging
 import os
 import urllib
@@ -197,6 +199,31 @@ class MorphCacheServer(cliapp.Application):
             except Exception, e:
                 response.status = 404
                 logging.debug('%s' % e)
+
+        @app.post('/sha1s')
+        def sha1s():
+            result = []
+            for pair in request.json:
+                repo = pair['repo']
+                ref = pair['ref']
+                try:
+                    sha1, tree = repo_cache.resolve_ref(repo, ref)
+                    result.append({
+                        'repo': '%s' % repo,
+                        'ref': '%s' % ref,
+                        'sha1': '%s' % sha1,
+                        'tree': '%s' % tree
+                    })
+                except Exception, e:
+                    logging.debug('%s' % e)
+                    result.append({
+                        'repo': '%s' % repo,
+                        'ref': '%s' % ref,
+                        'error': '%s' % e
+                    })
+            response.set_header('Cache-Control', 'no-cache')
+            response.set_header('Content-Type', 'application/json')
+            return json.dumps(result)
         
         @app.get('/files')
         def file():
@@ -211,6 +238,32 @@ class MorphCacheServer(cliapp.Application):
                 response.status = 404
                 logging.debug('%s' % e)
 
+        @app.post('/files')
+        def files():
+            result = []
+            for pair in request.json:
+                repo = pair['repo']
+                ref = pair['ref']
+                filename = pair['filename']
+                try:
+                    content = repo_cache.cat_file(repo, ref, filename)
+                    result.append({
+                        'repo': '%s' % repo,
+                        'ref': '%s' % ref,
+                        'filename': '%s' % filename,
+                        'data': '%s' % base64.b64encode(content),
+                    })
+                except Exception, e:
+                    logging.debug('%s' % e)
+                    result.append({
+                        'repo': '%s' % repo,
+                        'ref': '%s' % ref,
+                        'filename': '%s' % filename,
+                        'error': '%s' % e
+                    })
+            response.set_header('Content-Type', 'application/json')
+            return json.dumps(result)
+
         @app.get('/trees')
         def tree():
             repo = self._unescape_parameter(request.query.repo)
-- 
cgit v1.2.1


From 2b94817befc8538f85f454a8e112d7ae828ecf52 Mon Sep 17 00:00:00 2001
From: Jonathan Maw <jonathan.maw@codethink.co.uk>
Date: Tue, 4 Jun 2013 11:32:00 +0100
Subject: Handle requesting a sha1 of a sha1

---
 morphcacheserver/repocache.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index 668d8fbb..cd2eab56 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -16,6 +16,7 @@
 
 import cliapp
 import os
+import re
 import string
 import urlparse
 
@@ -59,9 +60,13 @@ class RepoCache(object):
             if not os.path.exists(repo_dir):
                 raise RepositoryNotFoundError(repo_url)
         try:
-            if not self.direct_mode and not ref.startswith('refs/origin/'):
-                ref = 'refs/origin/' + ref
-            sha1 = self._rev_parse(repo_dir, ref)
+            if re.match('^[0-9a-fA-F]{40}$', ref):
+                sha1 = ref
+            else:
+                if (not self.direct_mode and
+                    not ref.startswith('refs/origin/')):
+                    ref = 'refs/origin/' + ref
+                sha1 = self._rev_parse(repo_dir, ref)
             return sha1, self._tree_from_commit(repo_dir, sha1)
 
         except cliapp.AppException:
-- 
cgit v1.2.1


From 5e4ad62d4d5b8ecabf4aa01b6de918fc322b8292 Mon Sep 17 00:00:00 2001
From: Tiago Gomes <tiago.gomes@codethink.co.uk>
Date: Wed, 4 Sep 2013 16:29:50 +0100
Subject: Allow to use flup as the backend server for bottle.

lighttpd does not support WSGI, so we need to be able to run
the server in FCGI mode. The flup package provides one.

Running with the default bottle server is still useful
for testing purposes.
---
 morph-cache-server | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/morph-cache-server b/morph-cache-server
index d3e42c62..1c9a53de 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -26,7 +26,7 @@ import urllib2
 import shutil
 
 from bottle import Bottle, request, response, run, static_file
-
+from flup.server.fcgi import WSGIServer
 from morphcacheserver.repocache import RepoCache
 
 
@@ -61,6 +61,10 @@ class MorphCacheServer(cliapp.Application):
                               'cache directories are directly managed')
         self.settings.boolean(['enable-writes'],
                               'enable the write methods (fetch and delete)')
+        self.settings.boolean(['fcgi-server'],
+                              'runs a fcgi-server',
+                              default=True)
+
 
     def _fetch_artifact(self, url, filename):
         in_fh = None
@@ -302,8 +306,12 @@ class MorphCacheServer(cliapp.Application):
         
         root = Bottle()
         root.mount(app, '/1.0')
-        
-        run(root, host='0.0.0.0', port=self.settings['port'], reloader=True)
+
+
+	if self.settings['fcgi-server']:
+            WSGIServer(root).run()
+        else:
+            run(root, host='0.0.0.0', port=self.settings['port'], reloader=True)
 
     def _unescape_parameter(self, param):
         return urllib.unquote(param)
-- 
cgit v1.2.1


From cc5f95fa563c4817cdcffc428da18e263bd02ae2 Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Mon, 16 Dec 2013 13:54:09 +0000
Subject: Update copyright notice

---
 README                        | 21 ---------------------
 morph-cache-server            |  2 +-
 morphcacheserver/__init__.py  |  2 +-
 morphcacheserver/repocache.py |  2 +-
 4 files changed, 3 insertions(+), 24 deletions(-)
 delete mode 100644 README

diff --git a/README b/README
deleted file mode 100644
index a57e9a06..00000000
--- a/README
+++ /dev/null
@@ -1,21 +0,0 @@
-README for morph-cache-server
-=============================
-
-Legalese
---------
-
-Copyright (C) 2012  Codethink Limited
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
diff --git a/morph-cache-server b/morph-cache-server
index 1c9a53de..dbf67856 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2013  Codethink Limited
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
diff --git a/morphcacheserver/__init__.py b/morphcacheserver/__init__.py
index 9ad5a305..2c25ce28 100644
--- a/morphcacheserver/__init__.py
+++ b/morphcacheserver/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2013  Codethink Limited
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index cd2eab56..0e4d909e 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2013  Codethink Limited
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
-- 
cgit v1.2.1


From 15b7e4fad677c127d4011babc01f7e4a71f259d8 Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Thu, 3 Apr 2014 14:41:31 +0100
Subject: Add post request for /artifacts

With this we can request the state of a set of artifacts
in a single request. Artifacts are sent as a json array.
We check whether each artifact is in the cache or not
and send our findings back to the client as a json object.
---
 morph-cache-server | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/morph-cache-server b/morph-cache-server
index dbf67856..a3c3c978 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright (C) 2013  Codethink Limited
+# Copyright (C) 2013, 2014 Codethink Limited
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -303,12 +303,40 @@ class MorphCacheServer(cliapp.Application):
             else:
                 response.status = 404
                 logging.debug('artifact %s does not exist' % basename)
-        
+
+        @app.post('/artifacts')
+        def post_artifacts():
+            if request.content_type != 'application/json':
+                logging.warning('Content-type is not json: '
+                    'expecting a json post request')
+
+            artifacts = json.load(request.body)
+            results = {}
+
+            logging.debug('Received a POST request for /artifacts')
+
+            for artifact in artifacts:
+                if artifact.startswith('/'):
+                    response.status = 500
+                    logging.error("%s: artifact name cannot start with a '/'"
+                        % artifact)
+                    return
+
+                filename = os.path.join(self.settings['artifact-dir'], artifact)
+                results[artifact] = os.path.exists(filename)
+
+                if results[artifact]:
+                    logging.debug('%s is in the cache', artifact)
+                else:
+                    logging.debug('%s is NOT in the cache', artifact)
+
+            return results
+
         root = Bottle()
         root.mount(app, '/1.0')
 
 
-	if self.settings['fcgi-server']:
+        if self.settings['fcgi-server']:
             WSGIServer(root).run()
         else:
             run(root, host='0.0.0.0', port=self.settings['port'], reloader=True)
-- 
cgit v1.2.1