From 86dab9e0377370d6ee57b905051d3a5331ce9478 Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Thu, 8 Jan 2015 11:38:33 +0000
Subject: Add python.yaml

This adds a config file for the python extensions
---
 baserockimport/data/python.yaml | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 baserockimport/data/python.yaml

diff --git a/baserockimport/data/python.yaml b/baserockimport/data/python.yaml
new file mode 100644
index 0000000..6836714
--- /dev/null
+++ b/baserockimport/data/python.yaml
@@ -0,0 +1,2 @@
+---
+lorry-prefix: python
-- 
cgit v1.2.1


From 0553ce72422eb55ba6389037df3290b30142dd60 Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Thu, 8 Jan 2015 11:30:48 +0000
Subject: Read lorry prefix from yaml file

This adds PythonLorryExtension class to python.to_lorry to run the extension
in a more conventional way. Previously the PythonExtension class
was used to execute any of the extensions (it would call the extensions main()
function). We move away from this so the
extension can access useful methods, such as local_data_path(), that are
provided by the ImportExtension class.

This also removes use of pkg_resources.parse_requirement which is redundant.
This also removes the unused import of select.
---
 baserockimport/exts/python.to_lorry | 69 +++++++++++++++++++++----------------
 1 file changed, 39 insertions(+), 30 deletions(-)

diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry
index accc9dc..06ee31d 100755
--- a/baserockimport/exts/python.to_lorry
+++ b/baserockimport/exts/python.to_lorry
@@ -28,7 +28,7 @@ import shutil
 import tempfile
 import xmlrpclib
 import logging
-import select
+import yaml
 
 import pkg_resources
 
@@ -109,9 +109,8 @@ def get_compression(url):
     return None
 
 # Assumption: url passed to this function must have a 'standard' tar extension
-def make_tarball_lorry(package_name, url):
-    # TODO: this prefix probably shouldn't be hardcoded here either
-    name = 'python-packages/%s' % package_name.lower()
+def make_tarball_lorry(lorry_prefix, package_name, url):
+    name = '%s/%s' % (lorry_prefix, package_name.lower())
 
     lorry = {'type': 'tarball', 'url': url}
     compression = get_compression(url)
@@ -138,7 +137,7 @@ def get_releases(client, requirement):
 
     return releases
 
-def generate_tarball_lorry(client, requirement):
+def generate_tarball_lorry(lorry_prefix, client, requirement):
     releases = get_releases(client, requirement)
 
     if len(releases) == 0:
@@ -178,43 +177,53 @@ def generate_tarball_lorry(client, requirement):
 
     url = urls[0]['url']
 
-    return make_tarball_lorry(requirement.project_name, url)
+    return make_tarball_lorry(lorry_prefix, requirement.project_name, url)
 
-def str_repo_lorry(package_name, repo_type, url):
-    # TODO: this prefix probably shouldn't be hardcoded here
-    name = 'python-packages/%s' % package_name.lower()
+def str_repo_lorry(lorry_prefix, package_name, repo_type, url):
+    name = '%s/%s' % (lorry_prefix, package_name.lower())
 
     return json.dumps({name: {'type': repo_type, 'url': url}},
                       indent=4, sort_keys=True)
 
-def main():
-    if len(sys.argv) != 2:
-        # TODO explain the format of python requirements
-        # warn the user that they probably want to quote their arg
-        # > < will be interpreted as redirection by the shell
-        print('usage: %s requirement' % sys.argv[0], file=sys.stderr)
-        sys.exit(1)
+class PythonLorryExtension(ImportExtension):
 
-    client = xmlrpclib.ServerProxy(PYPI_URL)
+    def __init__(self):
+        super(PythonLorryExtension, self).__init__()
 
-    req = pkg_resources.parse_requirements(sys.argv[1]).next()
+    def run(self):
+        if len(sys.argv) != 2:
+            # TODO explain the format of python requirements
+            # warn the user that they probably want to quote their arg
+            # > < will be interpreted as redirection by the shell
+            print('usage: %s requirement' % sys.argv[0], file=sys.stderr)
+            sys.exit(1)
 
-    new_proj_name = name_or_closest(client, req.project_name)
+        client = xmlrpclib.ServerProxy(PYPI_URL)
 
-    if new_proj_name == None:
-        error("Couldn't find any project with name '%s'" % req.project_name)
+        req = pkg_resources.parse_requirements(sys.argv[1]).next()
 
-    logging.debug('Treating %s as %s' % (req.project_name, new_proj_name))
-    req.project_name = new_proj_name
+        with open(self.local_data_path('python.yaml')) as f:
+            lorry_prefix = yaml.load(f)['lorry-prefix']
 
-    metadata = fetch_package_metadata(req.project_name)
-    info = metadata['info']
+        new_proj_name = name_or_closest(client, req.project_name)
 
-    repo_type = (find_repo_type(info['home_page'])
-                 if 'home_page' in info else None)
+        if new_proj_name == None:
+            error("Couldn't find any project with name '%s'" % req.project_name)
 
-    print(str_repo_lorry(req.project_name, repo_type, info['home_page'])
-            if repo_type else generate_tarball_lorry(client, req))
+        logging.debug('Treating %s as %s' % (req.project_name, new_proj_name))
+        req.project_name = new_proj_name
+
+        metadata = fetch_package_metadata(req.project_name)
+        info = metadata['info']
+
+        repo_type = (find_repo_type(info['home_page'])
+                     if 'home_page' in info else None)
+
+        if repo_type:
+            print(str_repo_lorry(lorry_prefix, req.project_name,
+                                 repo_type, info['home_page']))
+        else:
+            print(generate_tarball_lorry(lorry_prefix, client, req))
 
 if __name__ == '__main__':
-    PythonExtension().run()
+    PythonLorryExtension().run()
-- 
cgit v1.2.1


From 827e9904332dbd17341648be81bcf2388d241ad4 Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Thu, 8 Jan 2015 11:59:00 +0000
Subject: Make python.to_lorry output x-products-python field

---
 baserockimport/exts/python.to_lorry | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry
index 06ee31d..fdbd04a 100755
--- a/baserockimport/exts/python.to_lorry
+++ b/baserockimport/exts/python.to_lorry
@@ -112,7 +112,10 @@ def get_compression(url):
 def make_tarball_lorry(lorry_prefix, package_name, url):
     name = '%s/%s' % (lorry_prefix, package_name.lower())
 
-    lorry = {'type': 'tarball', 'url': url}
+    # TODO: shouldn't have 'x-products-python' field hardcoded here either
+    lorry = {'type': 'tarball',
+             'url': url,
+             'x-products-python': [package_name.lower()]}
     compression = get_compression(url)
     if compression:
         lorry['compression'] = compression
@@ -182,7 +185,11 @@ def generate_tarball_lorry(lorry_prefix, client, requirement):
 def str_repo_lorry(lorry_prefix, package_name, repo_type, url):
     name = '%s/%s' % (lorry_prefix, package_name.lower())
 
-    return json.dumps({name: {'type': repo_type, 'url': url}},
+    # TODO: this products field 'x-products-python'
+    # probably shouldn't be hardcoded here
+    return json.dumps({name: {'type': repo_type,
+                              'url': url,
+                              'x-products-python': [package_name.lower()]}},
                       indent=4, sort_keys=True)
 
 class PythonLorryExtension(ImportExtension):
-- 
cgit v1.2.1


From e8bf3d6e80cf3f29f7a07536eab68c32dcaf7f4b Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Thu, 8 Jan 2015 14:23:10 +0000
Subject: Add package name comparison callback

This lets the importer specify how package names will be compared,
to do this the importer optionally supplies a callback, if no callback
is supplied then the tool will default to == for string comparison.
---
 baserockimport/lorryset.py | 5 +++--
 baserockimport/mainloop.py | 9 ++++++++-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/baserockimport/lorryset.py b/baserockimport/lorryset.py
index 8cc73af..f252b9f 100644
--- a/baserockimport/lorryset.py
+++ b/baserockimport/lorryset.py
@@ -107,7 +107,7 @@ class LorrySet(object):
         '''Return the lorry entry for the named project.'''
         return {name: self.data[name]}
 
-    def find_lorry_for_package(self, kind, package_name):
+    def find_lorry_for_package(self, kind, package_name, comp):
         '''Find the lorry entry for a given foreign package, or return None.
 
         This makes use of an extension to the .lorry format made by the
@@ -116,11 +116,12 @@ class LorrySet(object):
         named $KIND.
 
         '''
+
         key = 'x-products-%s' % kind
         for name, lorry in self.data.iteritems():
             products = lorry.get(key, [])
             for entry in products:
-                if entry == package_name:
+                if comp(entry, package_name):
                     return {name: lorry}
 
         return None
diff --git a/baserockimport/mainloop.py b/baserockimport/mainloop.py
index 057ab98..b9f1b9b 100644
--- a/baserockimport/mainloop.py
+++ b/baserockimport/mainloop.py
@@ -293,7 +293,14 @@ class ImportLoop(object):
         # files are named for project name rather than package name. In this
         # case we will generate the lorry, and try to add it to the set, at
         # which point LorrySet will notice the existing one and merge the two.
-        lorry = self.lorry_set.find_lorry_for_package(kind, name)
+        comp = None
+
+        if 'package_comp_callback' in self.importers[kind]['kwargs']:
+            comp = self.importers[kind]['kwargs']['package_comp_callback']
+        else:
+            comp = lambda x, y: x == y
+
+        lorry = self.lorry_set.find_lorry_for_package(kind, name, comp)
 
         if lorry is None:
             lorry = self._generate_lorry_for_package(kind, name)
-- 
cgit v1.2.1


From ecf6dbb587b676405809f26657be4f2b5d8a55b7 Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Thu, 8 Jan 2015 16:06:21 +0000
Subject: Use comparison callback in python importer

As well as being case insensitive project names on python treat '-' and '_'
as equivalent characters [1]

[1]: http://legacy.python.org/dev/peps/pep-0426/#name
---
 baserockimport/app.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/baserockimport/app.py b/baserockimport/app.py
index cd3203e..0b190e5 100644
--- a/baserockimport/app.py
+++ b/baserockimport/app.py
@@ -206,6 +206,10 @@ class BaserockImportApplication(cliapp.Application):
 
     def import_python(self, args):
         '''Import one or more python packages.'''
+
+        def comp(x, y):
+            return x.replace('-', '_').lower() == y.replace('-', '_').lower()
+
         if len(args) < 1 or len(args) > 2:
             raise cliapp.AppException(
                 'Please pass the name of the python package on the commandline.')
@@ -218,5 +222,6 @@ class BaserockImportApplication(cliapp.Application):
                                                   goal_kind='python',
                                                   goal_name=package_name,
                                                   goal_version=package_version)
-        loop.enable_importer('python', strata=['strata/core.morph'])
+        loop.enable_importer('python', strata=['strata/core.morph'],
+                             package_comp_callback=comp)
         loop.run()
-- 
cgit v1.2.1


From ec93dd99f5924b7741a4b3c2c1fab961a213d519 Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Thu, 8 Jan 2015 15:34:42 +0000
Subject: Don't convert package names to lower case

This is redundant.
---
 baserockimport/exts/python.to_lorry | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry
index fdbd04a..3f0a463 100755
--- a/baserockimport/exts/python.to_lorry
+++ b/baserockimport/exts/python.to_lorry
@@ -110,12 +110,12 @@ def get_compression(url):
 
 # Assumption: url passed to this function must have a 'standard' tar extension
 def make_tarball_lorry(lorry_prefix, package_name, url):
-    name = '%s/%s' % (lorry_prefix, package_name.lower())
+    name = '%s/%s' % (lorry_prefix, package_name)
 
     # TODO: shouldn't have 'x-products-python' field hardcoded here either
     lorry = {'type': 'tarball',
              'url': url,
-             'x-products-python': [package_name.lower()]}
+             'x-products-python': [package_name]}
     compression = get_compression(url)
     if compression:
         lorry['compression'] = compression
@@ -183,13 +183,13 @@ def generate_tarball_lorry(lorry_prefix, client, requirement):
     return make_tarball_lorry(lorry_prefix, requirement.project_name, url)
 
 def str_repo_lorry(lorry_prefix, package_name, repo_type, url):
-    name = '%s/%s' % (lorry_prefix, package_name.lower())
+    name = '%s/%s' % (lorry_prefix, package_name)
 
     # TODO: this products field 'x-products-python'
     # probably shouldn't be hardcoded here
     return json.dumps({name: {'type': repo_type,
                               'url': url,
-                              'x-products-python': [package_name.lower()]}},
+                              'x-products-python': [package_name]}},
                       indent=4, sort_keys=True)
 
 class PythonLorryExtension(ImportExtension):
-- 
cgit v1.2.1


From 48e41ded9cb6222dbfdad017ef9d41a89ff4c6c0 Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Thu, 8 Jan 2015 16:01:23 +0000
Subject: Fix bug in name_or_closest

---
 baserockimport/exts/importer_python_common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/baserockimport/exts/importer_python_common.py b/baserockimport/exts/importer_python_common.py
index b2e7c51..b51e680 100644
--- a/baserockimport/exts/importer_python_common.py
+++ b/baserockimport/exts/importer_python_common.py
@@ -60,6 +60,7 @@ def name_or_closest(client, package_name):
     #
     # so look for both the hyphenated version that is passed to this function
     # and the underscored version.
+    package_name = package_name.replace('_', '-')
     underscored_package_name = package_name.replace('-', '_')
 
     for name in [package_name, underscored_package_name]:
-- 
cgit v1.2.1


From 28eff204f1162ddc516a8ff4cfef894c2273ff7f Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Thu, 8 Jan 2015 17:21:07 +0000
Subject: Remove generic PythonExtension class

The two extensions have diverged so this generic class is no longer useful.
---
 baserockimport/exts/importer_python_common.py | 10 +------
 baserockimport/exts/python.find_deps          | 43 +++++++++++++++------------
 2 files changed, 25 insertions(+), 28 deletions(-)

diff --git a/baserockimport/exts/importer_python_common.py b/baserockimport/exts/importer_python_common.py
index b51e680..6c8fc66 100644
--- a/baserockimport/exts/importer_python_common.py
+++ b/baserockimport/exts/importer_python_common.py
@@ -89,12 +89,4 @@ def name_or_closest(client, package_name):
 
     return results[0]['name'] if len(results) > 0 else None
 
-# We subclass the ImportExtension to setup the logger,
-# so that we can send logs to the import tool's log
-class PythonExtension(ImportExtension):
-    def __init__(self):
-        super(PythonExtension, self).__init__()
-
-    def process_args(self, _):
-        import __main__
-        __main__.main()
+
diff --git a/baserockimport/exts/python.find_deps b/baserockimport/exts/python.find_deps
index b9791ef..524191e 100755
--- a/baserockimport/exts/python.find_deps
+++ b/baserockimport/exts/python.find_deps
@@ -327,31 +327,36 @@ def find_runtime_deps(source, name, version=None, use_requirements_file=False):
 
     return runtime_deps
 
-def main():
-    if len(sys.argv) not in [3, 4]:
-        print('usage: %s PACKAGE_SOURCE_DIR NAME [VERSION]' % sys.argv[0])
-        sys.exit(1)
+class PythonFindDepsExtension(ImportExtension):
 
-    logging.debug('%s: sys.argv[1:]: %s' % (sys.argv[0], sys.argv[1:]))
-    source, name = sys.argv[1:3]
-    version = sys.argv[3] if len(sys.argv) == 4 else None
+    def __init__(self):
+        super(PythonFindDepsExtension, self).__init__()
 
-    client = xmlrpclib.ServerProxy(PYPI_URL)
-    new_name = name_or_closest(client, name)
+    def run(self):
+        if len(sys.argv) not in [3, 4]:
+            print('usage: %s PACKAGE_SOURCE_DIR NAME [VERSION]' % sys.argv[0])
+            sys.exit(1)
 
-    if new_name == None:
-        error("Couldn't find any project with name '%s'" % name)
+        logging.debug('%s: sys.argv[1:]: %s' % (sys.argv[0], sys.argv[1:]))
+        source, name = sys.argv[1:3]
+        version = sys.argv[3] if len(sys.argv) == 4 else None
 
-    logging.debug('Treating %s as %s' % (name, new_name))
-    name = new_name
+        client = xmlrpclib.ServerProxy(PYPI_URL)
+        new_name = name_or_closest(client, name)
+
+        if new_name == None:
+            error("Couldn't find any project with name '%s'" % name)
+
+        logging.debug('Treating %s as %s' % (name, new_name))
+        name = new_name
 
-    deps = {}
-    deps['build-dependencies'] = find_build_deps(source, name, version)
-    deps['runtime-dependencies'] = find_runtime_deps(source, name, version)
+        deps = {}
+        deps['build-dependencies'] = find_build_deps(source, name, version)
+        deps['runtime-dependencies'] = find_runtime_deps(source, name, version)
 
-    root = {'python': deps}
+        root = {'python': deps}
 
-    print(json.dumps(root))
+        print(json.dumps(root))
 
 if __name__ == '__main__':
-    PythonExtension().run()
+    PythonFindDepsExtension().run()
-- 
cgit v1.2.1


From 9b01e0431e33c7ffc3aa40bc01e026b37da4745f Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Thu, 8 Jan 2015 18:13:09 +0000
Subject: Add option for debugging vcss

It's sometime useful to see the output of the vcs, but having this enabled
all the time clutters the log
---
 baserockimport/exts/python.to_lorry | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry
index 3f0a463..1452be8 100755
--- a/baserockimport/exts/python.to_lorry
+++ b/baserockimport/exts/python.to_lorry
@@ -47,6 +47,8 @@ def fetch_package_metadata(package_name):
 
 def find_repo_type(url):
 
+    debug_vcss = False
+
     # Don't bother with detection if we can't get a 200 OK
     logging.debug("Getting '%s' ..." % url)
 
@@ -79,7 +81,8 @@ def find_repo_type(url):
             if line == '':
                 break
 
-            logging.debug(line.rstrip('\n'))
+            if debug_vcss:
+                logging.debug(line.rstrip('\n'))
 
         p.wait()    # even with eof on both streams, we still wait
 
-- 
cgit v1.2.1


From 5bcb22616579cff9707324063f9310dfe51d9108 Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Thu, 8 Jan 2015 18:22:00 +0000
Subject: Add debug msg stating requirement

---
 baserockimport/exts/python.to_lorry | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry
index 1452be8..cb4e4ec 100755
--- a/baserockimport/exts/python.to_lorry
+++ b/baserockimport/exts/python.to_lorry
@@ -223,6 +223,8 @@ class PythonLorryExtension(ImportExtension):
         logging.debug('Treating %s as %s' % (req.project_name, new_proj_name))
         req.project_name = new_proj_name
 
+        logging.debug('Looking for requirement: %s', req)
+
         metadata = fetch_package_metadata(req.project_name)
         info = metadata['info']
 
-- 
cgit v1.2.1


From 4544eb81de6cea8ef0f81d89531bd3c61673586e Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Fri, 9 Jan 2015 11:25:32 +0000
Subject: Make import tool pass version to to_lorry ext

---
 baserockimport/mainloop.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/baserockimport/mainloop.py b/baserockimport/mainloop.py
index b9f1b9b..8a7ae26 100644
--- a/baserockimport/mainloop.py
+++ b/baserockimport/mainloop.py
@@ -202,7 +202,7 @@ class ImportLoop(object):
 
         # 1. Make the source code available.
 
-        lorry = self._find_or_create_lorry_file(kind, name)
+        lorry = self._find_or_create_lorry_file(kind, name, version)
         source_repo, url = self._fetch_or_update_source(lorry)
 
         checked_out_version, ref = self._checkout_source_version_for_package(
@@ -288,7 +288,7 @@ class ImportLoop(object):
             dep_package.set_is_build_dep(True)
             processed.add_edge(dep_package, current_item)
 
-    def _find_or_create_lorry_file(self, kind, name):
+    def _find_or_create_lorry_file(self, kind, name, version):
         # Note that the lorry file may already exist for 'name', but lorry
         # files are named for project name rather than package name. In this
         # case we will generate the lorry, and try to add it to the set, at
@@ -303,7 +303,7 @@ class ImportLoop(object):
         lorry = self.lorry_set.find_lorry_for_package(kind, name, comp)
 
         if lorry is None:
-            lorry = self._generate_lorry_for_package(kind, name)
+            lorry = self._generate_lorry_for_package(kind, name, version)
 
             if len(lorry) != 1:
                 raise Exception(
@@ -330,14 +330,18 @@ class ImportLoop(object):
 
         return lorry
 
-    def _generate_lorry_for_package(self, kind, name):
+    def _generate_lorry_for_package(self, kind, name, version):
         tool = '%s.to_lorry' % kind
         if kind not in self.importers:
             raise Exception('Importer for %s was not enabled.' % kind)
         extra_args = self.importers[kind]['extra_args']
         self.app.status(
             '%s: calling %s to generate lorry', name, tool)
-        lorry_text = run_extension(tool, extra_args + [name])
+
+        args = extra_args + [name]
+        if version != 'master':
+            args.append(version)
+        lorry_text = run_extension(tool, args)
         try:
             lorry = json.loads(lorry_text)
         except ValueError:
-- 
cgit v1.2.1


From 48de25fab7a0d88c1acf0a2a93ff1e98662c8ad1 Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Mon, 12 Jan 2015 15:40:08 +0000
Subject: Fix python.to_lorry to use version passed on cmdline

---
 baserockimport/exts/python.to_lorry | 64 ++++++++++++++++++-------------------
 1 file changed, 31 insertions(+), 33 deletions(-)

diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry
index cb4e4ec..db33e5f 100755
--- a/baserockimport/exts/python.to_lorry
+++ b/baserockimport/exts/python.to_lorry
@@ -135,36 +135,35 @@ def filter_urls(urls):
 
     return filter(allowed_extension, urls)
 
-def get_releases(client, requirement):
+def get_releases(client, package_name):
     try:
-        releases = client.package_releases(requirement.project_name)
+        releases = client.package_releases(package_name)
     except Exception as e:
         error("Couldn't fetch release data:", e)
 
     return releases
 
-def generate_tarball_lorry(lorry_prefix, client, requirement):
-    releases = get_releases(client, requirement)
+def generate_tarball_lorry(lorry_prefix, client, package_name, version=None):
+    releases = get_releases(client, package_name)
 
     if len(releases) == 0:
-        error("Couldn't find any releases for package %s"
-              % requirement.project_name)
+        error("Couldn't find any releases for package %s" % package_name)
 
-    releases = [v for v in releases if specs_satisfied(v, requirement.specs)]
+    logging.debug('Found releases: %s', str(releases))
 
-    if len(releases) == 0:
-        error("Couldn't find any releases of %s"
-              " that satisfy version constraints: %s"
-              % (requirement.project_name, requirement.specs))
+    # Use latest release if no version specified
+    version = version or releases[0]
 
-    release_version = releases[0]
+    if version not in releases:
+        error("Couldn't find any releases of %s with version: %s"
+              % (package_name, version))
 
-    logging.debug('Fetching urls for package %s with version %s'
-          % (requirement.project_name, release_version))
+    logging.debug('Fetching urls for package %s with version %s',
+                  package_name, version)
 
     try:
         # Get a list of dicts, the dicts contain the urls.
-        urls = client.release_urls(requirement.project_name, release_version)
+        urls = client.release_urls(package_name, version)
     except Exception as e:
         error("Couldn't fetch release urls:", e)
 
@@ -178,12 +177,11 @@ def generate_tarball_lorry(lorry_prefix, client, requirement):
             warn("\t%s" % url['url'])
         error("Cannot proceed")
     else:
-        error("Couldn't find any download urls for package %s"
-              % requirement.project_name)
+        error("Couldn't find any download urls for package %s" % package_name)
 
     url = urls[0]['url']
 
-    return make_tarball_lorry(lorry_prefix, requirement.project_name, url)
+    return make_tarball_lorry(lorry_prefix, package_name, url)
 
 def str_repo_lorry(lorry_prefix, package_name, repo_type, url):
     name = '%s/%s' % (lorry_prefix, package_name)
@@ -201,41 +199,41 @@ class PythonLorryExtension(ImportExtension):
         super(PythonLorryExtension, self).__init__()
 
     def run(self):
-        if len(sys.argv) != 2:
-            # TODO explain the format of python requirements
-            # warn the user that they probably want to quote their arg
-            # > < will be interpreted as redirection by the shell
-            print('usage: %s requirement' % sys.argv[0], file=sys.stderr)
+        if len(sys.argv) not in [2, 3]:
+            print('usage: %s NAME [VERSION]' % sys.argv[0], file=sys.stderr)
             sys.exit(1)
 
         client = xmlrpclib.ServerProxy(PYPI_URL)
 
-        req = pkg_resources.parse_requirements(sys.argv[1]).next()
+        package_name = sys.argv[1]
+        version = sys.argv[2] if len(sys.argv) == 3 else None
+
+        logging.debug('Looking for package: %s with version %s',
+                      package_name, version)
 
         with open(self.local_data_path('python.yaml')) as f:
             lorry_prefix = yaml.load(f)['lorry-prefix']
 
-        new_proj_name = name_or_closest(client, req.project_name)
+        new_proj_name = name_or_closest(client, package_name)
 
         if new_proj_name == None:
-            error("Couldn't find any project with name '%s'" % req.project_name)
-
-        logging.debug('Treating %s as %s' % (req.project_name, new_proj_name))
-        req.project_name = new_proj_name
+            error("Couldn't find any project with name '%s'" % package_name)
 
-        logging.debug('Looking for requirement: %s', req)
+        logging.debug('Treating %s as %s' % (package_name, new_proj_name))
+        package_name = new_proj_name
 
-        metadata = fetch_package_metadata(req.project_name)
+        metadata = fetch_package_metadata(package_name)
         info = metadata['info']
 
         repo_type = (find_repo_type(info['home_page'])
                      if 'home_page' in info else None)
 
         if repo_type:
-            print(str_repo_lorry(lorry_prefix, req.project_name,
+            print(str_repo_lorry(lorry_prefix, package_name,
                                  repo_type, info['home_page']))
         else:
-            print(generate_tarball_lorry(lorry_prefix, client, req))
+            print(generate_tarball_lorry(lorry_prefix, client,
+                                         package_name, version))
 
 if __name__ == '__main__':
     PythonLorryExtension().run()
-- 
cgit v1.2.1


From dfd8ab146d790ad582b3da8d417735d5134903eb Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Mon, 12 Jan 2015 15:40:47 +0000
Subject: Move get_releases into common and make all extensions use it

We also call package_releases with True, so that we also get versions
of releases that have been hidden. pip is willing to install from hidden
releases so we should too, the concept of hidden releases will eventually
disappear from pypi as well.
---
 baserockimport/exts/importer_python_common.py | 11 ++++++++++-
 baserockimport/exts/python.find_deps          |  2 +-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/baserockimport/exts/importer_python_common.py b/baserockimport/exts/importer_python_common.py
index 6c8fc66..eedc81b 100644
--- a/baserockimport/exts/importer_python_common.py
+++ b/baserockimport/exts/importer_python_common.py
@@ -23,6 +23,15 @@ from importer_base import ImportExtension
 
 PYPI_URL = 'http://pypi.python.org/pypi'
 
+def get_releases(client, package_name):
+    try:
+        # True here just means show hidden releases
+        releases = client.package_releases(package_name, True)
+    except Exception as e:
+        error("Couldn't fetch release data:", e)
+
+    return releases
+
 def warn(*args, **kwargs):
     print('%s:' % sys.argv[0], *args, file=sys.stderr, **kwargs)
 
@@ -64,7 +73,7 @@ def name_or_closest(client, package_name):
     underscored_package_name = package_name.replace('-', '_')
 
     for name in [package_name, underscored_package_name]:
-        results = client.package_releases(name)
+        results = get_releases(client, name)
 
         if len(results) > 0:
             logging.debug('Found package %s' % name)
diff --git a/baserockimport/exts/python.find_deps b/baserockimport/exts/python.find_deps
index 524191e..a73c457 100755
--- a/baserockimport/exts/python.find_deps
+++ b/baserockimport/exts/python.find_deps
@@ -189,7 +189,7 @@ def resolve_versions(specsets):
         logging.debug("Treating %s as %s" % (proj_name, new_proj_name))
         proj_name = new_proj_name
 
-        releases = client.package_releases(proj_name)
+        releases = get_releases(client, proj_name)
 
         logging.debug('Found %d releases of %s: %s'
                       % (len(releases), proj_name, releases))
-- 
cgit v1.2.1


From c7faa275594bfe3ffd84be0622fb856753137cfe Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Fri, 9 Jan 2015 17:30:51 +0000
Subject: Make rubygems lorry ext work with new interface

---
 baserockimport/exts/rubygems.to_lorry | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/baserockimport/exts/rubygems.to_lorry b/baserockimport/exts/rubygems.to_lorry
index d5f1efa..0413204 100755
--- a/baserockimport/exts/rubygems.to_lorry
+++ b/baserockimport/exts/rubygems.to_lorry
@@ -76,9 +76,10 @@ class RubyGemLorryGenerator(ImportExtension):
             "Loaded %i known source URIs from local metadata.", len(self.known_source_uris))
 
     def process_args(self, args):
-        if len(args) != 1:
+        if len(args) not in [1, 2]:
             raise ImportException(
-                'Please call me with the name of a RubyGem as an argument.')
+                'Please call me with the name of a RubyGem as an argument'
+                ' and optionally its version number (format: NAME [VERSION])')
 
         gem_name = args[0]
 
-- 
cgit v1.2.1


From 9e464b41e3255c95b6d72b8eb5f682c44171a536 Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Fri, 9 Jan 2015 18:17:25 +0000
Subject: Make npm lorry ext work with new interface

---
 baserockimport/exts/npm.to_lorry | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/baserockimport/exts/npm.to_lorry b/baserockimport/exts/npm.to_lorry
index ba0f442..65d8712 100755
--- a/baserockimport/exts/npm.to_lorry
+++ b/baserockimport/exts/npm.to_lorry
@@ -20,11 +20,13 @@ npm = require("npm");
 base = require("./importer_base");
 
 npm.load(function(er, npm) {
-  if (process.argv.length === 3)
-    packageName = process.argv[2];
-  else
-    throw ("Error! Too many command line arguments! Usage: " +
-           "./npm.to_lorry PACKAGENAME");
+
+  if (! (process.argv.length == 3 || process.argv.length == 4)) {
+    throw ("Usage: ./npm.to_lorry PACKAGENAME [VERSION]");
+  }
+
+  packageName = process.argv[2];
+
   if (er) throw er;
 
   npm.commands.view([packageName], "silent", getRepo);
-- 
cgit v1.2.1


From 8d0448d044f20a148f98700a691a83c95308a8e8 Mon Sep 17 00:00:00 2001
From: Richard Ipsum <richard.ipsum@codethink.co.uk>
Date: Tue, 13 Jan 2015 16:56:34 +0000
Subject: Update tests for python.to_lorry

---
 baserockimport/exts/python_lorry_tests.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/baserockimport/exts/python_lorry_tests.py b/baserockimport/exts/python_lorry_tests.py
index 12ef564..be4bcac 100755
--- a/baserockimport/exts/python_lorry_tests.py
+++ b/baserockimport/exts/python_lorry_tests.py
@@ -21,6 +21,8 @@ import json
 
 import unittest
 
+LORRY_PREFIX = 'python'
+
 class Tests(unittest.TestCase):
 
     def test_make_tarball_lorry(self):
@@ -35,18 +37,19 @@ class Tests(unittest.TestCase):
             return 'http://foobar/baz.%s' % extension
 
         def get_tarball_lorry_url(name, lorry_json):
-            return json.loads(lorry_json)['python-packages/'
+            return json.loads(lorry_json)[LORRY_PREFIX + '/'
                                             + name + '-tarball']['url']
 
         def get_tarball_lorry_compression(name, lorry_json):
-            return json.loads(lorry_json)['python-packages/'
+            return json.loads(lorry_json)[LORRY_PREFIX + '/'
                                             + name + '-tarball']['compression']
 
         fake_package_name = 'name'
         urls = [(make_url(ext), ext) for ext in valid_extensions]
 
         for (url, ext) in urls:
-            lorry_json = python_lorry.make_tarball_lorry('name', url)
+            lorry_json = python_lorry.make_tarball_lorry(LORRY_PREFIX,
+                                                         'name', url)
             print lorry_json
 
             tarball_url = get_tarball_lorry_url(fake_package_name, lorry_json)
@@ -61,7 +64,7 @@ class Tests(unittest.TestCase):
             self.assertEqual(tarball_compression, valid_extensions[ext])
 
         url = 'http://foobar/baz.tar'
-        lorry_json = python_lorry.make_tarball_lorry('name', url)
+        lorry_json = python_lorry.make_tarball_lorry(LORRY_PREFIX, 'name', url)
         self.assertEqual(get_tarball_lorry_url(fake_package_name,
                                                lorry_json), url)
         self.assertTrue('compression' not in lorry_json)
-- 
cgit v1.2.1