summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRichard Ipsum <richard.ipsum@codethink.co.uk>2015-01-14 16:06:53 +0000
committerRichard Ipsum <richard.ipsum@codethink.co.uk>2015-01-14 16:06:53 +0000
commit8f32749511f4cab0ac1c6d74025fc1b21578996c (patch)
treef6cc78d83f1a7b5c330b8a54732c2ee84191f34a
parentb092a66c21cb977a872d2b45d8edec31d96eeb28 (diff)
parent8d0448d044f20a148f98700a691a83c95308a8e8 (diff)
downloadimport-8f32749511f4cab0ac1c6d74025fc1b21578996c.tar.gz
Merge branch 'baserock/richardipsum/fix-lorry-bug-3'
Reviewed by: Sam Thursfield <sam.thursfield@codethink.co.uk>
-rw-r--r--baserockimport/app.py7
-rw-r--r--baserockimport/data/python.yaml2
-rw-r--r--baserockimport/exts/importer_python_common.py20
-rwxr-xr-xbaserockimport/exts/npm.to_lorry12
-rwxr-xr-xbaserockimport/exts/python.find_deps45
-rwxr-xr-xbaserockimport/exts/python.to_lorry117
-rwxr-xr-xbaserockimport/exts/python_lorry_tests.py11
-rwxr-xr-xbaserockimport/exts/rubygems.to_lorry5
-rw-r--r--baserockimport/lorryset.py5
-rw-r--r--baserockimport/mainloop.py23
10 files changed, 149 insertions, 98 deletions
diff --git a/baserockimport/app.py b/baserockimport/app.py
index cd3203e..0b190e5 100644
--- a/baserockimport/app.py
+++ b/baserockimport/app.py
@@ -206,6 +206,10 @@ class BaserockImportApplication(cliapp.Application):
def import_python(self, args):
'''Import one or more python packages.'''
+
+ def comp(x, y):
+ return x.replace('-', '_').lower() == y.replace('-', '_').lower()
+
if len(args) < 1 or len(args) > 2:
raise cliapp.AppException(
'Please pass the name of the python package on the commandline.')
@@ -218,5 +222,6 @@ class BaserockImportApplication(cliapp.Application):
goal_kind='python',
goal_name=package_name,
goal_version=package_version)
- loop.enable_importer('python', strata=['strata/core.morph'])
+ loop.enable_importer('python', strata=['strata/core.morph'],
+ package_comp_callback=comp)
loop.run()
diff --git a/baserockimport/data/python.yaml b/baserockimport/data/python.yaml
new file mode 100644
index 0000000..6836714
--- /dev/null
+++ b/baserockimport/data/python.yaml
@@ -0,0 +1,2 @@
+---
+lorry-prefix: python
diff --git a/baserockimport/exts/importer_python_common.py b/baserockimport/exts/importer_python_common.py
index b2e7c51..eedc81b 100644
--- a/baserockimport/exts/importer_python_common.py
+++ b/baserockimport/exts/importer_python_common.py
@@ -23,6 +23,15 @@ from importer_base import ImportExtension
PYPI_URL = 'http://pypi.python.org/pypi'
+def get_releases(client, package_name):
+ try:
+ # True here just means show hidden releases
+ releases = client.package_releases(package_name, True)
+ except Exception as e:
+ error("Couldn't fetch release data:", e)
+
+ return releases
+
def warn(*args, **kwargs):
print('%s:' % sys.argv[0], *args, file=sys.stderr, **kwargs)
@@ -60,10 +69,11 @@ def name_or_closest(client, package_name):
#
# so look for both the hyphenated version that is passed to this function
# and the underscored version.
+ package_name = package_name.replace('_', '-')
underscored_package_name = package_name.replace('-', '_')
for name in [package_name, underscored_package_name]:
- results = client.package_releases(name)
+ results = get_releases(client, name)
if len(results) > 0:
logging.debug('Found package %s' % name)
@@ -88,12 +98,4 @@ def name_or_closest(client, package_name):
return results[0]['name'] if len(results) > 0 else None
-# We subclass the ImportExtension to setup the logger,
-# so that we can send logs to the import tool's log
-class PythonExtension(ImportExtension):
- def __init__(self):
- super(PythonExtension, self).__init__()
- def process_args(self, _):
- import __main__
- __main__.main()
diff --git a/baserockimport/exts/npm.to_lorry b/baserockimport/exts/npm.to_lorry
index ba0f442..65d8712 100755
--- a/baserockimport/exts/npm.to_lorry
+++ b/baserockimport/exts/npm.to_lorry
@@ -20,11 +20,13 @@ npm = require("npm");
base = require("./importer_base");
npm.load(function(er, npm) {
- if (process.argv.length === 3)
- packageName = process.argv[2];
- else
- throw ("Error! Too many command line arguments! Usage: " +
- "./npm.to_lorry PACKAGENAME");
+
+ if (! (process.argv.length == 3 || process.argv.length == 4)) {
+ throw ("Usage: ./npm.to_lorry PACKAGENAME [VERSION]");
+ }
+
+ packageName = process.argv[2];
+
if (er) throw er;
npm.commands.view([packageName], "silent", getRepo);
diff --git a/baserockimport/exts/python.find_deps b/baserockimport/exts/python.find_deps
index b9791ef..a73c457 100755
--- a/baserockimport/exts/python.find_deps
+++ b/baserockimport/exts/python.find_deps
@@ -189,7 +189,7 @@ def resolve_versions(specsets):
logging.debug("Treating %s as %s" % (proj_name, new_proj_name))
proj_name = new_proj_name
- releases = client.package_releases(proj_name)
+ releases = get_releases(client, proj_name)
logging.debug('Found %d releases of %s: %s'
% (len(releases), proj_name, releases))
@@ -327,31 +327,36 @@ def find_runtime_deps(source, name, version=None, use_requirements_file=False):
return runtime_deps
-def main():
- if len(sys.argv) not in [3, 4]:
- print('usage: %s PACKAGE_SOURCE_DIR NAME [VERSION]' % sys.argv[0])
- sys.exit(1)
+class PythonFindDepsExtension(ImportExtension):
- logging.debug('%s: sys.argv[1:]: %s' % (sys.argv[0], sys.argv[1:]))
- source, name = sys.argv[1:3]
- version = sys.argv[3] if len(sys.argv) == 4 else None
+ def __init__(self):
+ super(PythonFindDepsExtension, self).__init__()
- client = xmlrpclib.ServerProxy(PYPI_URL)
- new_name = name_or_closest(client, name)
+ def run(self):
+ if len(sys.argv) not in [3, 4]:
+ print('usage: %s PACKAGE_SOURCE_DIR NAME [VERSION]' % sys.argv[0])
+ sys.exit(1)
- if new_name == None:
- error("Couldn't find any project with name '%s'" % name)
+ logging.debug('%s: sys.argv[1:]: %s' % (sys.argv[0], sys.argv[1:]))
+ source, name = sys.argv[1:3]
+ version = sys.argv[3] if len(sys.argv) == 4 else None
- logging.debug('Treating %s as %s' % (name, new_name))
- name = new_name
+ client = xmlrpclib.ServerProxy(PYPI_URL)
+ new_name = name_or_closest(client, name)
+
+ if new_name == None:
+ error("Couldn't find any project with name '%s'" % name)
+
+ logging.debug('Treating %s as %s' % (name, new_name))
+ name = new_name
- deps = {}
- deps['build-dependencies'] = find_build_deps(source, name, version)
- deps['runtime-dependencies'] = find_runtime_deps(source, name, version)
+ deps = {}
+ deps['build-dependencies'] = find_build_deps(source, name, version)
+ deps['runtime-dependencies'] = find_runtime_deps(source, name, version)
- root = {'python': deps}
+ root = {'python': deps}
- print(json.dumps(root))
+ print(json.dumps(root))
if __name__ == '__main__':
- PythonExtension().run()
+ PythonFindDepsExtension().run()
diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry
index accc9dc..db33e5f 100755
--- a/baserockimport/exts/python.to_lorry
+++ b/baserockimport/exts/python.to_lorry
@@ -28,7 +28,7 @@ import shutil
import tempfile
import xmlrpclib
import logging
-import select
+import yaml
import pkg_resources
@@ -47,6 +47,8 @@ def fetch_package_metadata(package_name):
def find_repo_type(url):
+ debug_vcss = False
+
# Don't bother with detection if we can't get a 200 OK
logging.debug("Getting '%s' ..." % url)
@@ -79,7 +81,8 @@ def find_repo_type(url):
if line == '':
break
- logging.debug(line.rstrip('\n'))
+ if debug_vcss:
+ logging.debug(line.rstrip('\n'))
p.wait() # even with eof on both streams, we still wait
@@ -109,11 +112,13 @@ def get_compression(url):
return None
# Assumption: url passed to this function must have a 'standard' tar extension
-def make_tarball_lorry(package_name, url):
- # TODO: this prefix probably shouldn't be hardcoded here either
- name = 'python-packages/%s' % package_name.lower()
+def make_tarball_lorry(lorry_prefix, package_name, url):
+ name = '%s/%s' % (lorry_prefix, package_name)
- lorry = {'type': 'tarball', 'url': url}
+ # TODO: shouldn't have 'x-products-python' field hardcoded here either
+ lorry = {'type': 'tarball',
+ 'url': url,
+ 'x-products-python': [package_name]}
compression = get_compression(url)
if compression:
lorry['compression'] = compression
@@ -130,36 +135,35 @@ def filter_urls(urls):
return filter(allowed_extension, urls)
-def get_releases(client, requirement):
+def get_releases(client, package_name):
try:
- releases = client.package_releases(requirement.project_name)
+ releases = client.package_releases(package_name)
except Exception as e:
error("Couldn't fetch release data:", e)
return releases
-def generate_tarball_lorry(client, requirement):
- releases = get_releases(client, requirement)
+def generate_tarball_lorry(lorry_prefix, client, package_name, version=None):
+ releases = get_releases(client, package_name)
if len(releases) == 0:
- error("Couldn't find any releases for package %s"
- % requirement.project_name)
+ error("Couldn't find any releases for package %s" % package_name)
- releases = [v for v in releases if specs_satisfied(v, requirement.specs)]
+ logging.debug('Found releases: %s', str(releases))
- if len(releases) == 0:
- error("Couldn't find any releases of %s"
- " that satisfy version constraints: %s"
- % (requirement.project_name, requirement.specs))
+ # Use latest release if no version specified
+ version = version or releases[0]
- release_version = releases[0]
+ if version not in releases:
+ error("Couldn't find any releases of %s with version: %s"
+ % (package_name, version))
- logging.debug('Fetching urls for package %s with version %s'
- % (requirement.project_name, release_version))
+ logging.debug('Fetching urls for package %s with version %s',
+ package_name, version)
try:
# Get a list of dicts, the dicts contain the urls.
- urls = client.release_urls(requirement.project_name, release_version)
+ urls = client.release_urls(package_name, version)
except Exception as e:
error("Couldn't fetch release urls:", e)
@@ -173,48 +177,63 @@ def generate_tarball_lorry(client, requirement):
warn("\t%s" % url['url'])
error("Cannot proceed")
else:
- error("Couldn't find any download urls for package %s"
- % requirement.project_name)
+ error("Couldn't find any download urls for package %s" % package_name)
url = urls[0]['url']
- return make_tarball_lorry(requirement.project_name, url)
+ return make_tarball_lorry(lorry_prefix, package_name, url)
-def str_repo_lorry(package_name, repo_type, url):
- # TODO: this prefix probably shouldn't be hardcoded here
- name = 'python-packages/%s' % package_name.lower()
+def str_repo_lorry(lorry_prefix, package_name, repo_type, url):
+ name = '%s/%s' % (lorry_prefix, package_name)
- return json.dumps({name: {'type': repo_type, 'url': url}},
+ # TODO: this products field 'x-products-python'
+ # probably shouldn't be hardcoded here
+ return json.dumps({name: {'type': repo_type,
+ 'url': url,
+ 'x-products-python': [package_name]}},
indent=4, sort_keys=True)
-def main():
- if len(sys.argv) != 2:
- # TODO explain the format of python requirements
- # warn the user that they probably want to quote their arg
- # > < will be interpreted as redirection by the shell
- print('usage: %s requirement' % sys.argv[0], file=sys.stderr)
- sys.exit(1)
+class PythonLorryExtension(ImportExtension):
+
+ def __init__(self):
+ super(PythonLorryExtension, self).__init__()
+
+ def run(self):
+ if len(sys.argv) not in [2, 3]:
+ print('usage: %s NAME [VERSION]' % sys.argv[0], file=sys.stderr)
+ sys.exit(1)
+
+ client = xmlrpclib.ServerProxy(PYPI_URL)
+
+ package_name = sys.argv[1]
+ version = sys.argv[2] if len(sys.argv) == 3 else None
- client = xmlrpclib.ServerProxy(PYPI_URL)
+ logging.debug('Looking for package: %s with version %s',
+ package_name, version)
- req = pkg_resources.parse_requirements(sys.argv[1]).next()
+ with open(self.local_data_path('python.yaml')) as f:
+ lorry_prefix = yaml.load(f)['lorry-prefix']
- new_proj_name = name_or_closest(client, req.project_name)
+ new_proj_name = name_or_closest(client, package_name)
- if new_proj_name == None:
- error("Couldn't find any project with name '%s'" % req.project_name)
+ if new_proj_name == None:
+ error("Couldn't find any project with name '%s'" % package_name)
- logging.debug('Treating %s as %s' % (req.project_name, new_proj_name))
- req.project_name = new_proj_name
+ logging.debug('Treating %s as %s' % (package_name, new_proj_name))
+ package_name = new_proj_name
- metadata = fetch_package_metadata(req.project_name)
- info = metadata['info']
+ metadata = fetch_package_metadata(package_name)
+ info = metadata['info']
- repo_type = (find_repo_type(info['home_page'])
- if 'home_page' in info else None)
+ repo_type = (find_repo_type(info['home_page'])
+ if 'home_page' in info else None)
- print(str_repo_lorry(req.project_name, repo_type, info['home_page'])
- if repo_type else generate_tarball_lorry(client, req))
+ if repo_type:
+ print(str_repo_lorry(lorry_prefix, package_name,
+ repo_type, info['home_page']))
+ else:
+ print(generate_tarball_lorry(lorry_prefix, client,
+ package_name, version))
if __name__ == '__main__':
- PythonExtension().run()
+ PythonLorryExtension().run()
diff --git a/baserockimport/exts/python_lorry_tests.py b/baserockimport/exts/python_lorry_tests.py
index 12ef564..be4bcac 100755
--- a/baserockimport/exts/python_lorry_tests.py
+++ b/baserockimport/exts/python_lorry_tests.py
@@ -21,6 +21,8 @@ import json
import unittest
+LORRY_PREFIX = 'python'
+
class Tests(unittest.TestCase):
def test_make_tarball_lorry(self):
@@ -35,18 +37,19 @@ class Tests(unittest.TestCase):
return 'http://foobar/baz.%s' % extension
def get_tarball_lorry_url(name, lorry_json):
- return json.loads(lorry_json)['python-packages/'
+ return json.loads(lorry_json)[LORRY_PREFIX + '/'
+ name + '-tarball']['url']
def get_tarball_lorry_compression(name, lorry_json):
- return json.loads(lorry_json)['python-packages/'
+ return json.loads(lorry_json)[LORRY_PREFIX + '/'
+ name + '-tarball']['compression']
fake_package_name = 'name'
urls = [(make_url(ext), ext) for ext in valid_extensions]
for (url, ext) in urls:
- lorry_json = python_lorry.make_tarball_lorry('name', url)
+ lorry_json = python_lorry.make_tarball_lorry(LORRY_PREFIX,
+ 'name', url)
print lorry_json
tarball_url = get_tarball_lorry_url(fake_package_name, lorry_json)
@@ -61,7 +64,7 @@ class Tests(unittest.TestCase):
self.assertEqual(tarball_compression, valid_extensions[ext])
url = 'http://foobar/baz.tar'
- lorry_json = python_lorry.make_tarball_lorry('name', url)
+ lorry_json = python_lorry.make_tarball_lorry(LORRY_PREFIX, 'name', url)
self.assertEqual(get_tarball_lorry_url(fake_package_name,
lorry_json), url)
self.assertTrue('compression' not in lorry_json)
diff --git a/baserockimport/exts/rubygems.to_lorry b/baserockimport/exts/rubygems.to_lorry
index d5f1efa..0413204 100755
--- a/baserockimport/exts/rubygems.to_lorry
+++ b/baserockimport/exts/rubygems.to_lorry
@@ -76,9 +76,10 @@ class RubyGemLorryGenerator(ImportExtension):
"Loaded %i known source URIs from local metadata.", len(self.known_source_uris))
def process_args(self, args):
- if len(args) != 1:
+ if len(args) not in [1, 2]:
raise ImportException(
- 'Please call me with the name of a RubyGem as an argument.')
+ 'Please call me with the name of a RubyGem as an argument'
+ ' and optionally its version number (format: NAME [VERSION])')
gem_name = args[0]
diff --git a/baserockimport/lorryset.py b/baserockimport/lorryset.py
index 8cc73af..f252b9f 100644
--- a/baserockimport/lorryset.py
+++ b/baserockimport/lorryset.py
@@ -107,7 +107,7 @@ class LorrySet(object):
'''Return the lorry entry for the named project.'''
return {name: self.data[name]}
- def find_lorry_for_package(self, kind, package_name):
+ def find_lorry_for_package(self, kind, package_name, comp):
'''Find the lorry entry for a given foreign package, or return None.
This makes use of an extension to the .lorry format made by the
@@ -116,11 +116,12 @@ class LorrySet(object):
named $KIND.
'''
+
key = 'x-products-%s' % kind
for name, lorry in self.data.iteritems():
products = lorry.get(key, [])
for entry in products:
- if entry == package_name:
+ if comp(entry, package_name):
return {name: lorry}
return None
diff --git a/baserockimport/mainloop.py b/baserockimport/mainloop.py
index 057ab98..8a7ae26 100644
--- a/baserockimport/mainloop.py
+++ b/baserockimport/mainloop.py
@@ -202,7 +202,7 @@ class ImportLoop(object):
# 1. Make the source code available.
- lorry = self._find_or_create_lorry_file(kind, name)
+ lorry = self._find_or_create_lorry_file(kind, name, version)
source_repo, url = self._fetch_or_update_source(lorry)
checked_out_version, ref = self._checkout_source_version_for_package(
@@ -288,15 +288,22 @@ class ImportLoop(object):
dep_package.set_is_build_dep(True)
processed.add_edge(dep_package, current_item)
- def _find_or_create_lorry_file(self, kind, name):
+ def _find_or_create_lorry_file(self, kind, name, version):
# Note that the lorry file may already exist for 'name', but lorry
# files are named for project name rather than package name. In this
# case we will generate the lorry, and try to add it to the set, at
# which point LorrySet will notice the existing one and merge the two.
- lorry = self.lorry_set.find_lorry_for_package(kind, name)
+ comp = None
+
+ if 'package_comp_callback' in self.importers[kind]['kwargs']:
+ comp = self.importers[kind]['kwargs']['package_comp_callback']
+ else:
+ comp = lambda x, y: x == y
+
+ lorry = self.lorry_set.find_lorry_for_package(kind, name, comp)
if lorry is None:
- lorry = self._generate_lorry_for_package(kind, name)
+ lorry = self._generate_lorry_for_package(kind, name, version)
if len(lorry) != 1:
raise Exception(
@@ -323,14 +330,18 @@ class ImportLoop(object):
return lorry
- def _generate_lorry_for_package(self, kind, name):
+ def _generate_lorry_for_package(self, kind, name, version):
tool = '%s.to_lorry' % kind
if kind not in self.importers:
raise Exception('Importer for %s was not enabled.' % kind)
extra_args = self.importers[kind]['extra_args']
self.app.status(
'%s: calling %s to generate lorry', name, tool)
- lorry_text = run_extension(tool, extra_args + [name])
+
+ args = extra_args + [name]
+ if version != 'master':
+ args.append(version)
+ lorry_text = run_extension(tool, args)
try:
lorry = json.loads(lorry_text)
except ValueError: