diff options
Diffstat (limited to 'baserockimport/exts/python.to_lorry')
| -rwxr-xr-x | baserockimport/exts/python.to_lorry | 117 |
1 files changed, 68 insertions, 49 deletions
diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry index accc9dc..db33e5f 100755 --- a/baserockimport/exts/python.to_lorry +++ b/baserockimport/exts/python.to_lorry @@ -28,7 +28,7 @@ import shutil import tempfile import xmlrpclib import logging -import select +import yaml import pkg_resources @@ -47,6 +47,8 @@ def fetch_package_metadata(package_name): def find_repo_type(url): + debug_vcss = False + # Don't bother with detection if we can't get a 200 OK logging.debug("Getting '%s' ..." % url) @@ -79,7 +81,8 @@ def find_repo_type(url): if line == '': break - logging.debug(line.rstrip('\n')) + if debug_vcss: + logging.debug(line.rstrip('\n')) p.wait() # even with eof on both streams, we still wait @@ -109,11 +112,13 @@ def get_compression(url): return None # Assumption: url passed to this function must have a 'standard' tar extension -def make_tarball_lorry(package_name, url): - # TODO: this prefix probably shouldn't be hardcoded here either - name = 'python-packages/%s' % package_name.lower() +def make_tarball_lorry(lorry_prefix, package_name, url): + name = '%s/%s' % (lorry_prefix, package_name) - lorry = {'type': 'tarball', 'url': url} + # TODO: shouldn't have 'x-products-python' field hardcoded here either + lorry = {'type': 'tarball', + 'url': url, + 'x-products-python': [package_name]} compression = get_compression(url) if compression: lorry['compression'] = compression @@ -130,36 +135,35 @@ def filter_urls(urls): return filter(allowed_extension, urls) -def get_releases(client, requirement): +def get_releases(client, package_name): try: - releases = client.package_releases(requirement.project_name) + releases = client.package_releases(package_name) except Exception as e: error("Couldn't fetch release data:", e) return releases -def generate_tarball_lorry(client, requirement): - releases = get_releases(client, requirement) +def generate_tarball_lorry(lorry_prefix, client, package_name, version=None): + releases = get_releases(client, package_name) if len(releases) == 0: - error("Couldn't find any releases for package %s" - % requirement.project_name) + error("Couldn't find any releases for package %s" % package_name) - releases = [v for v in releases if specs_satisfied(v, requirement.specs)] + logging.debug('Found releases: %s', str(releases)) - if len(releases) == 0: - error("Couldn't find any releases of %s" - " that satisfy version constraints: %s" - % (requirement.project_name, requirement.specs)) + # Use latest release if no version specified + version = version or releases[0] - release_version = releases[0] + if version not in releases: + error("Couldn't find any releases of %s with version: %s" + % (package_name, version)) - logging.debug('Fetching urls for package %s with version %s' - % (requirement.project_name, release_version)) + logging.debug('Fetching urls for package %s with version %s', + package_name, version) try: # Get a list of dicts, the dicts contain the urls. - urls = client.release_urls(requirement.project_name, release_version) + urls = client.release_urls(package_name, version) except Exception as e: error("Couldn't fetch release urls:", e) @@ -173,48 +177,63 @@ def generate_tarball_lorry(client, requirement): warn("\t%s" % url['url']) error("Cannot proceed") else: - error("Couldn't find any download urls for package %s" - % requirement.project_name) + error("Couldn't find any download urls for package %s" % package_name) url = urls[0]['url'] - return make_tarball_lorry(requirement.project_name, url) + return make_tarball_lorry(lorry_prefix, package_name, url) -def str_repo_lorry(package_name, repo_type, url): - # TODO: this prefix probably shouldn't be hardcoded here - name = 'python-packages/%s' % package_name.lower() +def str_repo_lorry(lorry_prefix, package_name, repo_type, url): + name = '%s/%s' % (lorry_prefix, package_name) - return json.dumps({name: {'type': repo_type, 'url': url}}, + # TODO: this products field 'x-products-python' + # probably shouldn't be hardcoded here + return json.dumps({name: {'type': repo_type, + 'url': url, + 'x-products-python': [package_name]}}, indent=4, sort_keys=True) -def main(): - if len(sys.argv) != 2: - # TODO explain the format of python requirements - # warn the user that they probably want to quote their arg - # > < will be interpreted as redirection by the shell - print('usage: %s requirement' % sys.argv[0], file=sys.stderr) - sys.exit(1) +class PythonLorryExtension(ImportExtension): + + def __init__(self): + super(PythonLorryExtension, self).__init__() + + def run(self): + if len(sys.argv) not in [2, 3]: + print('usage: %s NAME [VERSION]' % sys.argv[0], file=sys.stderr) + sys.exit(1) + + client = xmlrpclib.ServerProxy(PYPI_URL) + + package_name = sys.argv[1] + version = sys.argv[2] if len(sys.argv) == 3 else None - client = xmlrpclib.ServerProxy(PYPI_URL) + logging.debug('Looking for package: %s with version %s', + package_name, version) - req = pkg_resources.parse_requirements(sys.argv[1]).next() + with open(self.local_data_path('python.yaml')) as f: + lorry_prefix = yaml.load(f)['lorry-prefix'] - new_proj_name = name_or_closest(client, req.project_name) + new_proj_name = name_or_closest(client, package_name) - if new_proj_name == None: - error("Couldn't find any project with name '%s'" % req.project_name) + if new_proj_name == None: + error("Couldn't find any project with name '%s'" % package_name) - logging.debug('Treating %s as %s' % (req.project_name, new_proj_name)) - req.project_name = new_proj_name + logging.debug('Treating %s as %s' % (package_name, new_proj_name)) + package_name = new_proj_name - metadata = fetch_package_metadata(req.project_name) - info = metadata['info'] + metadata = fetch_package_metadata(package_name) + info = metadata['info'] - repo_type = (find_repo_type(info['home_page']) - if 'home_page' in info else None) + repo_type = (find_repo_type(info['home_page']) + if 'home_page' in info else None) - print(str_repo_lorry(req.project_name, repo_type, info['home_page']) - if repo_type else generate_tarball_lorry(client, req)) + if repo_type: + print(str_repo_lorry(lorry_prefix, package_name, + repo_type, info['home_page'])) + else: + print(generate_tarball_lorry(lorry_prefix, client, + package_name, version)) if __name__ == '__main__': - PythonExtension().run() + PythonLorryExtension().run() |