From 0553ce72422eb55ba6389037df3290b30142dd60 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Thu, 8 Jan 2015 11:30:48 +0000 Subject: Read lorry prefix from yaml file This adds PythonLorryExtension class to python.to_lorry to run the extension in a more conventional way. Previously the PythonExtension class was used to execute any of the extensions (it would call the extensions main() function). We move away from this so the extension can access useful methods, such as local_data_path(), that are provided by the ImportExtension class. This also removes use of pkg_resources.parse_requirement which is redundant. This also removes the unused import of select. --- baserockimport/exts/python.to_lorry | 69 +++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 30 deletions(-) (limited to 'baserockimport/exts/python.to_lorry') diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry index accc9dc..06ee31d 100755 --- a/baserockimport/exts/python.to_lorry +++ b/baserockimport/exts/python.to_lorry @@ -28,7 +28,7 @@ import shutil import tempfile import xmlrpclib import logging -import select +import yaml import pkg_resources @@ -109,9 +109,8 @@ def get_compression(url): return None # Assumption: url passed to this function must have a 'standard' tar extension -def make_tarball_lorry(package_name, url): - # TODO: this prefix probably shouldn't be hardcoded here either - name = 'python-packages/%s' % package_name.lower() +def make_tarball_lorry(lorry_prefix, package_name, url): + name = '%s/%s' % (lorry_prefix, package_name.lower()) lorry = {'type': 'tarball', 'url': url} compression = get_compression(url) @@ -138,7 +137,7 @@ def get_releases(client, requirement): return releases -def generate_tarball_lorry(client, requirement): +def generate_tarball_lorry(lorry_prefix, client, requirement): releases = get_releases(client, requirement) if len(releases) == 0: @@ -178,43 +177,53 @@ def generate_tarball_lorry(client, requirement): url = urls[0]['url'] - return make_tarball_lorry(requirement.project_name, url) + return make_tarball_lorry(lorry_prefix, requirement.project_name, url) -def str_repo_lorry(package_name, repo_type, url): - # TODO: this prefix probably shouldn't be hardcoded here - name = 'python-packages/%s' % package_name.lower() +def str_repo_lorry(lorry_prefix, package_name, repo_type, url): + name = '%s/%s' % (lorry_prefix, package_name.lower()) return json.dumps({name: {'type': repo_type, 'url': url}}, indent=4, sort_keys=True) -def main(): - if len(sys.argv) != 2: - # TODO explain the format of python requirements - # warn the user that they probably want to quote their arg - # > < will be interpreted as redirection by the shell - print('usage: %s requirement' % sys.argv[0], file=sys.stderr) - sys.exit(1) +class PythonLorryExtension(ImportExtension): - client = xmlrpclib.ServerProxy(PYPI_URL) + def __init__(self): + super(PythonLorryExtension, self).__init__() - req = pkg_resources.parse_requirements(sys.argv[1]).next() + def run(self): + if len(sys.argv) != 2: + # TODO explain the format of python requirements + # warn the user that they probably want to quote their arg + # > < will be interpreted as redirection by the shell + print('usage: %s requirement' % sys.argv[0], file=sys.stderr) + sys.exit(1) - new_proj_name = name_or_closest(client, req.project_name) + client = xmlrpclib.ServerProxy(PYPI_URL) - if new_proj_name == None: - error("Couldn't find any project with name '%s'" % req.project_name) + req = pkg_resources.parse_requirements(sys.argv[1]).next() - logging.debug('Treating %s as %s' % (req.project_name, new_proj_name)) - req.project_name = new_proj_name + with open(self.local_data_path('python.yaml')) as f: + lorry_prefix = yaml.load(f)['lorry-prefix'] - metadata = fetch_package_metadata(req.project_name) - info = metadata['info'] + new_proj_name = name_or_closest(client, req.project_name) - repo_type = (find_repo_type(info['home_page']) - if 'home_page' in info else None) + if new_proj_name == None: + error("Couldn't find any project with name '%s'" % req.project_name) - print(str_repo_lorry(req.project_name, repo_type, info['home_page']) - if repo_type else generate_tarball_lorry(client, req)) + logging.debug('Treating %s as %s' % (req.project_name, new_proj_name)) + req.project_name = new_proj_name + + metadata = fetch_package_metadata(req.project_name) + info = metadata['info'] + + repo_type = (find_repo_type(info['home_page']) + if 'home_page' in info else None) + + if repo_type: + print(str_repo_lorry(lorry_prefix, req.project_name, + repo_type, info['home_page'])) + else: + print(generate_tarball_lorry(lorry_prefix, client, req)) if __name__ == '__main__': - PythonExtension().run() + PythonLorryExtension().run() -- cgit v1.2.1 From 827e9904332dbd17341648be81bcf2388d241ad4 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Thu, 8 Jan 2015 11:59:00 +0000 Subject: Make python.to_lorry output x-products-python field --- baserockimport/exts/python.to_lorry | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'baserockimport/exts/python.to_lorry') diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry index 06ee31d..fdbd04a 100755 --- a/baserockimport/exts/python.to_lorry +++ b/baserockimport/exts/python.to_lorry @@ -112,7 +112,10 @@ def get_compression(url): def make_tarball_lorry(lorry_prefix, package_name, url): name = '%s/%s' % (lorry_prefix, package_name.lower()) - lorry = {'type': 'tarball', 'url': url} + # TODO: shouldn't have 'x-products-python' field hardcoded here either + lorry = {'type': 'tarball', + 'url': url, + 'x-products-python': [package_name.lower()]} compression = get_compression(url) if compression: lorry['compression'] = compression @@ -182,7 +185,11 @@ def generate_tarball_lorry(lorry_prefix, client, requirement): def str_repo_lorry(lorry_prefix, package_name, repo_type, url): name = '%s/%s' % (lorry_prefix, package_name.lower()) - return json.dumps({name: {'type': repo_type, 'url': url}}, + # TODO: this products field 'x-products-python' + # probably shouldn't be hardcoded here + return json.dumps({name: {'type': repo_type, + 'url': url, + 'x-products-python': [package_name.lower()]}}, indent=4, sort_keys=True) class PythonLorryExtension(ImportExtension): -- cgit v1.2.1 From ec93dd99f5924b7741a4b3c2c1fab961a213d519 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Thu, 8 Jan 2015 15:34:42 +0000 Subject: Don't convert package names to lower case This is redundant. --- baserockimport/exts/python.to_lorry | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'baserockimport/exts/python.to_lorry') diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry index fdbd04a..3f0a463 100755 --- a/baserockimport/exts/python.to_lorry +++ b/baserockimport/exts/python.to_lorry @@ -110,12 +110,12 @@ def get_compression(url): # Assumption: url passed to this function must have a 'standard' tar extension def make_tarball_lorry(lorry_prefix, package_name, url): - name = '%s/%s' % (lorry_prefix, package_name.lower()) + name = '%s/%s' % (lorry_prefix, package_name) # TODO: shouldn't have 'x-products-python' field hardcoded here either lorry = {'type': 'tarball', 'url': url, - 'x-products-python': [package_name.lower()]} + 'x-products-python': [package_name]} compression = get_compression(url) if compression: lorry['compression'] = compression @@ -183,13 +183,13 @@ def generate_tarball_lorry(lorry_prefix, client, requirement): return make_tarball_lorry(lorry_prefix, requirement.project_name, url) def str_repo_lorry(lorry_prefix, package_name, repo_type, url): - name = '%s/%s' % (lorry_prefix, package_name.lower()) + name = '%s/%s' % (lorry_prefix, package_name) # TODO: this products field 'x-products-python' # probably shouldn't be hardcoded here return json.dumps({name: {'type': repo_type, 'url': url, - 'x-products-python': [package_name.lower()]}}, + 'x-products-python': [package_name]}}, indent=4, sort_keys=True) class PythonLorryExtension(ImportExtension): -- cgit v1.2.1 From 9b01e0431e33c7ffc3aa40bc01e026b37da4745f Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Thu, 8 Jan 2015 18:13:09 +0000 Subject: Add option for debugging vcss It's sometime useful to see the output of the vcs, but having this enabled all the time clutters the log --- baserockimport/exts/python.to_lorry | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'baserockimport/exts/python.to_lorry') diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry index 3f0a463..1452be8 100755 --- a/baserockimport/exts/python.to_lorry +++ b/baserockimport/exts/python.to_lorry @@ -47,6 +47,8 @@ def fetch_package_metadata(package_name): def find_repo_type(url): + debug_vcss = False + # Don't bother with detection if we can't get a 200 OK logging.debug("Getting '%s' ..." % url) @@ -79,7 +81,8 @@ def find_repo_type(url): if line == '': break - logging.debug(line.rstrip('\n')) + if debug_vcss: + logging.debug(line.rstrip('\n')) p.wait() # even with eof on both streams, we still wait -- cgit v1.2.1 From 5bcb22616579cff9707324063f9310dfe51d9108 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Thu, 8 Jan 2015 18:22:00 +0000 Subject: Add debug msg stating requirement --- baserockimport/exts/python.to_lorry | 2 ++ 1 file changed, 2 insertions(+) (limited to 'baserockimport/exts/python.to_lorry') diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry index 1452be8..cb4e4ec 100755 --- a/baserockimport/exts/python.to_lorry +++ b/baserockimport/exts/python.to_lorry @@ -223,6 +223,8 @@ class PythonLorryExtension(ImportExtension): logging.debug('Treating %s as %s' % (req.project_name, new_proj_name)) req.project_name = new_proj_name + logging.debug('Looking for requirement: %s', req) + metadata = fetch_package_metadata(req.project_name) info = metadata['info'] -- cgit v1.2.1 From 48de25fab7a0d88c1acf0a2a93ff1e98662c8ad1 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Mon, 12 Jan 2015 15:40:08 +0000 Subject: Fix python.to_lorry to use version passed on cmdline --- baserockimport/exts/python.to_lorry | 64 ++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 33 deletions(-) (limited to 'baserockimport/exts/python.to_lorry') diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry index cb4e4ec..db33e5f 100755 --- a/baserockimport/exts/python.to_lorry +++ b/baserockimport/exts/python.to_lorry @@ -135,36 +135,35 @@ def filter_urls(urls): return filter(allowed_extension, urls) -def get_releases(client, requirement): +def get_releases(client, package_name): try: - releases = client.package_releases(requirement.project_name) + releases = client.package_releases(package_name) except Exception as e: error("Couldn't fetch release data:", e) return releases -def generate_tarball_lorry(lorry_prefix, client, requirement): - releases = get_releases(client, requirement) +def generate_tarball_lorry(lorry_prefix, client, package_name, version=None): + releases = get_releases(client, package_name) if len(releases) == 0: - error("Couldn't find any releases for package %s" - % requirement.project_name) + error("Couldn't find any releases for package %s" % package_name) - releases = [v for v in releases if specs_satisfied(v, requirement.specs)] + logging.debug('Found releases: %s', str(releases)) - if len(releases) == 0: - error("Couldn't find any releases of %s" - " that satisfy version constraints: %s" - % (requirement.project_name, requirement.specs)) + # Use latest release if no version specified + version = version or releases[0] - release_version = releases[0] + if version not in releases: + error("Couldn't find any releases of %s with version: %s" + % (package_name, version)) - logging.debug('Fetching urls for package %s with version %s' - % (requirement.project_name, release_version)) + logging.debug('Fetching urls for package %s with version %s', + package_name, version) try: # Get a list of dicts, the dicts contain the urls. - urls = client.release_urls(requirement.project_name, release_version) + urls = client.release_urls(package_name, version) except Exception as e: error("Couldn't fetch release urls:", e) @@ -178,12 +177,11 @@ def generate_tarball_lorry(lorry_prefix, client, requirement): warn("\t%s" % url['url']) error("Cannot proceed") else: - error("Couldn't find any download urls for package %s" - % requirement.project_name) + error("Couldn't find any download urls for package %s" % package_name) url = urls[0]['url'] - return make_tarball_lorry(lorry_prefix, requirement.project_name, url) + return make_tarball_lorry(lorry_prefix, package_name, url) def str_repo_lorry(lorry_prefix, package_name, repo_type, url): name = '%s/%s' % (lorry_prefix, package_name) @@ -201,41 +199,41 @@ class PythonLorryExtension(ImportExtension): super(PythonLorryExtension, self).__init__() def run(self): - if len(sys.argv) != 2: - # TODO explain the format of python requirements - # warn the user that they probably want to quote their arg - # > < will be interpreted as redirection by the shell - print('usage: %s requirement' % sys.argv[0], file=sys.stderr) + if len(sys.argv) not in [2, 3]: + print('usage: %s NAME [VERSION]' % sys.argv[0], file=sys.stderr) sys.exit(1) client = xmlrpclib.ServerProxy(PYPI_URL) - req = pkg_resources.parse_requirements(sys.argv[1]).next() + package_name = sys.argv[1] + version = sys.argv[2] if len(sys.argv) == 3 else None + + logging.debug('Looking for package: %s with version %s', + package_name, version) with open(self.local_data_path('python.yaml')) as f: lorry_prefix = yaml.load(f)['lorry-prefix'] - new_proj_name = name_or_closest(client, req.project_name) + new_proj_name = name_or_closest(client, package_name) if new_proj_name == None: - error("Couldn't find any project with name '%s'" % req.project_name) - - logging.debug('Treating %s as %s' % (req.project_name, new_proj_name)) - req.project_name = new_proj_name + error("Couldn't find any project with name '%s'" % package_name) - logging.debug('Looking for requirement: %s', req) + logging.debug('Treating %s as %s' % (package_name, new_proj_name)) + package_name = new_proj_name - metadata = fetch_package_metadata(req.project_name) + metadata = fetch_package_metadata(package_name) info = metadata['info'] repo_type = (find_repo_type(info['home_page']) if 'home_page' in info else None) if repo_type: - print(str_repo_lorry(lorry_prefix, req.project_name, + print(str_repo_lorry(lorry_prefix, package_name, repo_type, info['home_page'])) else: - print(generate_tarball_lorry(lorry_prefix, client, req)) + print(generate_tarball_lorry(lorry_prefix, client, + package_name, version)) if __name__ == '__main__': PythonLorryExtension().run() -- cgit v1.2.1