From 516b7ae1a334c4b7175bb59eaab80f2b66d749eb Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Thu, 9 Oct 2014 18:27:58 +0100 Subject: Only run our stuff if we're called as main --- import/pip.to_lorry | 172 --------------------------------------------------- import/pip_lorry.py | 174 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 172 deletions(-) delete mode 100755 import/pip.to_lorry create mode 100755 import/pip_lorry.py diff --git a/import/pip.to_lorry b/import/pip.to_lorry deleted file mode 100755 index f5c9335..0000000 --- a/import/pip.to_lorry +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env python -# -# Create a Baserock .lorry file for a given Python package -# -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -from __future__ import print_function - -import subprocess -import requests -import json -import sys -import shutil -import tempfile -import xmlrpclib - -PYPI_URL = 'http://pypi.python.org/pypi' - -def warn(*args, **kwargs): - print('%s:' % sys.argv[0], *args, file=sys.stderr, **kwargs) - -def error(*args, **kwargs): - warn(*args, **kwargs) - sys.exit(1) - -def fetch_package_metadata(package_name): - try: - return requests.get('%s/%s/json' % (PYPI_URL, package_name)).json() - except Exception as e: - error("Couldn't fetch package metadata: ", e) - -def find_repo_type(url): - vcss = [('git', 'clone'), ('hg', 'clone'), - ('svn', 'checkout'), ('bzr', 'branch')] - - for (vcs, vcs_command) in vcss: - tempdir = tempfile.mkdtemp() - - p = subprocess.Popen([vcs, vcs_command, url], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, cwd=tempdir) - - p.wait() - shutil.rmtree(tempdir) - - if p.returncode == 0: - return vcs - - return None - -def get_compression(url): - bzip = 'bzip2' - gzip = 'gzip' - lzma = 'lzma' - - m = {'tar.gz': gzip, 'tgz': gzip, 'tar.Z': gzip, - 'tar.bz2': bzip, 'tbz2': bzip, - 'tar.lzma': lzma, 'tar.xz': lzma, 'tlz': lzma, 'txz': lzma} - - for x in [1, 2]: - ext = '.'.join(url.split('.')[-x:]) - if ext in m: return m[ext] - - return None - -# Assumption: url passed to this function must have a 'standard' tar extension -def make_tarball_lorry(name, url): - lorry = {'type': 'tarball', 'url': url} - compression = get_compression(url) - if compression: - lorry['compression'] = compression - - return json.dumps({name.lower() + "-tarball": lorry}, - indent=4, sort_keys=True) - -def ask_user(client, xs, fn, prompt='--> '): - for n, x in enumerate(xs, 1): - print('[%s]: %s' % (n, fn(x))) - print('') - - s = raw_input(prompt) - choice = int(s) if s.isdigit() else None - choice = choice - 1 if choice != None and choice <= len(xs) else None - - if choice == None: - print("Invalid choice", file=sys.stderr) - sys.exit(1) - - return choice - -def filter_urls(urls): - allowed_extensions = ['tar.gz', 'tgz', 'tar.Z', 'tar.bz2', 'tbz2', - 'tar.lzma', 'tar.xz', 'tlz', 'txz', 'tar'] - - def allowed_extension(url): - return ('.'.join(url['url'].split('.')[-2:]) in allowed_extensions - or url['url'].split('.')[-1:] in allowed_extensions) - - return filter(allowed_extension, urls) - -def generate_tarball_lorry(package_name): - try: - client = xmlrpclib.ServerProxy(PYPI_URL) - releases = client.package_releases(package_name) - except Exception as e: - error("Couldn't fetch release data:", e) - - if len(releases) == 0: - error("Couldn't find any releases for package %s" % package_name) - - def get_description(release): - return client.release_data(package_name, - release)['name'] + ' ' + release - - choice = (ask_user(client, releases, - get_description, prompt='Select release: ') - if len(releases) > 1 else 0) - release_version = releases[choice] - - print('Fetching urls for package %s with version %s' - % (package_name, release_version)) - - try: - urls = client.release_urls(package_name, release_version) - except Exception as e: - error("Couldn't fetch release urls:", e) - - tarball_urls = filter_urls(urls) - - if len(tarball_urls) > 0: - urls = tarball_urls - elif len(urls) > 0: - warn("None of these urls look like tarballs:") - for url in urls: - warn("\t%s", url) - error("Cannot proceed") - else: - error("Couldn't find any download urls for package") - - choice = (ask_user(client, urls, lambda url: url['url'], - prompt='Select url: ') if len(urls) > 1 else 0) - url = urls[choice]['url'] - - return make_tarball_lorry(package_name, url) - -def str_repo_lorry(package_name, repo_type, url): - return json.dumps({package_name.lower(): {'type': repo_type, 'url': url}}, - indent=4, sort_keys=True) - -if len(sys.argv) != 2: - error('usage: %s python_package' % sys.argv[0]) - -package_name = sys.argv[1] -metadata = fetch_package_metadata(package_name) -info = metadata['info'] - -repo_type = find_repo_type(info['home_page']) if 'home_page' in info else None - -print(str_repo_lorry(package_name, repo_type, info['home_page']) - if repo_type else generate_tarball_lorry(package_name)) diff --git a/import/pip_lorry.py b/import/pip_lorry.py new file mode 100755 index 0000000..580e94a --- /dev/null +++ b/import/pip_lorry.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# +# Create a Baserock .lorry file for a given Python package +# +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import print_function + +import subprocess +import requests +import json +import sys +import shutil +import tempfile +import xmlrpclib + +PYPI_URL = 'http://pypi.python.org/pypi' + +def warn(*args, **kwargs): + print('%s:' % sys.argv[0], *args, file=sys.stderr, **kwargs) + +def error(*args, **kwargs): + warn(*args, **kwargs) + sys.exit(1) + +def fetch_package_metadata(package_name): + try: + return requests.get('%s/%s/json' % (PYPI_URL, package_name)).json() + except Exception as e: + error("Couldn't fetch package metadata: ", e) + +def find_repo_type(url): + vcss = [('git', 'clone'), ('hg', 'clone'), + ('svn', 'checkout'), ('bzr', 'branch')] + + for (vcs, vcs_command) in vcss: + tempdir = tempfile.mkdtemp() + + p = subprocess.Popen([vcs, vcs_command, url], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, cwd=tempdir) + + p.wait() + shutil.rmtree(tempdir) + + if p.returncode == 0: + return vcs + + return None + +def get_compression(url): + bzip = 'bzip2' + gzip = 'gzip' + lzma = 'lzma' + + m = {'tar.gz': gzip, 'tgz': gzip, 'tar.Z': gzip, + 'tar.bz2': bzip, 'tbz2': bzip, + 'tar.lzma': lzma, 'tar.xz': lzma, 'tlz': lzma, 'txz': lzma} + + for x in [1, 2]: + ext = '.'.join(url.split('.')[-x:]) + if ext in m: return m[ext] + + return None + +# Assumption: url passed to this function must have a 'standard' tar extension +def make_tarball_lorry(name, url): + lorry = {'type': 'tarball', 'url': url} + compression = get_compression(url) + if compression: + lorry['compression'] = compression + + return json.dumps({name.lower() + "-tarball": lorry}, + indent=4, sort_keys=True) + +def ask_user(client, xs, fn, prompt='--> '): + for n, x in enumerate(xs, 1): + print('[%s]: %s' % (n, fn(x))) + print('') + + s = raw_input(prompt) + choice = int(s) if s.isdigit() else None + choice = choice - 1 if choice != None and choice <= len(xs) else None + + if choice == None: + print("Invalid choice", file=sys.stderr) + sys.exit(1) + + return choice + +def filter_urls(urls): + allowed_extensions = ['tar.gz', 'tgz', 'tar.Z', 'tar.bz2', 'tbz2', + 'tar.lzma', 'tar.xz', 'tlz', 'txz', 'tar'] + + def allowed_extension(url): + return ('.'.join(url['url'].split('.')[-2:]) in allowed_extensions + or url['url'].split('.')[-1:] in allowed_extensions) + + return filter(allowed_extension, urls) + +def generate_tarball_lorry(package_name): + try: + client = xmlrpclib.ServerProxy(PYPI_URL) + releases = client.package_releases(package_name) + except Exception as e: + error("Couldn't fetch release data:", e) + + if len(releases) == 0: + error("Couldn't find any releases for package %s" % package_name) + + def get_description(release): + return client.release_data(package_name, + release)['name'] + ' ' + release + + choice = (ask_user(client, releases, + get_description, prompt='Select release: ') + if len(releases) > 1 else 0) + release_version = releases[choice] + + print('Fetching urls for package %s with version %s' + % (package_name, release_version)) + + try: + urls = client.release_urls(package_name, release_version) + except Exception as e: + error("Couldn't fetch release urls:", e) + + tarball_urls = filter_urls(urls) + + if len(tarball_urls) > 0: + urls = tarball_urls + elif len(urls) > 0: + warn("None of these urls look like tarballs:") + for url in urls: + warn("\t%s", url) + error("Cannot proceed") + else: + error("Couldn't find any download urls for package") + + choice = (ask_user(client, urls, lambda url: url['url'], + prompt='Select url: ') if len(urls) > 1 else 0) + url = urls[choice]['url'] + + return make_tarball_lorry(package_name, url) + +def str_repo_lorry(package_name, repo_type, url): + return json.dumps({package_name.lower(): {'type': repo_type, 'url': url}}, + indent=4, sort_keys=True) + +if __name__ == '__main__': + if len(sys.argv) != 2: + error('usage: %s python_package' % sys.argv[0]) + + package_name = sys.argv[1] + metadata = fetch_package_metadata(package_name) + info = metadata['info'] + + repo_type = (find_repo_type(info['home_page']) + if 'home_page' in info else None) + + print(str_repo_lorry(package_name, repo_type, info['home_page']) + if repo_type else generate_tarball_lorry(package_name)) -- cgit v1.2.1