From 2af8d66d35a748f3e5b83256054ca285e33c0e94 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Thu, 28 May 2015 15:40:09 +0100 Subject: Move common functions into a utils module These functions will also be useful for our forthcoming cpan extension Change-Id: I9df87dee09bbcf43dd0868f062fb873632f1f5ae --- baserockimport/exts/importer_python_common.py | 8 --- baserockimport/exts/python.to_lorry | 51 +++---------------- baserockimport/exts/python_lorry_tests.py | 16 +++--- baserockimport/exts/utils.py | 73 +++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 60 deletions(-) create mode 100644 baserockimport/exts/utils.py diff --git a/baserockimport/exts/importer_python_common.py b/baserockimport/exts/importer_python_common.py index eedc81b..42cc758 100644 --- a/baserockimport/exts/importer_python_common.py +++ b/baserockimport/exts/importer_python_common.py @@ -16,7 +16,6 @@ from __future__ import print_function -import sys import logging from importer_base import ImportExtension @@ -32,13 +31,6 @@ def get_releases(client, package_name): return releases -def warn(*args, **kwargs): - print('%s:' % sys.argv[0], *args, file=sys.stderr, **kwargs) - -def error(*args, **kwargs): - warn(*args, **kwargs) - sys.exit(1) - def specs_satisfied(version, specs): def mapping_error(op): # We parse ops with requirements-parser, so any invalid user input diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry index 8ee20eb..2c57a9b 100755 --- a/baserockimport/exts/python.to_lorry +++ b/baserockimport/exts/python.to_lorry @@ -33,6 +33,8 @@ import yaml import pkg_resources from importer_python_common import * +from utils import warn, error +import utils def fetch_package_metadata(package_name): try: @@ -96,35 +98,6 @@ def find_repo_type(url): return None -def get_compression(url): - bzip = 'bzip2' - gzip = 'gzip' - lzma = 'lzma' - - m = {'tar.gz': gzip, 'tgz': gzip, 'tar.Z': gzip, - 'tar.bz2': bzip, 'tbz2': bzip, - 'tar.lzma': lzma, 'tar.xz': lzma, 'tlz': lzma, 'txz': lzma} - - for x in [1, 2]: - ext = '.'.join(url.split('.')[-x:]) - if ext in m: return m[ext] - - return None - -# Assumption: url passed to this function must have a 'standard' tar extension -def make_tarball_lorry(lorry_prefix, package_name, url): - name = '%s/%s' % (lorry_prefix, package_name) - - # TODO: shouldn't have 'x-products-python' field hardcoded here either - lorry = {'type': 'tarball', - 'url': url, - 'x-products-python': [package_name]} - compression = get_compression(url) - if compression: - lorry['compression'] = compression - - return json.dumps({name + "-tarball": lorry}, indent=4, sort_keys=True) - def filter_urls(urls): allowed_extensions = ['tar.gz', 'tgz', 'tar.Z', 'tar.bz2', 'tbz2', 'tar.lzma', 'tar.xz', 'tlz', 'txz', 'tar'] @@ -181,20 +154,9 @@ def generate_tarball_lorry(lorry_prefix, client, package_name, version=None): url = urls[0]['url'] - return make_tarball_lorry(lorry_prefix, package_name, url) - -def str_repo_lorry(lorry_prefix, package_name, repo_type, url): - name = '%s/%s' % (lorry_prefix, package_name) - - # TODO: this products field 'x-products-python' - # probably shouldn't be hardcoded here - - lorry = {'type': repo_type, 'url': url, 'x-products-python': [package_name]} - - if repo_type == 'svn': - lorry['layout'] = 'standard' + # TODO: shouldn't be hardcoding ext name here + return utils.str_tarball_lorry('python', lorry_prefix, package_name, url) - return json.dumps({name: lorry}, indent=4, sort_keys=True) class PythonLorryExtension(ImportExtension): @@ -232,8 +194,9 @@ class PythonLorryExtension(ImportExtension): if 'home_page' in info else None) if repo_type: - print(str_repo_lorry(lorry_prefix, package_name, - repo_type, info['home_page'])) + # TODO: Don't hardcode extname here. + print(utils.str_repo_lorry('python', lorry_prefix, package_name, + repo_type, info['home_page'])) else: print(generate_tarball_lorry(lorry_prefix, client, package_name, version)) diff --git a/baserockimport/exts/python_lorry_tests.py b/baserockimport/exts/python_lorry_tests.py index f57be6d..298f9b0 100755 --- a/baserockimport/exts/python_lorry_tests.py +++ b/baserockimport/exts/python_lorry_tests.py @@ -17,18 +17,17 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import imp -python_lorry = imp.load_source('python_lorry', 'python.to_lorry') - import json - import unittest +import utils + LORRY_PREFIX = 'python' +EXTNAME = 'python' class Tests(unittest.TestCase): - def test_make_tarball_lorry(self): + def test_str_tarball_lorry(self): gzip, bzip, lzma = 'gzip', 'bzip2', 'lzma' valid_extensions = {'tar.gz': gzip, 'tgz': gzip, 'tar.Z': gzip, @@ -51,8 +50,8 @@ class Tests(unittest.TestCase): urls = [(make_url(ext), ext) for ext in valid_extensions] for (url, ext) in urls: - lorry_json = python_lorry.make_tarball_lorry(LORRY_PREFIX, - 'name', url) + lorry_json = utils.str_tarball_lorry(EXTNAME, LORRY_PREFIX, + 'name', url) print lorry_json tarball_url = get_tarball_lorry_url(fake_package_name, lorry_json) @@ -67,7 +66,8 @@ class Tests(unittest.TestCase): self.assertEqual(tarball_compression, valid_extensions[ext]) url = 'http://foobar/baz.tar' - lorry_json = python_lorry.make_tarball_lorry(LORRY_PREFIX, 'name', url) + lorry_json = utils.str_tarball_lorry(EXTNAME, LORRY_PREFIX, + 'name', url) self.assertEqual(get_tarball_lorry_url(fake_package_name, lorry_json), url) self.assertTrue('compression' not in lorry_json) diff --git a/baserockimport/exts/utils.py b/baserockimport/exts/utils.py new file mode 100644 index 0000000..aadefdf --- /dev/null +++ b/baserockimport/exts/utils.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# +# Copyright © 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from __future__ import print_function + +import json +import sys + +def warn(*args, **kwargs): + print('%s:' % sys.argv[0], *args, file=sys.stderr, **kwargs) + +def error(*args, **kwargs): + warn(*args, **kwargs) + sys.exit(1) + +def get_compression(url): + ''' Given a tarball url, return its compression type ''' + + bzip = 'bzip2' + gzip = 'gzip' + lzma = 'lzma' + + m = {'tar.gz': gzip, 'tgz': gzip, 'tar.Z': gzip, + 'tar.bz2': bzip, 'tbz2': bzip, + 'tar.lzma': lzma, 'tar.xz': lzma, 'tlz': lzma, 'txz': lzma} + + for x in [1, 2]: + ext = '.'.join(url.split('.')[-x:]) + if ext in m: return m[ext] + + return None + +# Assumption: url passed to this function must have a 'standard' tar extension +def str_tarball_lorry(extname, lorry_prefix, package_name, url): + ''' Create a json lorry file ''' + + name = '%s/%s' % (lorry_prefix, package_name) + + lorry = {'type': 'tarball', + 'url': url, + 'x-products-%s' % extname: [package_name]} + + compression = get_compression(url) + if compression: + lorry['compression'] = compression + + return json.dumps({name + "-tarball": lorry}, indent=4, sort_keys=True) + +def str_repo_lorry(extname, lorry_prefix, package_name, repo_type, url): + name = '%s/%s' % (lorry_prefix, package_name) + + lorry = {'type': repo_type, + 'url': url, + 'x-products-%s' % extname: [package_name]} + + if repo_type == 'svn': + lorry['layout'] = 'standard' + + return json.dumps({name: lorry}, indent=4, sort_keys=True) -- cgit v1.2.1