diff options
Diffstat (limited to 'baserockimport/exts')
| -rwxr-xr-x | baserockimport/exts/python.find_deps | 124 | ||||
| -rwxr-xr-x | baserockimport/exts/python.to_lorry | 73 | ||||
| -rwxr-xr-x | baserockimport/exts/python_run_pip | 32 |
3 files changed, 121 insertions, 108 deletions
diff --git a/baserockimport/exts/python.find_deps b/baserockimport/exts/python.find_deps index 91a9e39..b173110 100755 --- a/baserockimport/exts/python.find_deps +++ b/baserockimport/exts/python.find_deps @@ -24,6 +24,7 @@ from __future__ import print_function +import contextlib import sys import subprocess import os @@ -32,6 +33,7 @@ import tempfile import logging import select import signal +import shutil import pkg_resources import xmlrpclib @@ -262,65 +264,109 @@ def find_build_deps(source, name, version=None): return build_deps + +@contextlib.contextmanager +def temporary_virtualenv(): + tempdir = tempfile.mkdtemp() + + logging.debug('Creating virtualenv in %s', tempdir) + p = subprocess.Popen(['virtualenv', tempdir], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + + while True: + line = p.stdout.readline() + if line == '': + break + + logging.debug(line.rstrip('\n')) + + p.wait() # even with eof, wait for termination + + try: + yield tempdir + finally: + logging.debug('Removing virtualenv at %s', tempdir) + shutil.rmtree(tempdir) + + def find_runtime_deps(source, name, version=None, use_requirements_file=False): logging.debug('Finding runtime dependencies for %s%s at %s' % (name, ' %s' % version if version else '', source)) - # Run our patched pip to get a list of installed deps - # Run pip install . --list-dependencies=instdeps.txt with cwd=source - # Some temporary file needed for storing the requirements tmpfd, tmppath = tempfile.mkstemp() logging.debug('Writing install requirements to: %s', tmppath) - args = ['pip', 'install', '.', '--list-dependencies=%s' % tmppath] - if use_requirements_file: - args.insert(args.index('.') + 1, '-r') - args.insert(args.index('.') + 2, 'requirements.txt') + with temporary_virtualenv() as virtenv_path: + shutil.copytree(source, os.path.join(virtenv_path, 'source')) - logging.debug('Running pip, args: %s' % args) + pip_runner = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'python_run_pip') + logging.debug('pip_runner: %s', pip_runner) - p = subprocess.Popen(args, cwd=source, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + subprocess_env = os.environ.copy() + subprocess_env['TMPDIR'] = os.path.join(virtenv_path, 'tmp') + logging.debug('Using %s as TMPDIR', subprocess_env['TMPDIR']) + p = subprocess.Popen(pip_runner, cwd=virtenv_path, env=subprocess_env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = [] - while True: - line = p.stdout.readline() - if line == '': - break + output = [] + while True: + line = p.stdout.readline() + if line == '': + break - logging.debug(line.rstrip('\n')) - output.append(line) + logging.debug(line.rstrip('\n')) + output.append(line) - p.wait() # even with eof, wait for termination + p.wait() # even with eof, wait for termination - logging.debug('pip exited with code: %d' % p.returncode) + logging.debug('pip exited with code: %d' % p.returncode) - if p.returncode != 0: - error('Failed to get runtime dependencies for %s%s at %s. Output from ' - 'Pip: %s' % (name, ' %s' % version if version else '', source, - ' '.join(output))) + if p.returncode != 0: + error('Failed to get runtime dependencies for %s%s at %s. Output ' + 'from Pip: %s' % (name, ' %s' % version if version else '', + source, ' '.join(output))) + + # Now run pip freeze + logging.debug('Running pip freeze') + + p = subprocess.Popen(['/bin/bash', '-c', + 'source bin/activate; ' + 'pip freeze --disable-pip-version-check'], + cwd=virtenv_path, + stdout=tmpfd, stderr=subprocess.PIPE) - with os.fdopen(tmpfd) as tmpfile: - ss = resolve_specs(pkg_resources.parse_requirements(tmpfile)) - logging.debug("Resolved specs for %s: %s" % (name, ss)) + _, err = p.communicate() + os.close(tmpfd) - logging.debug("Removing root package from specs") + if p.returncode != 0: + error('failed to get runtime dependencies for %s%s at %s: %s' + % (name, ' %s' % version if version else '', source, err)) - # filter out "root" package - # hyphens and underscores are treated as equivalents - # in distribution names - specsets = {k: v for (k, v) in ss.iteritems() - if k not in [name, name.replace('_', '-')]} + with open(tmppath) as f: + logging.debug(f.read()) - versions = resolve_versions(specsets) - logging.debug('Resolved versions: %s' % versions) + with open(tmppath) as tmpfile: + ss = resolve_specs(pkg_resources.parse_requirements(tmpfile)) + logging.debug("Resolved specs for %s: %s" % (name, ss)) - # Since any of the candidates in versions should satisfy - # all specs, we just pick the first version we see - runtime_deps = {name: vs[0] for (name, vs) in versions.iteritems()} + logging.debug("Removing root package from specs") - os.remove(tmppath) + # filter out "root" package + # hyphens and underscores are treated as equivalents + # in distribution names + specsets = {k: v for (k, v) in ss.iteritems() + if k not in [name, name.replace('_', '-')]} + + versions = resolve_versions(specsets) + logging.debug('Resolved versions: %s' % versions) + + # Since any of the candidates in versions should satisfy + # all specs, we just pick the first version we see + runtime_deps = {name: vs[0] for (name, vs) in versions.iteritems()} + + os.remove(tmppath) if (len(runtime_deps) == 0 and not use_requirements_file and os.path.isfile(os.path.join(source, 'requirements.txt'))): @@ -360,6 +406,8 @@ class PythonFindDepsExtension(ImportExtension): root = {'python': deps} + logging.debug('Returning %s', root) + print(json.dumps(root)) if __name__ == '__main__': diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry index ebde27a..30f38e7 100755 --- a/baserockimport/exts/python.to_lorry +++ b/baserockimport/exts/python.to_lorry @@ -37,58 +37,6 @@ from importer_python_common import * from utils import warn, error import utils - -def find_repo_type(url): - - debug_vcss = False - - # Don't bother with detection if we can't get a 200 OK - logging.debug("Getting '%s' ..." % url) - - status_code = requests.get(url).status_code - if status_code != 200: - logging.debug('Got %d status code from %s, aborting repo detection' - % (status_code, url)) - return None - - logging.debug('200 OK for %s' % url) - logging.debug('Finding repo type for %s' % url) - - vcss = [('git', 'clone'), ('hg', 'clone'), - ('svn', 'checkout'), ('bzr', 'branch')] - - for (vcs, vcs_command) in vcss: - logging.debug('Trying %s %s' % (vcs, vcs_command)) - tempdir = tempfile.mkdtemp() - - p = subprocess.Popen([vcs, vcs_command, url], stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, stdin=subprocess.PIPE, - cwd=tempdir) - - # We close stdin on parent side to prevent the child from blocking - # if it reads on stdin - p.stdin.close() - - while True: - line = p.stdout.readline() - if line == '': - break - - if debug_vcss: - logging.debug(line.rstrip('\n')) - - p.wait() # even with eof on both streams, we still wait - - shutil.rmtree(tempdir) - - if p.returncode == 0: - logging.debug('%s is a %s repo' % (url, vcs)) - return vcs - - logging.debug("%s doesn't seem to be a repo" % url) - - return None - def filter_urls(urls): allowed_extensions = ['tar.gz', 'tgz', 'tar.Z', 'tar.bz2', 'tbz2', 'tar.lzma', 'tar.xz', 'tlz', 'txz', 'tar'] @@ -101,7 +49,7 @@ def filter_urls(urls): def get_releases(client, package_name): try: - releases = client.package_releases(package_name) + releases = client.package_releases(package_name, True) except Exception as e: error("Couldn't fetch release data:", e) @@ -185,23 +133,8 @@ class PythonLorryExtension(ImportExtension): logging.debug('Treating %s as %s' % (package_name, new_proj_name)) package_name = new_proj_name - try: - metadata = self.fetch_package_metadata(package_name) - except Exception as e: - error("Couldn't fetch package metadata: ", e) - - info = metadata.json()['info'] - - repo_type = (find_repo_type(info['home_page']) - if 'home_page' in info else None) - - if repo_type: - # TODO: Don't hardcode extname here. - print(utils.str_repo_lorry('python', lorry_prefix, package_name, - repo_type, info['home_page'])) - else: - print(generate_tarball_lorry(lorry_prefix, client, - package_name, version)) + print(generate_tarball_lorry(lorry_prefix, client, + package_name, version)) if __name__ == '__main__': PythonLorryExtension().run() diff --git a/baserockimport/exts/python_run_pip b/baserockimport/exts/python_run_pip new file mode 100755 index 0000000..f3877b4 --- /dev/null +++ b/baserockimport/exts/python_run_pip @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Copyright © 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +set -e + +pip_install="pip install --disable-pip-version-check ." + +source bin/activate +cd source + +if [[ -e requirements.txt ]] +then + echo "Running $pip_install -r requirements.txt" + $pip_install -r requirements.txt +else + echo "Running $pip_install" + $pip_install +fi |