Use virtualenv and Pip to find runtime deps; remove searching for upstream

Co-authored-by: Sam Thursfield <sam.thursfield@codethink.co.uk>

Since upstream pip do not want to merge https://github.com/pypa/pip/pull/2371
we should avoid depending on this pull request.

To find runtime dependencies we now run pip install inside a virtual env
then run pip freeze to obtain the dependency set, this has the advantage
that nearly all the work is being done by pip.

Originally the python extensions were designed to look for upstream git
repos, in practice this is unreliable and won't be compatible with obtaining
dependencies using pip install, so the downside of this approach is that
all lorries will be tarballs, the upshot is that we can now automatically
import many packages that we couldn't import before.

Another upshot of this approach is that we may be able to consider the
removal of a lot of the spec processing and validation code if we're willing
to worry less about build dependencies, we're not sure whether we should
be willing to worry less about build dependencies though.

We've had encouraging results using this patch so far, we are now able
to import, without user intervention, packages that failed previously,
such as boto, persistent-pineapple, jmespath, coverage, requests also
almost imported successfully but appears to require a release of pytest
that is uploaded as a zip.

Change-Id: I705c6f6bd722df041d17630287382f851008e97a

4 files changed, 123 insertions, 109 deletions

diff --git a/baserockimport/app.py b/baserockimport/app.py
index 5f3d435..ae95d58 100644
--- a/baserockimport/app.py
+++ b/baserockimport/app.py
@@ -227,7 +227,8 @@ class BaserockImportApplication(cliapp.Application):
         loop = baserockimport.mainloop.ImportLoop(app=self,
                                                   goal_kind='python',
                                                   goal_name=package_name,
-                                                  goal_version=package_version)
+                                                  goal_version=package_version,
+                                                  ignore_version_field=True)
         loop.enable_importer('python', strata=['strata/core.morph'],
                              package_comp_callback=comp)
         loop.run()
diff --git a/baserockimport/exts/python.find_deps b/baserockimport/exts/python.find_deps
index 91a9e39..b173110 100755
--- a/baserockimport/exts/python.find_deps
+++ b/baserockimport/exts/python.find_deps
@@ -24,6 +24,7 @@
 
 from __future__ import print_function
 
+import contextlib
 import sys
 import subprocess
 import os
@@ -32,6 +33,7 @@ import tempfile
 import logging
 import select
 import signal
+import shutil
 
 import pkg_resources
 import xmlrpclib
@@ -262,65 +264,109 @@ def find_build_deps(source, name, version=None):
 
     return build_deps
 
+
+@contextlib.contextmanager
+def temporary_virtualenv():
+    tempdir = tempfile.mkdtemp()
+
+    logging.debug('Creating virtualenv in %s', tempdir)
+    p = subprocess.Popen(['virtualenv', tempdir], stdout=subprocess.PIPE,
+                     stderr=subprocess.STDOUT)
+
+    while True:
+        line = p.stdout.readline()
+        if line == '':
+            break
+
+        logging.debug(line.rstrip('\n'))
+
+    p.wait()    # even with eof, wait for termination
+
+    try:
+        yield tempdir
+    finally:
+        logging.debug('Removing virtualenv at %s', tempdir)
+        shutil.rmtree(tempdir)
+
+
 def find_runtime_deps(source, name, version=None, use_requirements_file=False):
     logging.debug('Finding runtime dependencies for %s%s at %s'
                   % (name, ' %s' % version if version else '', source))
 
-    # Run our patched pip to get a list of installed deps
-    # Run pip install . --list-dependencies=instdeps.txt with cwd=source
-
     # Some temporary file needed for storing the requirements
     tmpfd, tmppath = tempfile.mkstemp()
     logging.debug('Writing install requirements to: %s', tmppath)
 
-    args = ['pip', 'install', '.', '--list-dependencies=%s' % tmppath]
-    if use_requirements_file:
-        args.insert(args.index('.') + 1, '-r')
-        args.insert(args.index('.') + 2, 'requirements.txt')
+    with temporary_virtualenv() as virtenv_path:
+        shutil.copytree(source, os.path.join(virtenv_path, 'source'))
 
-    logging.debug('Running pip, args: %s' % args)
+        pip_runner = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                                  'python_run_pip')
+        logging.debug('pip_runner: %s', pip_runner)
 
-    p = subprocess.Popen(args, cwd=source, stdout=subprocess.PIPE,
-                         stderr=subprocess.STDOUT)
+        subprocess_env = os.environ.copy()
+        subprocess_env['TMPDIR'] = os.path.join(virtenv_path, 'tmp')
+        logging.debug('Using %s as TMPDIR', subprocess_env['TMPDIR'])
+        p = subprocess.Popen(pip_runner, cwd=virtenv_path, env=subprocess_env,
+                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
-    output = []
-    while True:
-        line = p.stdout.readline()
-        if line == '':
-            break
+        output = []
+        while True:
+            line = p.stdout.readline()
+            if line == '':
+                break
 
-        logging.debug(line.rstrip('\n'))
-        output.append(line)
+            logging.debug(line.rstrip('\n'))
+            output.append(line)
 
-    p.wait()    # even with eof, wait for termination
+        p.wait()    # even with eof, wait for termination
 
-    logging.debug('pip exited with code: %d' % p.returncode)
+        logging.debug('pip exited with code: %d' % p.returncode)
 
-    if p.returncode != 0:
-        error('Failed to get runtime dependencies for %s%s at %s. Output from '
-              'Pip: %s' % (name, ' %s' % version if version else '', source,
-              '  '.join(output)))
+        if p.returncode != 0:
+            error('Failed to get runtime dependencies for %s%s at %s. Output '
+                  'from Pip: %s' % (name, ' %s' % version if version else '',
+                                    source, '  '.join(output)))
+
+        # Now run pip freeze
+        logging.debug('Running pip freeze')
+
+        p = subprocess.Popen(['/bin/bash', '-c',
+                              'source bin/activate; '
+                              'pip freeze --disable-pip-version-check'],
+                              cwd=virtenv_path,
+                              stdout=tmpfd, stderr=subprocess.PIPE)
 
-    with os.fdopen(tmpfd) as tmpfile:
-        ss = resolve_specs(pkg_resources.parse_requirements(tmpfile))
-        logging.debug("Resolved specs for %s: %s" % (name, ss))
+        _, err = p.communicate()
+        os.close(tmpfd)
 
-        logging.debug("Removing root package from specs")
+        if p.returncode != 0:
+            error('failed to get runtime dependencies for %s%s at %s: %s'
+                % (name, ' %s' % version if version else '', source, err))
 
-        # filter out "root" package
-        # hyphens and underscores are treated as equivalents
-        # in distribution names
-        specsets = {k: v for (k, v) in ss.iteritems()
-            if k not in [name, name.replace('_', '-')]}
+        with open(tmppath) as f:
+            logging.debug(f.read())
 
-        versions = resolve_versions(specsets)
-        logging.debug('Resolved versions: %s' % versions)
+        with open(tmppath) as tmpfile:
+            ss = resolve_specs(pkg_resources.parse_requirements(tmpfile))
+            logging.debug("Resolved specs for %s: %s" % (name, ss))
 
-        # Since any of the candidates in versions should satisfy
-        # all specs, we just pick the first version we see
-        runtime_deps = {name: vs[0] for (name, vs) in versions.iteritems()}
+            logging.debug("Removing root package from specs")
 
-    os.remove(tmppath)
+            # filter out "root" package
+            # hyphens and underscores are treated as equivalents
+            # in distribution names
+            specsets = {k: v for (k, v) in ss.iteritems()
+                if k not in [name, name.replace('_', '-')]}
+
+            versions = resolve_versions(specsets)
+            logging.debug('Resolved versions: %s' % versions)
+
+            # Since any of the candidates in versions should satisfy
+            # all specs, we just pick the first version we see
+            runtime_deps = {name: vs[0] for (name, vs) in versions.iteritems()}
+
+        os.remove(tmppath)
 
     if (len(runtime_deps) == 0 and not use_requirements_file
             and os.path.isfile(os.path.join(source, 'requirements.txt'))):
@@ -360,6 +406,8 @@ class PythonFindDepsExtension(ImportExtension):
 
         root = {'python': deps}
 
+        logging.debug('Returning %s', root)
+
         print(json.dumps(root))
 
 if __name__ == '__main__':
diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry
index ebde27a..30f38e7 100755
--- a/baserockimport/exts/python.to_lorry
+++ b/baserockimport/exts/python.to_lorry
@@ -37,58 +37,6 @@ from importer_python_common import *
 from utils import warn, error
 import utils
 
-
-def find_repo_type(url):
-
-    debug_vcss = False
-
-    # Don't bother with detection if we can't get a 200 OK
-    logging.debug("Getting '%s' ..." % url)
-
-    status_code = requests.get(url).status_code
-    if status_code != 200:
-        logging.debug('Got %d status code from %s, aborting repo detection'
-                      % (status_code, url))
-        return None
-
-    logging.debug('200 OK for %s' % url)
-    logging.debug('Finding repo type for %s' % url)
-
-    vcss = [('git', 'clone'), ('hg', 'clone'),
-            ('svn', 'checkout'), ('bzr', 'branch')]
-
-    for (vcs, vcs_command) in vcss:
-        logging.debug('Trying %s %s' % (vcs, vcs_command))
-        tempdir = tempfile.mkdtemp()
-
-        p = subprocess.Popen([vcs, vcs_command, url], stdout=subprocess.PIPE,
-                             stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
-                             cwd=tempdir)
-
-        # We close stdin on parent side to prevent the child from blocking
-        # if it reads on stdin
-        p.stdin.close()
-
-        while True:
-            line = p.stdout.readline()
-            if line == '':
-                break
-
-            if debug_vcss:
-                logging.debug(line.rstrip('\n'))
-
-        p.wait()    # even with eof on both streams, we still wait
-
-        shutil.rmtree(tempdir)
-
-        if p.returncode == 0:
-            logging.debug('%s is a %s repo' % (url, vcs))
-            return vcs
-
-    logging.debug("%s doesn't seem to be a repo" % url)
-
-    return None
-
 def filter_urls(urls):
     allowed_extensions = ['tar.gz', 'tgz', 'tar.Z', 'tar.bz2', 'tbz2',
                           'tar.lzma', 'tar.xz', 'tlz', 'txz', 'tar']
@@ -101,7 +49,7 @@ def filter_urls(urls):
 
 def get_releases(client, package_name):
     try:
-        releases = client.package_releases(package_name)
+        releases = client.package_releases(package_name, True)
     except Exception as e:
         error("Couldn't fetch release data:", e)
 
@@ -185,23 +133,8 @@ class PythonLorryExtension(ImportExtension):
         logging.debug('Treating %s as %s' % (package_name, new_proj_name))
         package_name = new_proj_name
 
-        try:
-            metadata = self.fetch_package_metadata(package_name)
-        except Exception as e:
-            error("Couldn't fetch package metadata: ", e)
-
-        info = metadata.json()['info']
-
-        repo_type = (find_repo_type(info['home_page'])
-                     if 'home_page' in info else None)
-
-        if repo_type:
-            # TODO: Don't hardcode extname here.
-            print(utils.str_repo_lorry('python', lorry_prefix, package_name,
-                                       repo_type, info['home_page']))
-        else:
-            print(generate_tarball_lorry(lorry_prefix, client,
-                                         package_name, version))
+        print(generate_tarball_lorry(lorry_prefix, client,
+                                     package_name, version))
 
 if __name__ == '__main__':
     PythonLorryExtension().run()
diff --git a/baserockimport/exts/python_run_pip b/baserockimport/exts/python_run_pip
new file mode 100755
index 0000000..f3877b4
--- /dev/null
+++ b/baserockimport/exts/python_run_pip
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Copyright © 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+set -e
+
+pip_install="pip install --disable-pip-version-check ."
+
+source bin/activate
+cd source
+
+if [[ -e requirements.txt ]]
+then
+	echo "Running $pip_install -r requirements.txt"
+	$pip_install -r requirements.txt
+else
+	echo "Running $pip_install"
+	$pip_install
+fi