Merge branch 'pip-elem-install-from-pip-source' into 'master'

Pip elem install from pip source

Closes #589

See merge request BuildStream/buildstream!1336

4 files changed, 159 insertions, 14 deletions

diff --git a/buildstream/plugins/elements/pip.yaml b/buildstream/plugins/elements/pip.yaml
index b2b3d3857..294d4ad9a 100644
--- a/buildstream/plugins/elements/pip.yaml
+++ b/buildstream/plugins/elements/pip.yaml
@@ -3,6 +3,14 @@
 variables:
 
   pip: pip
+  pip-flags: |
+    %{pip} install --no-deps --root=%{install-root} --prefix=%{prefix}
+  pip-install-package: |
+    %{pip-flags} %{conf-root}
+  pip-download-dir: |
+    .bst_pip_downloads
+  pip-install-dependencies: |
+    if [ -e %{pip-download-dir} ]; then %{pip-flags} %{pip-download-dir}/*; fi
 
 config:
 
@@ -14,7 +22,9 @@ config:
   #
   install-commands:
   - |
-    %{pip} install --no-deps --root=%{install-root} --prefix=%{prefix} %{conf-root} 
+    %{pip-install-package}
+  - |
+    %{pip-install-dependencies}
 
   # Commands for stripping debugging information out of
   # installed binaries
diff --git a/tests/integration/pip_element.py b/tests/integration/pip_element.py
index 9ef163125..94130d9ab 100644
--- a/tests/integration/pip_element.py
+++ b/tests/integration/pip_element.py
@@ -6,6 +6,7 @@ from buildstream import _yaml
 
 from buildstream.testing import cli_integration as cli
 from buildstream.testing.integration import assert_contains
+from tests.testutils import setup_pypi_repo
 from tests.testutils.site import HAVE_SANDBOX
 
 
@@ -67,3 +68,64 @@ def test_pip_run(cli, datafiles):
     result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello'])
     assert result.exit_code == 0
     assert result.output == 'Hello, world!\n'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+def test_pip_element_should_install_pip_deps(cli, datafiles, setup_pypi_repo):
+    project = str(datafiles)
+    elements_path = os.path.join(project, 'elements')
+    element_name = 'pip/hello.bst'
+
+    # check that exotically named packages are imported correctly
+    myreqs_packages = 'alohalib'
+    dependencies = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
+    mock_packages = {
+        myreqs_packages: {
+            package: {} for package in dependencies
+        }
+    }
+
+    # set up directories
+    pypi_repo = os.path.join(project, 'files', 'pypi-repo')
+    os.makedirs(pypi_repo, exist_ok=True)
+    os.makedirs(os.path.dirname(os.path.join(elements_path, element_name)), exist_ok=True)
+    setup_pypi_repo(mock_packages, pypi_repo)
+
+    # create pip element
+    element = {
+        'kind': 'pip',
+        'variables': {
+            'pip': 'pip3'
+        },
+        'depends': [{
+            'filename': 'base.bst'
+        }],
+        'sources': [
+            {
+                'kind': 'tar',
+                'url': 'file://{}/files/hello.tar.xz'.format(project),
+                # FIXME: remove hardcoded ref once issue #1010 is closed
+                'ref': 'ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d'
+            },
+            {
+                'kind': 'pip',
+                'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
+                'packages': [myreqs_packages],
+            }
+        ]
+    }
+    _yaml.dump(element, os.path.join(elements_path, element_name))
+
+    result = cli.run(project=project, args=['source', 'track', element_name])
+    assert result.exit_code == 0
+
+    result = cli.run(project=project, args=['build', element_name])
+    assert result.exit_code == 0
+
+    # get installed packages in sandbox
+    installed_packages = set(
+        cli.run(project=project, args=['shell', element_name, 'pip3', 'freeze']).output.split('\n'))
+    # compare with packages that are expected to be installed
+    pip_source_packages = {package.replace('_', "-") + '==0.1' for package in dependencies + [myreqs_packages]}
+    assert pip_source_packages.issubset(installed_packages)
diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py
index 645dad444..3ba3a50b4 100644
--- a/tests/integration/pip_source.py
+++ b/tests/integration/pip_source.py
@@ -19,20 +19,83 @@ DATA_DIR = os.path.join(
 
 
 @pytest.mark.datafiles(DATA_DIR)
-def test_pip_source_import(cli, datafiles, setup_pypi_repo):
+def test_pip_source_import_packages(cli, datafiles, setup_pypi_repo):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
     element_path = os.path.join(project, 'elements')
     element_name = 'pip/hello.bst'
 
     # check that exotically named packages are imported correctly
-    myreqs_packages = ['hellolib']
-    packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
+    myreqs_packages = 'hellolib'
+    dependencies = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
+    mock_packages = {
+        myreqs_packages: {
+            package: {} for package in dependencies
+        }
+    }
+
+    # create mock pypi repository
+    pypi_repo = os.path.join(project, 'files', 'pypi-repo')
+    os.makedirs(pypi_repo, exist_ok=True)
+    setup_pypi_repo(mock_packages, pypi_repo)
+
+    element = {
+        'kind': 'import',
+        'sources': [
+            {
+                'kind': 'local',
+                'path': 'files/pip-source'
+            },
+            {
+                'kind': 'pip',
+                'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
+                'packages': [myreqs_packages]
+            }
+        ]
+    }
+    os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
+    _yaml.dump(element, os.path.join(element_path, element_name))
+
+    result = cli.run(project=project, args=['source', 'track', element_name])
+    assert result.exit_code == 0
+
+    result = cli.run(project=project, args=['build', element_name])
+    assert result.exit_code == 0
+
+    result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+    assert result.exit_code == 0
+
+    assert_contains(checkout, ['/.bst_pip_downloads',
+                               '/.bst_pip_downloads/hellolib-0.1.tar.gz',
+                               '/.bst_pip_downloads/app2-0.1.tar.gz',
+                               '/.bst_pip_downloads/app.3-0.1.tar.gz',
+                               '/.bst_pip_downloads/app-4-0.1.tar.gz',
+                               '/.bst_pip_downloads/app_5-0.1.tar.gz',
+                               '/.bst_pip_downloads/app.no.6-0.1.tar.gz',
+                               '/.bst_pip_downloads/app-no-7-0.1.tar.gz',
+                               '/.bst_pip_downloads/app_no_8-0.1.tar.gz'])
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_pip_source_import_requirements_files(cli, datafiles, setup_pypi_repo):
+    project = str(datafiles)
+    checkout = os.path.join(cli.directory, 'checkout')
+    element_path = os.path.join(project, 'elements')
+    element_name = 'pip/hello.bst'
+
+    # check that exotically named packages are imported correctly
+    myreqs_packages = 'hellolib'
+    dependencies = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
+    mock_packages = {
+        myreqs_packages: {
+            package: {} for package in dependencies
+        }
+    }
 
     # create mock pypi repository
     pypi_repo = os.path.join(project, 'files', 'pypi-repo')
     os.makedirs(pypi_repo, exist_ok=True)
-    setup_pypi_repo(myreqs_packages + packages, pypi_repo)
+    setup_pypi_repo(mock_packages, pypi_repo)
 
     element = {
         'kind': 'import',
@@ -45,7 +108,6 @@ def test_pip_source_import(cli, datafiles, setup_pypi_repo):
                 'kind': 'pip',
                 'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
                 'requirements-files': ['myreqs.txt'],
-                'packages': packages
             }
         ]
     }
@@ -80,13 +142,18 @@ def test_pip_source_build(cli, datafiles, setup_pypi_repo):
     element_name = 'pip/hello.bst'
 
     # check that exotically named packages are imported correctly
-    myreqs_packages = ['hellolib']
-    packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
+    myreqs_packages = 'hellolib'
+    dependencies = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
+    mock_packages = {
+        myreqs_packages: {
+            package: {} for package in dependencies
+        }
+    }
 
     # create mock pypi repository
     pypi_repo = os.path.join(project, 'files', 'pypi-repo')
     os.makedirs(pypi_repo, exist_ok=True)
-    setup_pypi_repo(myreqs_packages + packages, pypi_repo)
+    setup_pypi_repo(mock_packages, pypi_repo)
 
     element = {
         'kind': 'manual',
@@ -100,7 +167,7 @@ def test_pip_source_build(cli, datafiles, setup_pypi_repo):
                 'kind': 'pip',
                 'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
                 'requirements-files': ['myreqs.txt'],
-                'packages': packages
+                'packages': dependencies
             }
         ],
         'config': {
diff --git a/tests/testutils/python_repo.py b/tests/testutils/python_repo.py
index fcaf2d293..c8e5bf343 100644
--- a/tests/testutils/python_repo.py
+++ b/tests/testutils/python_repo.py
@@ -15,6 +15,7 @@ setup(
     version='{version}',
     description='{name}',
     packages=['{pkgdirname}'],
+    install_requires={pkgdeps},
     entry_points={{
         'console_scripts': [
             '{pkgdirname}={pkgdirname}:main'
@@ -56,7 +57,9 @@ HTML_TEMPLATE = '''\
 # Returns:
 #    None
 #
-def generate_pip_package(tmpdir, pypi, name, version='0.1'):
+def generate_pip_package(tmpdir, pypi, name, version='0.1', dependencies=None):
+    if dependencies is None:
+        dependencies = []
     # check if package already exists in pypi
     pypi_package = os.path.join(pypi, re.sub('[^0-9a-zA-Z]+', '-', name))
     if os.path.exists(pypi_package):
@@ -77,7 +80,8 @@ def generate_pip_package(tmpdir, pypi, name, version='0.1'):
             SETUP_TEMPLATE.format(
                 name=name,
                 version=version,
-                pkgdirname=pkgdirname
+                pkgdirname=pkgdirname,
+                pkgdeps=dependencies
             )
         )
     os.chmod(setup_file, 0o755)
@@ -125,8 +129,10 @@ def setup_pypi_repo(tmpdir):
         return pkgdir
 
     def add_packages(packages, pypi_repo):
-        for package in packages:
+        for package, dependencies in packages.items():
             pkgdir = create_pkgdir(package)
-            generate_pip_package(pkgdir, pypi_repo, package)
+            generate_pip_package(pkgdir, pypi_repo, package, dependencies=list(dependencies.keys()))
+            for dependency, dependency_dependencies in dependencies.items():
+                add_packages({dependency: dependency_dependencies}, pypi_repo)
 
     return add_packages