diff options
author | Tom Pollard <tom.pollard@codethink.co.uk> | 2020-01-22 10:14:09 +0000 |
---|---|---|
committer | Tom Pollard <tom.pollard@codethink.co.uk> | 2020-02-11 15:39:50 +0000 |
commit | 6b2b1fe82017df09c493d875fd813570f9359a9c (patch) | |
tree | 3b3c956ba2680f2be97b049c3e271b9ee3754cf3 | |
parent | fef1b54426f0711ffcbed138fea209b936d7cbb8 (diff) | |
download | buildstream-tpollard/remove-pip-plugins.tar.gz |
plugins/sources/pip.py: Remove the pip source plugintpollard/remove-pip-plugins
The plugin & related tests are moved to bst-plugins-experimental,
NEWS updated to reflect this
-rw-r--r-- | NEWS | 3 | ||||
-rw-r--r-- | doc/source/core_plugins.rst | 1 | ||||
-rw-r--r-- | src/buildstream/plugins/sources/pip.py | 264 | ||||
-rw-r--r-- | tests/integration/pip_source.py | 177 | ||||
-rw-r--r-- | tests/integration/project/files/pip-source/app1.py | 11 | ||||
-rw-r--r-- | tests/integration/project/files/pip-source/myreqs.txt | 1 | ||||
-rw-r--r-- | tests/sources/pip.py | 58 | ||||
-rw-r--r-- | tests/sources/pip/first-source-pip/target.bst | 6 | ||||
-rw-r--r-- | tests/sources/pip/no-packages/file | 1 | ||||
-rw-r--r-- | tests/sources/pip/no-packages/target.bst | 6 | ||||
-rw-r--r-- | tests/sources/pip/no-ref/file | 1 | ||||
-rw-r--r-- | tests/sources/pip/no-ref/target.bst | 8 |
12 files changed, 3 insertions, 534 deletions
@@ -16,6 +16,9 @@ API Plugins ------- + o BREAKING CHANGE: pip source plugin has been moved to the + bst-plugins-experimental repository. + o BREAKING CHANGE: pip element plugin has been moved to the bst-plugins-experimental repository. diff --git a/doc/source/core_plugins.rst b/doc/source/core_plugins.rst index f21ed89e4..096b80777 100644 --- a/doc/source/core_plugins.rst +++ b/doc/source/core_plugins.rst @@ -48,7 +48,6 @@ Sources sources/git sources/bzr sources/patch - sources/pip .. _plugins_external: diff --git a/src/buildstream/plugins/sources/pip.py b/src/buildstream/plugins/sources/pip.py deleted file mode 100644 index eac2f3d01..000000000 --- a/src/buildstream/plugins/sources/pip.py +++ /dev/null @@ -1,264 +0,0 @@ -# -# Copyright 2018 Bloomberg Finance LP -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# -# Authors: -# Chandan Singh <csingh43@bloomberg.net> - -""" -pip - stage python packages using pip -===================================== - -**Host depndencies:** - - * ``pip`` python module - -This plugin will download source distributions for specified packages using -``pip`` but will not install them. It is expected that the elements using this -source will install the downloaded packages. - -Downloaded tarballs will be stored in a directory called ".bst_pip_downloads". - -**Usage:** - -.. code:: yaml - - # Specify the pip source kind - kind: pip - - # Optionally specify index url, defaults to PyPi - # This url is used to discover new versions of packages and download them - # Projects intending to mirror their sources to a permanent location should - # use an aliased url, and declare the alias in the project configuration - url: https://mypypi.example.com/simple - - # Optionally specify the path to requirements files - # Note that either 'requirements-files' or 'packages' must be defined - requirements-files: - - requirements.txt - - # Optionally specify a list of additional packages - # Note that either 'requirements-files' or 'packages' must be defined - packages: - - flake8 - - # Specify the ref. It is a list of strings of format - # "<package-name>==<version>", separated by "\\n". - # Usually this will be contents of a requirements.txt file where all - # package versions have been frozen. - ref: "flake8==3.5.0\\nmccabe==0.6.1\\npkg-resources==0.0.0\\npycodestyle==2.3.1\\npyflakes==1.6.0" - -See :ref:`built-in functionality doumentation <core_source_builtins>` for -details on common configuration options for sources. - -.. note:: - - The ``pip`` plugin is available since :ref:`format version 16 <project_format_version>` -""" - -import hashlib -import os -import re - -from buildstream import Source, SourceError, utils - -_OUTPUT_DIRNAME = ".bst_pip_downloads" -_PYPI_INDEX_URL = "https://pypi.org/simple/" - -# Used only for finding pip command -_PYTHON_VERSIONS = [ - "python", # when running in a venv, we might not have the exact version - "python2.7", - "python3.0", - "python3.1", - "python3.2", - "python3.3", - "python3.4", - "python3.5", - "python3.6", - "python3.7", -] - -# List of allowed extensions taken from -# https://docs.python.org/3/distutils/sourcedist.html. -# Names of source distribution archives must be of the form -# '%{package-name}-%{version}.%{extension}'. -_SDIST_RE = re.compile(r"^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$", re.IGNORECASE) - - -class PipSource(Source): - # pylint: disable=attribute-defined-outside-init - - # We need access to previous sources at track time to use requirements.txt - # but not at fetch time as self.ref should contain sufficient information - # for this plugin - BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True - - def configure(self, node): - node.validate_keys(["url", "packages", "ref", "requirements-files"] + Source.COMMON_CONFIG_KEYS) - self.ref = node.get_str("ref", None) - self.original_url = node.get_str("url", _PYPI_INDEX_URL) - self.index_url = self.translate_url(self.original_url) - self.packages = node.get_str_list("packages", []) - self.requirements_files = node.get_str_list("requirements-files", []) - - if not (self.packages or self.requirements_files): - raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified".format(self)) - - def preflight(self): - # Try to find a pip version that spports download command - self.host_pip = None - for python in reversed(_PYTHON_VERSIONS): - try: - host_python = utils.get_host_tool(python) - rc = self.call([host_python, "-m", "pip", "download", "--help"]) - if rc == 0: - self.host_pip = [host_python, "-m", "pip"] - break - except utils.ProgramNotFoundError: - pass - - if self.host_pip is None: - raise SourceError("{}: Unable to find a suitable pip command".format(self)) - - def get_unique_key(self): - return [self.original_url, self.ref] - - def is_cached(self): - return os.path.exists(self._mirror) and os.listdir(self._mirror) - - def get_ref(self): - return self.ref - - def load_ref(self, node): - self.ref = node.get_str("ref", None) - - def set_ref(self, ref, node): - node["ref"] = self.ref = ref - - def track(self, previous_sources_dir): # pylint: disable=arguments-differ - # XXX pip does not offer any public API other than the CLI tool so it - # is not feasible to correctly parse the requirements file or to check - # which package versions pip is going to install. - # See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program - # for details. - # As a result, we have to wastefully install the packages during track. - with self.tempdir() as tmpdir: - install_args = self.host_pip + [ - "download", - "--no-binary", - ":all:", - "--index-url", - self.index_url, - "--dest", - tmpdir, - ] - for requirement_file in self.requirements_files: - fpath = os.path.join(previous_sources_dir, requirement_file) - install_args += ["-r", fpath] - install_args += self.packages - - self.call(install_args, fail="Failed to install python packages") - reqs = self._parse_sdist_names(tmpdir) - - return "\n".join(["{}=={}".format(pkg, ver) for pkg, ver in reqs]) - - def fetch(self): # pylint: disable=arguments-differ - with self.tempdir() as tmpdir: - packages = self.ref.strip().split("\n") - package_dir = os.path.join(tmpdir, "packages") - os.makedirs(package_dir) - self.call( - [ - *self.host_pip, - "download", - "--no-binary", - ":all:", - "--index-url", - self.index_url, - "--dest", - package_dir, - *packages, - ], - fail="Failed to install python packages: {}".format(packages), - ) - - # If the mirror directory already exists, assume that some other - # process has fetched the sources before us and ensure that we do - # not raise an error in that case. - try: - utils.move_atomic(package_dir, self._mirror) - except utils.DirectoryExistsError: - # Another process has beaten us and has fetched the sources - # before us. - pass - except OSError as e: - raise SourceError( - "{}: Failed to move downloaded pip packages from '{}' to '{}': {}".format( - self, package_dir, self._mirror, e - ) - ) from e - - def stage(self, directory): - with self.timed_activity("Staging Python packages", silent_nested=True): - utils.copy_files(self._mirror, os.path.join(directory, _OUTPUT_DIRNAME)) - - # Directory where this source should stage its files - # - @property - def _mirror(self): - if not self.ref: - return None - return os.path.join( - self.get_mirror_directory(), - utils.url_directory_name(self.original_url), - hashlib.sha256(self.ref.encode()).hexdigest(), - ) - - # Parse names of downloaded source distributions - # - # Args: - # basedir (str): Directory containing source distribution archives - # - # Returns: - # (list): List of (package_name, version) tuples in sorted order - # - def _parse_sdist_names(self, basedir): - reqs = [] - for f in os.listdir(basedir): - pkg = _match_package_name(f) - if pkg is not None: - reqs.append(pkg) - - return sorted(reqs) - - -# Extract the package name and version of a source distribution -# -# Args: -# filename (str): Filename of the source distribution -# -# Returns: -# (tuple): A tuple of (package_name, version) -# -def _match_package_name(filename): - pkg_match = _SDIST_RE.match(filename) - if pkg_match is None: - return None - return pkg_match.groups() - - -def setup(): - return PipSource diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py deleted file mode 100644 index 5d314974d..000000000 --- a/tests/integration/pip_source.py +++ /dev/null @@ -1,177 +0,0 @@ -# Pylint doesn't play well with fixtures and dependency injection from pytest -# pylint: disable=redefined-outer-name - -import os -import pytest - -from buildstream import _yaml - -from buildstream.testing import cli_integration as cli # pylint: disable=unused-import -from buildstream.testing.integration import assert_contains -from buildstream.testing._utils.site import HAVE_SANDBOX - -from tests.testutils.python_repo import setup_pypi_repo # pylint: disable=unused-import - - -pytestmark = pytest.mark.integration - - -DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") - - -@pytest.mark.datafiles(DATA_DIR) -def test_pip_source_import_packages(cli, datafiles, setup_pypi_repo): - project = str(datafiles) - checkout = os.path.join(cli.directory, "checkout") - element_path = os.path.join(project, "elements") - element_name = "pip/hello.bst" - - # check that exotically named packages are imported correctly - myreqs_packages = "hellolib" - dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"] - mock_packages = {myreqs_packages: {package: {} for package in dependencies}} - - # create mock pypi repository - pypi_repo = os.path.join(project, "files", "pypi-repo") - os.makedirs(pypi_repo, exist_ok=True) - setup_pypi_repo(mock_packages, pypi_repo) - - element = { - "kind": "import", - "sources": [ - {"kind": "local", "path": "files/pip-source"}, - {"kind": "pip", "url": "file://{}".format(os.path.realpath(pypi_repo)), "packages": [myreqs_packages]}, - ], - } - os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) - _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) - - result = cli.run(project=project, args=["source", "track", element_name]) - assert result.exit_code == 0 - - result = cli.run(project=project, args=["build", element_name]) - assert result.exit_code == 0 - - result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) - assert result.exit_code == 0 - - assert_contains( - checkout, - [ - "/.bst_pip_downloads", - "/.bst_pip_downloads/hellolib-0.1.tar.gz", - "/.bst_pip_downloads/app2-0.1.tar.gz", - "/.bst_pip_downloads/app.3-0.1.tar.gz", - "/.bst_pip_downloads/app-4-0.1.tar.gz", - "/.bst_pip_downloads/app_5-0.1.tar.gz", - "/.bst_pip_downloads/app.no.6-0.1.tar.gz", - "/.bst_pip_downloads/app-no-7-0.1.tar.gz", - "/.bst_pip_downloads/app_no_8-0.1.tar.gz", - ], - ) - - -@pytest.mark.datafiles(DATA_DIR) -def test_pip_source_import_requirements_files(cli, datafiles, setup_pypi_repo): - project = str(datafiles) - checkout = os.path.join(cli.directory, "checkout") - element_path = os.path.join(project, "elements") - element_name = "pip/hello.bst" - - # check that exotically named packages are imported correctly - myreqs_packages = "hellolib" - dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"] - mock_packages = {myreqs_packages: {package: {} for package in dependencies}} - - # create mock pypi repository - pypi_repo = os.path.join(project, "files", "pypi-repo") - os.makedirs(pypi_repo, exist_ok=True) - setup_pypi_repo(mock_packages, pypi_repo) - - element = { - "kind": "import", - "sources": [ - {"kind": "local", "path": "files/pip-source"}, - { - "kind": "pip", - "url": "file://{}".format(os.path.realpath(pypi_repo)), - "requirements-files": ["myreqs.txt"], - }, - ], - } - os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) - _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) - - result = cli.run(project=project, args=["source", "track", element_name]) - assert result.exit_code == 0 - - result = cli.run(project=project, args=["build", element_name]) - assert result.exit_code == 0 - - result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) - assert result.exit_code == 0 - - assert_contains( - checkout, - [ - "/.bst_pip_downloads", - "/.bst_pip_downloads/hellolib-0.1.tar.gz", - "/.bst_pip_downloads/app2-0.1.tar.gz", - "/.bst_pip_downloads/app.3-0.1.tar.gz", - "/.bst_pip_downloads/app-4-0.1.tar.gz", - "/.bst_pip_downloads/app_5-0.1.tar.gz", - "/.bst_pip_downloads/app.no.6-0.1.tar.gz", - "/.bst_pip_downloads/app-no-7-0.1.tar.gz", - "/.bst_pip_downloads/app_no_8-0.1.tar.gz", - ], - ) - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") -def test_pip_source_build(cli, datafiles, setup_pypi_repo): - project = str(datafiles) - element_path = os.path.join(project, "elements") - element_name = "pip/hello.bst" - - # check that exotically named packages are imported correctly - myreqs_packages = "hellolib" - dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"] - mock_packages = {myreqs_packages: {package: {} for package in dependencies}} - - # create mock pypi repository - pypi_repo = os.path.join(project, "files", "pypi-repo") - os.makedirs(pypi_repo, exist_ok=True) - setup_pypi_repo(mock_packages, pypi_repo) - - element = { - "kind": "manual", - "depends": ["base.bst"], - "sources": [ - {"kind": "local", "path": "files/pip-source"}, - { - "kind": "pip", - "url": "file://{}".format(os.path.realpath(pypi_repo)), - "requirements-files": ["myreqs.txt"], - "packages": dependencies, - }, - ], - "config": { - "install-commands": [ - "pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz", - "install app1.py %{install-root}/usr/bin/", - ] - }, - } - os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) - _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) - - result = cli.run(project=project, args=["source", "track", element_name]) - assert result.exit_code == 0 - - result = cli.run(project=project, args=["build", element_name]) - assert result.exit_code == 0 - - result = cli.run(project=project, args=["shell", element_name, "/usr/bin/app1.py"]) - assert result.exit_code == 0 - assert result.output == "Hello App1! This is hellolib\n" diff --git a/tests/integration/project/files/pip-source/app1.py b/tests/integration/project/files/pip-source/app1.py deleted file mode 100644 index b96d14b00..000000000 --- a/tests/integration/project/files/pip-source/app1.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -from hellolib import hello - - -def main(): - hello("App1") - - -if __name__ == "__main__": - main() diff --git a/tests/integration/project/files/pip-source/myreqs.txt b/tests/integration/project/files/pip-source/myreqs.txt deleted file mode 100644 index c805aae53..000000000 --- a/tests/integration/project/files/pip-source/myreqs.txt +++ /dev/null @@ -1 +0,0 @@ -hellolib diff --git a/tests/sources/pip.py b/tests/sources/pip.py deleted file mode 100644 index 1eacb4462..000000000 --- a/tests/sources/pip.py +++ /dev/null @@ -1,58 +0,0 @@ -# Pylint doesn't play well with fixtures and dependency injection from pytest -# pylint: disable=redefined-outer-name - -import os -import pytest - -from buildstream.exceptions import ErrorDomain -from buildstream.plugins.sources.pip import _match_package_name -from buildstream.testing import cli, generate_project # pylint: disable=unused-import - -DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pip",) - - -# Test that without ref, consistency is set appropriately. -@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref")) -def test_no_ref(cli, datafiles): - project = str(datafiles) - generate_project(project) - assert cli.get_element_state(project, "target.bst") == "no reference" - - -# Test that pip is not allowed to be the first source -@pytest.mark.datafiles(os.path.join(DATA_DIR, "first-source-pip")) -def test_first_source(cli, datafiles): - project = str(datafiles) - generate_project(project) - result = cli.run(project=project, args=["show", "target.bst"]) - result.assert_main_error(ErrorDomain.ELEMENT, None) - - -# Test that error is raised when neither packges nor requirements files -# have been specified -@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-packages")) -def test_no_packages(cli, datafiles): - project = str(datafiles) - generate_project(project) - result = cli.run(project=project, args=["show", "target.bst"]) - result.assert_main_error(ErrorDomain.SOURCE, None) - - -# Test that pip source parses tar ball names correctly for the ref -@pytest.mark.parametrize( - "tarball, expected_name, expected_version", - [ - ("dotted.package-0.9.8.tar.gz", "dotted.package", "0.9.8"), - ("hyphenated-package-2.6.0.tar.gz", "hyphenated-package", "2.6.0"), - ("underscore_pkg-3.1.0.tar.gz", "underscore_pkg", "3.1.0"), - ("numbers2and5-1.0.1.tar.gz", "numbers2and5", "1.0.1"), - ("multiple.dots.package-5.6.7.tar.gz", "multiple.dots.package", "5.6.7"), - ("multiple-hyphens-package-1.2.3.tar.gz", "multiple-hyphens-package", "1.2.3"), - ("multiple_underscore_pkg-3.4.5.tar.gz", "multiple_underscore_pkg", "3.4.5"), - ("shortversion-1.0.tar.gz", "shortversion", "1.0"), - ("longversion-1.2.3.4.tar.gz", "longversion", "1.2.3.4"), - ], -) -def test_match_package_name(tarball, expected_name, expected_version): - name, version = _match_package_name(tarball) - assert (expected_name, expected_version) == (name, version) diff --git a/tests/sources/pip/first-source-pip/target.bst b/tests/sources/pip/first-source-pip/target.bst deleted file mode 100644 index e5f20ab0b..000000000 --- a/tests/sources/pip/first-source-pip/target.bst +++ /dev/null @@ -1,6 +0,0 @@ -kind: import -description: pip should not be allowed to be the first source -sources: -- kind: pip - packages: - - flake8 diff --git a/tests/sources/pip/no-packages/file b/tests/sources/pip/no-packages/file deleted file mode 100644 index 980a0d5f1..000000000 --- a/tests/sources/pip/no-packages/file +++ /dev/null @@ -1 +0,0 @@ -Hello World! diff --git a/tests/sources/pip/no-packages/target.bst b/tests/sources/pip/no-packages/target.bst deleted file mode 100644 index 0d8b948c4..000000000 --- a/tests/sources/pip/no-packages/target.bst +++ /dev/null @@ -1,6 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: local - path: file -- kind: pip diff --git a/tests/sources/pip/no-ref/file b/tests/sources/pip/no-ref/file deleted file mode 100644 index 980a0d5f1..000000000 --- a/tests/sources/pip/no-ref/file +++ /dev/null @@ -1 +0,0 @@ -Hello World! diff --git a/tests/sources/pip/no-ref/target.bst b/tests/sources/pip/no-ref/target.bst deleted file mode 100644 index ec450b7ef..000000000 --- a/tests/sources/pip/no-ref/target.bst +++ /dev/null @@ -1,8 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: local - path: file -- kind: pip - packages: - - flake8 |