diff options
33 files changed, 9 insertions, 304 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ab79e131..38a55053b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -361,7 +361,7 @@ docs: fetchers: 2 EOF - dnf install -y ostree python3-gobject-base - - pip3 install -r requirements/requirements.txt -r requirements/plugin-requirements.txt + - pip3 install -r requirements/requirements.txt - pip3 wheel --wheel-dir wheels/ --no-deps . - pip3 install --no-index wheels/* - pip3 install --user -e ${BST_EXT_URL}@${BST_EXT_REF} @@ -41,6 +41,12 @@ Core its ref (most plugins would handle that correctly), you will need to delete the internal source cache first. +Plugins +------- + + o BREAKING CHANGE: deb plugin have been moved to the bst-plugins-experimental + repository. + Miscellaneous ------------- diff --git a/doc/source/core_plugins.rst b/doc/source/core_plugins.rst index 682c9970f..70691672b 100644 --- a/doc/source/core_plugins.rst +++ b/doc/source/core_plugins.rst @@ -49,7 +49,6 @@ Sources sources/git sources/bzr sources/patch - sources/deb sources/pip diff --git a/requirements/plugin-requirements.in b/requirements/plugin-requirements.in deleted file mode 100644 index a30105a45..000000000 --- a/requirements/plugin-requirements.in +++ /dev/null @@ -1 +0,0 @@ -arpy diff --git a/requirements/plugin-requirements.txt b/requirements/plugin-requirements.txt deleted file mode 100644 index 950062f73..000000000 --- a/requirements/plugin-requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -arpy==1.1.1 -## The following requirements were added by pip freeze: @@ -27,7 +27,7 @@ warn_no_return = True # Ignore missing stubs for third-party packages. # In future, these should be re-enabled if/when stubs for them become available. -[mypy-copyreg,arpy,grpc,pluginbase,psutil,py,pyroaring,pytest,_pytest.*,ruamel] +[mypy-copyreg,grpc,pluginbase,psutil,py,pyroaring,pytest,_pytest.*,ruamel] ignore_missing_imports=True # Ignore missing stubs for Cythonized modules. diff --git a/src/buildstream/plugins/sources/deb.py b/src/buildstream/plugins/sources/deb.py deleted file mode 100644 index a7d06b57c..000000000 --- a/src/buildstream/plugins/sources/deb.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (C) 2017 Codethink Limited -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# -# Authors: -# Phillip Smyth <phillip.smyth@codethink.co.uk> -# Jonathan Maw <jonathan.maw@codethink.co.uk> -# Richard Maw <richard.maw@codethink.co.uk> - -""" -deb - stage files from .deb packages -==================================== - -**Host dependencies:** - - * arpy (python package) - -**Usage:** - -.. code:: yaml - - # Specify the deb source kind - kind: deb - - # Specify the deb url. Using an alias defined in your project - # configuration is encouraged. 'bst source track' will update the - # sha256sum in 'ref' to the downloaded file's sha256sum. - url: upstream:foo.deb - - # Specify the ref. It's a sha256sum of the file you download. - ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b - - # Specify the basedir to return only the specified dir and its children - base-dir: '' - -See :ref:`built-in functionality doumentation <core_source_builtins>` for -details on common configuration options for sources. -""" - -import tarfile -from contextlib import contextmanager -import arpy - -from .tar import TarSource - - -class DebSource(TarSource): - # pylint: disable=attribute-defined-outside-init - - def configure(self, node): - super().configure(node) - - self.base_dir = node.get_str("base-dir", None) - - def preflight(self): - return - - @contextmanager - def _get_tar(self): - with open(self._get_mirror_file(), "rb") as deb_file: - arpy_archive = arpy.Archive(fileobj=deb_file) - arpy_archive.read_all_headers() - data_tar_arpy = [v for k, v in arpy_archive.archived_files.items() if b"data.tar" in k][0] - # ArchiveFileData is not enough like a file object for tarfile to use. - # Monkey-patching a seekable method makes it close enough for TarFile to open. - data_tar_arpy.seekable = lambda *args: True - tar = tarfile.open(fileobj=data_tar_arpy, mode="r:*") - yield tar - - -def setup(): - return DebSource diff --git a/src/buildstream/testing/_utils/site.py b/src/buildstream/testing/_utils/site.py index 9fbddf13e..0dfbce222 100644 --- a/src/buildstream/testing/_utils/site.py +++ b/src/buildstream/testing/_utils/site.py @@ -59,13 +59,6 @@ try: except ProgramNotFoundError: HAVE_LZIP = False -try: - import arpy # pylint: disable=unused-import - - HAVE_ARPY = True -except ImportError: - HAVE_ARPY = False - casd_path = utils.get_host_tool("buildbox-casd") CASD_SEPARATE_USER = bool(os.stat(casd_path).st_mode & stat.S_ISUID) del casd_path diff --git a/tests/sources/deb.py b/tests/sources/deb.py deleted file mode 100644 index 96060a1a4..000000000 --- a/tests/sources/deb.py +++ /dev/null @@ -1,166 +0,0 @@ -# Pylint doesn't play well with fixtures and dependency injection from pytest -# pylint: disable=redefined-outer-name - -import os -import shutil - -import pytest - -from buildstream._exceptions import ErrorDomain -from buildstream import _yaml -from buildstream.testing import cli # pylint: disable=unused-import -from buildstream.testing._utils.site import HAVE_ARPY -from . import list_dir_contents - -DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "deb",) - -deb_name = "a_deb.deb" - - -def generate_project(project_dir, tmpdir): - project_file = os.path.join(project_dir, "project.conf") - _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": "file:///" + str(tmpdir)}}, project_file) - - -def _copy_deb(start_location, tmpdir): - source = os.path.join(start_location, deb_name) - destination = os.path.join(str(tmpdir), deb_name) - shutil.copyfile(source, destination) - - -# Test that without ref, consistency is set appropriately. -@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref")) -def test_no_ref(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, tmpdir) - assert cli.get_element_state(project, "target.bst") == "no reference" - - -# Test that when I fetch a nonexistent URL, errors are handled gracefully and a retry is performed. -@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) -def test_fetch_bad_url(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, tmpdir) - - # Try to fetch it - result = cli.run(project=project, args=["source", "fetch", "target.bst"]) - assert "FAILURE Try #" in result.stderr - result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.SOURCE, None) - - -@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) -def test_fetch_bad_ref(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, tmpdir) - - # Copy test deb to tmpdir - _copy_deb(DATA_DIR, tmpdir) - - # Try to fetch it - result = cli.run(project=project, args=["source", "fetch", "target.bst"]) - result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.SOURCE, None) - - -# Test that when tracking with a ref set, there is a warning -@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) -def test_track_warning(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, tmpdir) - - # Copy test deb to tmpdir - _copy_deb(DATA_DIR, tmpdir) - - # Track it - result = cli.run(project=project, args=["source", "track", "target.bst"]) - result.assert_success() - assert "Potential man-in-the-middle attack!" in result.stderr - - -# Test that a staged checkout matches what was tarred up, with the default first subdir -@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) -def test_stage_default_basedir(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, tmpdir) - checkoutdir = os.path.join(str(tmpdir), "checkout") - - # Copy test deb to tmpdir - _copy_deb(DATA_DIR, tmpdir) - - # Track, fetch, build, checkout - result = cli.run(project=project, args=["source", "track", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["source", "fetch", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["build", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) - result.assert_success() - - # Check that the content of the first directory is checked out (base-dir: '') - original_dir = os.path.join(str(datafiles), "content") - original_contents = list_dir_contents(original_dir) - checkout_contents = list_dir_contents(checkoutdir) - assert checkout_contents == original_contents - - -# Test that a staged checkout matches what was tarred up, with an empty base-dir -@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-basedir")) -def test_stage_no_basedir(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, tmpdir) - checkoutdir = os.path.join(str(tmpdir), "checkout") - - # Copy test deb to tmpdir - _copy_deb(DATA_DIR, tmpdir) - - # Track, fetch, build, checkout - result = cli.run(project=project, args=["source", "track", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["source", "fetch", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["build", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) - result.assert_success() - - # Check that the full content of the tarball is checked out (base-dir: '') - original_dir = os.path.join(str(datafiles), "content") - original_contents = list_dir_contents(original_dir) - checkout_contents = list_dir_contents(checkoutdir) - assert checkout_contents == original_contents - - -# Test that a staged checkout matches what was tarred up, with an explicit basedir -@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, "explicit-basedir")) -def test_stage_explicit_basedir(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, tmpdir) - checkoutdir = os.path.join(str(tmpdir), "checkout") - - # Copy test deb to tmpdir - _copy_deb(DATA_DIR, tmpdir) - - # Track, fetch, build, checkout - result = cli.run(project=project, args=["source", "track", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["source", "fetch", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["build", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) - result.assert_success() - - # Check that the content of the first directory is checked out (base-dir: '') - original_dir = os.path.join(str(datafiles), "content") - original_contents = list_dir_contents(original_dir) - checkout_contents = list_dir_contents(checkoutdir) - assert checkout_contents == original_contents diff --git a/tests/sources/deb/a_deb.deb b/tests/sources/deb/a_deb.deb Binary files differdeleted file mode 100644 index c8fef9117..000000000 --- a/tests/sources/deb/a_deb.deb +++ /dev/null diff --git a/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/README b/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/README deleted file mode 100644 index e69de29bb..000000000 --- a/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/README +++ /dev/null diff --git a/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/changelog.Debian.gz b/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/changelog.Debian.gz Binary files differdeleted file mode 100644 index a9090a51c..000000000 --- a/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/changelog.Debian.gz +++ /dev/null diff --git a/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/copyright b/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/copyright deleted file mode 100644 index e69de29bb..000000000 --- a/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/copyright +++ /dev/null diff --git a/tests/sources/deb/explicit-basedir/content/share/lua/5.1/clod.lua b/tests/sources/deb/explicit-basedir/content/share/lua/5.1/clod.lua deleted file mode 100644 index e69de29bb..000000000 --- a/tests/sources/deb/explicit-basedir/content/share/lua/5.1/clod.lua +++ /dev/null diff --git a/tests/sources/deb/explicit-basedir/content/share/lua/5.2/clod.lua b/tests/sources/deb/explicit-basedir/content/share/lua/5.2/clod.lua deleted file mode 120000 index 79531e49d..000000000 --- a/tests/sources/deb/explicit-basedir/content/share/lua/5.2/clod.lua +++ /dev/null @@ -1 +0,0 @@ -../5.1/clod.lua
\ No newline at end of file diff --git a/tests/sources/deb/explicit-basedir/target.bst b/tests/sources/deb/explicit-basedir/target.bst deleted file mode 100644 index a75881151..000000000 --- a/tests/sources/deb/explicit-basedir/target.bst +++ /dev/null @@ -1,7 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: deb - url: tmpdir:/a_deb.deb - ref: foo - base-dir: 'usr' diff --git a/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/README b/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/README deleted file mode 100644 index e69de29bb..000000000 --- a/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/README +++ /dev/null diff --git a/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/changelog.Debian.gz b/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/changelog.Debian.gz Binary files differdeleted file mode 100644 index 040dd15a7..000000000 --- a/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/changelog.Debian.gz +++ /dev/null diff --git a/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/copyright b/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/copyright deleted file mode 100644 index e69de29bb..000000000 --- a/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/copyright +++ /dev/null diff --git a/tests/sources/deb/fetch/content/usr/share/lua/5.1/clod.lua b/tests/sources/deb/fetch/content/usr/share/lua/5.1/clod.lua deleted file mode 100644 index e69de29bb..000000000 --- a/tests/sources/deb/fetch/content/usr/share/lua/5.1/clod.lua +++ /dev/null diff --git a/tests/sources/deb/fetch/content/usr/share/lua/5.2/clod.lua b/tests/sources/deb/fetch/content/usr/share/lua/5.2/clod.lua deleted file mode 120000 index 79531e49d..000000000 --- a/tests/sources/deb/fetch/content/usr/share/lua/5.2/clod.lua +++ /dev/null @@ -1 +0,0 @@ -../5.1/clod.lua
\ No newline at end of file diff --git a/tests/sources/deb/fetch/target-lz.bst b/tests/sources/deb/fetch/target-lz.bst deleted file mode 100644 index b0569129e..000000000 --- a/tests/sources/deb/fetch/target-lz.bst +++ /dev/null @@ -1,6 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: tar - url: tmpdir:/a.tar.lz - ref: foo diff --git a/tests/sources/deb/fetch/target.bst b/tests/sources/deb/fetch/target.bst deleted file mode 100644 index 919e66a37..000000000 --- a/tests/sources/deb/fetch/target.bst +++ /dev/null @@ -1,6 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: deb - url: tmpdir:/a_deb.deb - ref: foo diff --git a/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/README b/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/README deleted file mode 100644 index e69de29bb..000000000 --- a/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/README +++ /dev/null diff --git a/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/changelog.Debian.gz b/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/changelog.Debian.gz Binary files differdeleted file mode 100644 index be777e65f..000000000 --- a/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/changelog.Debian.gz +++ /dev/null diff --git a/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/copyright b/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/copyright deleted file mode 100644 index e69de29bb..000000000 --- a/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/copyright +++ /dev/null diff --git a/tests/sources/deb/no-basedir/content/usr/share/lua/5.1/clod.lua b/tests/sources/deb/no-basedir/content/usr/share/lua/5.1/clod.lua deleted file mode 100644 index e69de29bb..000000000 --- a/tests/sources/deb/no-basedir/content/usr/share/lua/5.1/clod.lua +++ /dev/null diff --git a/tests/sources/deb/no-basedir/content/usr/share/lua/5.2/clod.lua b/tests/sources/deb/no-basedir/content/usr/share/lua/5.2/clod.lua deleted file mode 120000 index 79531e49d..000000000 --- a/tests/sources/deb/no-basedir/content/usr/share/lua/5.2/clod.lua +++ /dev/null @@ -1 +0,0 @@ -../5.1/clod.lua
\ No newline at end of file diff --git a/tests/sources/deb/no-basedir/target.bst b/tests/sources/deb/no-basedir/target.bst deleted file mode 100644 index 69fcd700c..000000000 --- a/tests/sources/deb/no-basedir/target.bst +++ /dev/null @@ -1,7 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: deb - url: tmpdir:/a_deb.deb - ref: foo - base-dir: '' diff --git a/tests/sources/deb/no-ref/a/b/d b/tests/sources/deb/no-ref/a/b/d deleted file mode 100644 index 4bcfe98e6..000000000 --- a/tests/sources/deb/no-ref/a/b/d +++ /dev/null @@ -1 +0,0 @@ -d diff --git a/tests/sources/deb/no-ref/a/c b/tests/sources/deb/no-ref/a/c deleted file mode 100644 index f2ad6c76f..000000000 --- a/tests/sources/deb/no-ref/a/c +++ /dev/null @@ -1 +0,0 @@ -c diff --git a/tests/sources/deb/no-ref/target.bst b/tests/sources/deb/no-ref/target.bst deleted file mode 100644 index 89b639662..000000000 --- a/tests/sources/deb/no-ref/target.bst +++ /dev/null @@ -1,5 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: deb - url: tmpdir:/a_deb.deb @@ -35,10 +35,9 @@ commands_post: deps = py{35,36,37,38}: -rrequirements/requirements.txt py{35,36,37,38}: -rrequirements/dev-requirements.txt - py{35,36,37,38}: -rrequirements/plugin-requirements.txt # Install external plugins for plugin tests - py{35,36,37,38}-plugins: git+https://gitlab.com/buildstream/bst-plugins-experimental.git@{env:BST_PLUGINS_EXPERIMENTAL_VERSION:{[config]BST_PLUGINS_EXPERIMENTAL_VERSION}}#egg=bst_plugins_experimental[ostree] + py{35,36,37,38}-plugins: git+https://gitlab.com/buildstream/bst-plugins-experimental.git@{env:BST_PLUGINS_EXPERIMENTAL_VERSION:{[config]BST_PLUGINS_EXPERIMENTAL_VERSION}}#egg=bst_plugins_experimental[ostree,deb] # Only require coverage and pytest-cov when using it !nocover: -rrequirements/cov-requirements.txt @@ -149,7 +148,6 @@ commands = deps = -rrequirements/requirements.txt -rrequirements/dev-requirements.txt - -rrequirements/plugin-requirements.txt # # Running static type checkers @@ -175,7 +173,6 @@ deps = sphinx_rtd_theme >= 0.4.2 pytest -rrequirements/requirements.txt - -rrequirements/plugin-requirements.txt git+https://gitlab.com/BuildStream/bst-plugins-experimental.git@be5ac19e5062bc23a46ed8ce7aa2958a2653c917#egg=bst_plugins_experimental[ostree] passenv = BST_FORCE_SESSION_REBUILD @@ -204,7 +201,6 @@ commands = {posargs} deps = -rrequirements/requirements.txt -rrequirements/dev-requirements.txt - -rrequirements/plugin-requirements.txt whitelist_externals = * @@ -225,7 +221,6 @@ commands = pytest --basetemp {envtmpdir} --ignore tests -k "{posargs}" deps = -rrequirements/requirements.txt -rrequirements/dev-requirements.txt - -rrequirements/plugin-requirements.txt # When building using PEP518 and 517, we don't want default dependencies # installed by the base environment. |