diff options
author | Javier Jardón <jjardon@gnome.org> | 2019-11-29 23:40:38 +0900 |
---|---|---|
committer | Javier Jardón <jjardon@gnome.org> | 2020-01-27 21:29:35 +0900 |
commit | 1073ae660ccba5fcc76157f04ff13facc084530d (patch) | |
tree | 10f79f2fff7c29292c624881ca9f9b40ea265508 | |
parent | 1c3baa29dac7ee49a7d6b44d906d419645e22b4d (diff) | |
download | buildstream-jjardon/tar_plugin_move.tar.gz |
Remove "tar" pluginjjardon/tar_plugin_move
Use the one from bst-plugins-experimental instead
45 files changed, 49 insertions, 843 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 38a55053b..1bae0dc66 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -351,7 +351,7 @@ docs: image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-${DOCKER_IMAGE_VERSION} variables: BST_EXT_URL: git+https://gitlab.com/BuildStream/bst-plugins-experimental.git - BST_EXT_REF: 0.12.0-173-gbe5ac19#egg=bst_plugins_experimental[ostree,cargo] + BST_EXT_REF: 1.93.0-0-g4dea6cfc9c5f64e4b8e5fe5bb76f34c77af891a0#egg=bst_plugins_experimental[ostree,cargo,tar] FD_SDK_REF: freedesktop-sdk-19.08.5-buildstream2-0-g22588e2f41acecbcfc555942eb3086296bc14c6c before_script: - | diff --git a/doc/examples/autotools/project.conf b/doc/examples/autotools/project.conf index 6c1142340..617de5d5d 100644 --- a/doc/examples/autotools/project.conf +++ b/doc/examples/autotools/project.conf @@ -11,3 +11,9 @@ element-path: elements aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ gnu: http://ftpmirror.gnu.org/gnu/automake/ + +plugins: +- origin: pip + package-name: bst-plugins-experimental + sources: + tar: 0 diff --git a/doc/examples/developing/project.conf b/doc/examples/developing/project.conf index 3b0920991..ec3bae24f 100644 --- a/doc/examples/developing/project.conf +++ b/doc/examples/developing/project.conf @@ -10,3 +10,9 @@ element-path: elements # Define an alias for our alpine tarball aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ + +plugins: +- origin: pip + package-name: bst-plugins-experimental + sources: + tar: 0 diff --git a/doc/examples/integration-commands/project.conf b/doc/examples/integration-commands/project.conf index c6c59906b..7fffef058 100644 --- a/doc/examples/integration-commands/project.conf +++ b/doc/examples/integration-commands/project.conf @@ -10,3 +10,9 @@ element-path: elements # Define an alias for our alpine tarball aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ + +plugins: +- origin: pip + package-name: bst-plugins-experimental + sources: + tar: 0 diff --git a/doc/examples/running-commands/project.conf b/doc/examples/running-commands/project.conf index aafb6a9f2..b5ef5645d 100644 --- a/doc/examples/running-commands/project.conf +++ b/doc/examples/running-commands/project.conf @@ -10,3 +10,9 @@ element-path: elements # Define an alias for our alpine tarball aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ + +plugins: +- origin: pip + package-name: bst-plugins-experimental + sources: + tar: 0 diff --git a/doc/source/core_plugins.rst b/doc/source/core_plugins.rst index 70691672b..bfd1487db 100644 --- a/doc/source/core_plugins.rst +++ b/doc/source/core_plugins.rst @@ -44,7 +44,6 @@ Sources sources/local sources/remote - sources/tar sources/zip sources/git sources/bzr diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py deleted file mode 100644 index fc4f5736a..000000000 --- a/src/buildstream/plugins/sources/tar.py +++ /dev/null @@ -1,253 +0,0 @@ -# -# Copyright (C) 2017 Codethink Limited -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# -# Authors: -# Jonathan Maw <jonathan.maw@codethink.co.uk> - -""" -tar - stage files from tar archives -=================================== - -**Host dependencies:** - - * lzip (for .tar.lz files) - -**Usage:** - -.. code:: yaml - - # Specify the tar source kind - kind: tar - - # Specify the tar url. Using an alias defined in your project - # configuration is encouraged. 'bst source track' will update the - # sha256sum in 'ref' to the downloaded file's sha256sum. - url: upstream:foo.tar - - # Specify the ref. It's a sha256sum of the file you download. - ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b - - # Specify a glob pattern to indicate the base directory to extract - # from the tarball. The first matching directory will be used. - # - # Note that this is '*' by default since most standard release - # tarballs contain a self named subdirectory at the root which - # contains the files one normally wants to extract to build. - # - # To extract the root of the tarball directly, this can be set - # to an empty string. - base-dir: '*' - -See :ref:`built-in functionality doumentation <core_source_builtins>` for -details on common configuration options for sources. -""" - -import os -import tarfile -from contextlib import contextmanager -from tempfile import TemporaryFile - -from buildstream import SourceError -from buildstream import utils - -from ._downloadablefilesource import DownloadableFileSource - - -class ReadableTarInfo(tarfile.TarInfo): - """ - The goal is to override `TarFile`'s `extractall` semantics by ensuring that on extraction, the - files are readable by the owner of the file. This is done by overriding the accessor for the - `mode` attribute in `TarInfo`, the class that encapsulates the internal meta-data of the tarball, - so that the owner-read bit is always set. - """ - - @property - def mode(self): - # Respect umask instead of the file mode stored in the archive. - # The only bit used from the embedded mode is the executable bit for files. - umask = utils.get_umask() - if self.isdir() or bool(self.__permission | 0o100): - return 0o777 & ~umask - else: - return 0o666 & ~umask - - @mode.setter - def mode(self, permission): - self.__permission = permission # pylint: disable=attribute-defined-outside-init - - -class TarSource(DownloadableFileSource): - # pylint: disable=attribute-defined-outside-init - - def configure(self, node): - super().configure(node) - - self.base_dir = node.get_str("base-dir", "*") - node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ["base-dir"]) - - def preflight(self): - self.host_lzip = None - if self.url.endswith(".lz"): - self.host_lzip = utils.get_host_tool("lzip") - - def get_unique_key(self): - return super().get_unique_key() + [self.base_dir] - - @contextmanager - def _run_lzip(self): - assert self.host_lzip - with TemporaryFile() as lzip_stdout: - with open(self._get_mirror_file(), "r") as lzip_file: - self.call([self.host_lzip, "-d"], stdin=lzip_file, stdout=lzip_stdout) - - lzip_stdout.seek(0, 0) - yield lzip_stdout - - @contextmanager - def _get_tar(self): - if self.url.endswith(".lz"): - with self._run_lzip() as lzip_dec: - with tarfile.open(fileobj=lzip_dec, mode="r:", tarinfo=ReadableTarInfo) as tar: - yield tar - else: - with tarfile.open(self._get_mirror_file(), tarinfo=ReadableTarInfo) as tar: - yield tar - - def stage(self, directory): - try: - with self._get_tar() as tar: - base_dir = None - if self.base_dir: - base_dir = self._find_base_dir(tar, self.base_dir) - - if base_dir: - tar.extractall(path=directory, members=self._extract_members(tar, base_dir, directory)) - else: - tar.extractall(path=directory) - - except (tarfile.TarError, OSError) as e: - raise SourceError("{}: Error staging source: {}".format(self, e)) from e - - # Override and translate which filenames to extract - def _extract_members(self, tar, base_dir, target_dir): - - # Assert that a tarfile is safe to extract; specifically, make - # sure that we don't do anything outside of the target - # directory (this is possible, if, say, someone engineered a - # tarfile to contain paths that start with ..). - def assert_safe(member): - final_path = os.path.abspath(os.path.join(target_dir, member.path)) - if not final_path.startswith(target_dir): - raise SourceError( - "{}: Tarfile attempts to extract outside the staging area: " - "{} -> {}".format(self, member.path, final_path) - ) - - if member.islnk(): - linked_path = os.path.abspath(os.path.join(target_dir, member.linkname)) - if not linked_path.startswith(target_dir): - raise SourceError( - "{}: Tarfile attempts to hardlink outside the staging area: " - "{} -> {}".format(self, member.path, final_path) - ) - - # Don't need to worry about symlinks because they're just - # files here and won't be able to do much harm once we are - # in a sandbox. - - if not base_dir.endswith(os.sep): - base_dir = base_dir + os.sep - - L = len(base_dir) - for member in tar.getmembers(): - - # First, ensure that a member never starts with `./` - if member.path.startswith("./"): - member.path = member.path[2:] - if member.islnk() and member.linkname.startswith("./"): - member.linkname = member.linkname[2:] - - # Now extract only the paths which match the normalized path - if member.path.startswith(base_dir): - # Hardlinks are smart and collapse into the "original" - # when their counterpart doesn't exist. This means we - # only need to modify links to files whose location we - # change. - # - # Since we assert that we're not linking to anything - # outside the target directory, this should only ever - # be able to link to things inside the target - # directory, so we should cover all bases doing this. - # - if member.islnk() and member.linkname.startswith(base_dir): - member.linkname = member.linkname[L:] - - member.path = member.path[L:] - - assert_safe(member) - yield member - - # We want to iterate over all paths of a tarball, but getmembers() - # is not enough because some tarballs simply do not contain the leading - # directory paths for the archived files. - def _list_tar_paths(self, tar): - - visited = set() - for member in tar.getmembers(): - - # Remove any possible leading './', offer more consistent behavior - # across tarballs encoded with or without a leading '.' - member_name = member.name.lstrip("./") - - if not member.isdir(): - - # Loop over the components of a path, for a path of a/b/c/d - # we will first visit 'a', then 'a/b' and then 'a/b/c', excluding - # the final component - components = member_name.split("/") - for i in range(len(components) - 1): - dir_component = "/".join([components[j] for j in range(i + 1)]) - if dir_component not in visited: - visited.add(dir_component) - try: - # Dont yield directory members which actually do - # exist in the archive - _ = tar.getmember(dir_component) - except KeyError: - if dir_component != ".": - yield dir_component - - continue - - # Avoid considering the '.' directory, if any is included in the archive - # this is to avoid the default 'base-dir: *' value behaving differently - # depending on whether the tarball was encoded with a leading '.' or not - if member_name == ".": - continue - - yield member_name - - def _find_base_dir(self, tar, pattern): - paths = self._list_tar_paths(tar) - matches = sorted(list(utils.glob(paths, pattern))) - if not matches: - raise SourceError("{}: Could not find base directory matching pattern: {}".format(self, pattern)) - - return matches[0] - - -def setup(): - return TarSource diff --git a/src/buildstream/testing/_sourcetests/project/project.conf b/src/buildstream/testing/_sourcetests/project/project.conf index 05b68bfeb..96291e9b3 100644 --- a/src/buildstream/testing/_sourcetests/project/project.conf +++ b/src/buildstream/testing/_sourcetests/project/project.conf @@ -25,3 +25,9 @@ split-rules: fatal-warnings: - bad-element-suffix - bad-characters-in-name + +plugins: +- origin: pip + package-name: bst-plugins-experimental + sources: + tar: 0 diff --git a/src/buildstream/testing/_utils/site.py b/src/buildstream/testing/_utils/site.py index 0dfbce222..a04f65210 100644 --- a/src/buildstream/testing/_utils/site.py +++ b/src/buildstream/testing/_utils/site.py @@ -53,12 +53,6 @@ except ProgramNotFoundError: HAVE_BWRAP = False HAVE_BWRAP_JSON_STATUS = False -try: - utils.get_host_tool("lzip") - HAVE_LZIP = True -except ProgramNotFoundError: - HAVE_LZIP = False - casd_path = utils.get_host_tool("buildbox-casd") CASD_SEPARATE_USER = bool(os.stat(casd_path).st_mode & stat.S_ISUID) del casd_path diff --git a/tests/cachekey/project/sources/tar1.bst b/tests/cachekey/project/sources/tar1.bst deleted file mode 100644 index 1dbf27cd8..000000000 --- a/tests/cachekey/project/sources/tar1.bst +++ /dev/null @@ -1,5 +0,0 @@ -kind: import -sources: -- kind: tar - url: https://example.com/releases/1.4/foo-1.4.5.tar.gz - ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected deleted file mode 100644 index 153138437..000000000 --- a/tests/cachekey/project/sources/tar1.expected +++ /dev/null @@ -1 +0,0 @@ -6b550e20ab7b8a11912ca14171e39c76badf7fa161a01c83d817c789b84e45c3 diff --git a/tests/cachekey/project/sources/tar2.bst b/tests/cachekey/project/sources/tar2.bst deleted file mode 100644 index e357be392..000000000 --- a/tests/cachekey/project/sources/tar2.bst +++ /dev/null @@ -1,6 +0,0 @@ -kind: import -sources: -- kind: tar - url: https://example.com/releases/1.4/foo-1.4.5.tar.gz - ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b - base-dir: src diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected deleted file mode 100644 index 6440dfaed..000000000 --- a/tests/cachekey/project/sources/tar2.expected +++ /dev/null @@ -1 +0,0 @@ -f890b611cc83036b9c52dddf4eb2a02ccac5a73ae3ddcb34586406d7deba5a11 diff --git a/tests/cachekey/project/target.bst b/tests/cachekey/project/target.bst index cabf3f7bf..5d385de34 100644 --- a/tests/cachekey/project/target.bst +++ b/tests/cachekey/project/target.bst @@ -16,8 +16,6 @@ depends: - sources/pip1.bst - sources/remote1.bst - sources/remote2.bst -- sources/tar1.bst -- sources/tar2.bst - sources/zip1.bst - sources/zip2.bst - elements/build1.bst diff --git a/tests/conftest.py b/tests/conftest.py index 8d33fa024..4488bb3cd 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,7 +32,6 @@ from buildstream.testing.integration import integration_cache # pylint: disable from tests.testutils.repo.git import Git from tests.testutils.repo.bzr import Bzr -from tests.testutils.repo.tar import Tar from tests.testutils.repo.zip import Zip @@ -112,7 +111,6 @@ def remote_services(request): ################################################# register_repo_kind("git", Git, None) register_repo_kind("bzr", Bzr, None) -register_repo_kind("tar", Tar, None) register_repo_kind("zip", Zip, None) diff --git a/tests/integration/project/project.conf b/tests/integration/project/project.conf index ddfe47b6d..f97b613c9 100644 --- a/tests/integration/project/project.conf +++ b/tests/integration/project/project.conf @@ -21,3 +21,9 @@ split-rules: /tests - | /tests/* + +plugins: +- origin: pip + package-name: bst-plugins-experimental + sources: + tar: 0 diff --git a/tests/remoteexecution/project/project.conf b/tests/remoteexecution/project/project.conf index ddfe47b6d..63ca31988 100644 --- a/tests/remoteexecution/project/project.conf +++ b/tests/remoteexecution/project/project.conf @@ -21,3 +21,8 @@ split-rules: /tests - | /tests/* +plugins: +- origin: pip + package-name: bst-plugins-experimental + sources: + tar: 0 diff --git a/tests/sandboxes/project/project.conf b/tests/sandboxes/project/project.conf index ddfe47b6d..63ca31988 100644 --- a/tests/sandboxes/project/project.conf +++ b/tests/sandboxes/project/project.conf @@ -21,3 +21,8 @@ split-rules: /tests - | /tests/* +plugins: +- origin: pip + package-name: bst-plugins-experimental + sources: + tar: 0 diff --git a/tests/sources/tar.py b/tests/sources/tar.py deleted file mode 100644 index 8ac627492..000000000 --- a/tests/sources/tar.py +++ /dev/null @@ -1,471 +0,0 @@ -# Pylint doesn't play well with fixtures and dependency injection from pytest -# pylint: disable=redefined-outer-name - -import os -from shutil import copyfile -import subprocess -import tarfile -import tempfile -import urllib.parse - -import pytest - -from buildstream import utils -from buildstream.exceptions import ErrorDomain -from buildstream.testing import generate_project, generate_element -from buildstream.testing import cli # pylint: disable=unused-import -from buildstream.testing._utils.site import HAVE_LZIP -from tests.testutils.file_server import create_file_server -from . import list_dir_contents - -DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tar",) - - -def _assemble_tar(workingdir, srcdir, dstfile): - old_dir = os.getcwd() - os.chdir(workingdir) - with tarfile.open(dstfile, "w:gz") as tar: - tar.add(srcdir) - os.chdir(old_dir) - - -def _assemble_tar_lz(workingdir, srcdir, dstfile): - old_dir = os.getcwd() - os.chdir(workingdir) - with tempfile.TemporaryFile() as uncompressed: - with tarfile.open(fileobj=uncompressed, mode="w:") as tar: - tar.add(srcdir) - uncompressed.seek(0, 0) - with open(dstfile, "wb") as dst: - subprocess.call(["lzip"], stdin=uncompressed, stdout=dst) - os.chdir(old_dir) - - -# Test that without ref, consistency is set appropriately. -@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref")) -def test_no_ref(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) - assert cli.get_element_state(project, "target.bst") == "no reference" - - -# Test that when I fetch a nonexistent URL, errors are handled gracefully and a retry is performed. -@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) -def test_fetch_bad_url(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) - - # Try to fetch it - result = cli.run(project=project, args=["source", "fetch", "target.bst"]) - assert "FAILURE Try #" in result.stderr - result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.SOURCE, None) - - -# Test that when I fetch with an invalid ref, it fails. -@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) -def test_fetch_bad_ref(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) - - # Create a local tar - src_tar = os.path.join(str(tmpdir), "a.tar.gz") - _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) - - # Try to fetch it - result = cli.run(project=project, args=["source", "fetch", "target.bst"]) - result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.SOURCE, None) - - -# Test that when tracking with a ref set, there is a warning -@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) -def test_track_warning(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) - - # Create a local tar - src_tar = os.path.join(str(tmpdir), "a.tar.gz") - _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) - - # Track it - result = cli.run(project=project, args=["source", "track", "target.bst"]) - result.assert_success() - assert "Potential man-in-the-middle attack!" in result.stderr - - -# Test that a staged checkout matches what was tarred up, with the default first subdir -@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) -@pytest.mark.parametrize("srcdir", ["a", "./a"]) -def test_stage_default_basedir(cli, tmpdir, datafiles, srcdir): - project = str(datafiles) - generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) - checkoutdir = os.path.join(str(tmpdir), "checkout") - - # Create a local tar - src_tar = os.path.join(str(tmpdir), "a.tar.gz") - _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar) - - # Track, fetch, build, checkout - result = cli.run(project=project, args=["source", "track", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["source", "fetch", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["build", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) - result.assert_success() - - # Check that the content of the first directory is checked out (base-dir: '*') - original_dir = os.path.join(str(datafiles), "content", "a") - original_contents = list_dir_contents(original_dir) - checkout_contents = list_dir_contents(checkoutdir) - assert checkout_contents == original_contents - - -# Test that a staged checkout matches what was tarred up, with an empty base-dir -@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-basedir")) -@pytest.mark.parametrize("srcdir", ["a", "./a"]) -def test_stage_no_basedir(cli, tmpdir, datafiles, srcdir): - project = str(datafiles) - generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) - checkoutdir = os.path.join(str(tmpdir), "checkout") - - # Create a local tar - src_tar = os.path.join(str(tmpdir), "a.tar.gz") - _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar) - - # Track, fetch, build, checkout - result = cli.run(project=project, args=["source", "track", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["source", "fetch", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["build", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) - result.assert_success() - - # Check that the full content of the tarball is checked out (base-dir: '') - original_dir = os.path.join(str(datafiles), "content") - original_contents = list_dir_contents(original_dir) - checkout_contents = list_dir_contents(checkoutdir) - assert checkout_contents == original_contents - - -# Test that a staged checkout matches what was tarred up, with an explicit basedir -@pytest.mark.datafiles(os.path.join(DATA_DIR, "explicit-basedir")) -@pytest.mark.parametrize("srcdir", ["a", "./a"]) -def test_stage_explicit_basedir(cli, tmpdir, datafiles, srcdir): - project = str(datafiles) - generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) - checkoutdir = os.path.join(str(tmpdir), "checkout") - - # Create a local tar - src_tar = os.path.join(str(tmpdir), "a.tar.gz") - _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar) - - # Track, fetch, build, checkout - result = cli.run(project=project, args=["source", "track", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["source", "fetch", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["build", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) - result.assert_success() - - # Check that the content of the first directory is checked out (base-dir: '*') - original_dir = os.path.join(str(datafiles), "content", "a") - original_contents = list_dir_contents(original_dir) - checkout_contents = list_dir_contents(checkoutdir) - assert checkout_contents == original_contents - - -# Test that we succeed to extract tarballs with hardlinks when stripping the -# leading paths -@pytest.mark.datafiles(os.path.join(DATA_DIR, "contains-links")) -def test_stage_contains_links(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) - checkoutdir = os.path.join(str(tmpdir), "checkout") - - # Create a local tar - src_tar = os.path.join(str(tmpdir), "a.tar.gz") - - # Create a hardlink, we wont trust git to store that info for us - os.makedirs(os.path.join(str(datafiles), "content", "base-directory", "subdir2"), exist_ok=True) - file1 = os.path.join(str(datafiles), "content", "base-directory", "subdir1", "file.txt") - file2 = os.path.join(str(datafiles), "content", "base-directory", "subdir2", "file.txt") - os.link(file1, file2) - - _assemble_tar(os.path.join(str(datafiles), "content"), "base-directory", src_tar) - - # Track, fetch, build, checkout - result = cli.run(project=project, args=["source", "track", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["source", "fetch", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["build", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) - result.assert_success() - - # Check that the content of the first directory is checked out (base-dir: '*') - original_dir = os.path.join(str(datafiles), "content", "base-directory") - original_contents = list_dir_contents(original_dir) - checkout_contents = list_dir_contents(checkoutdir) - assert checkout_contents == original_contents - - -@pytest.mark.skipif(not HAVE_LZIP, reason="lzip is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) -@pytest.mark.parametrize("srcdir", ["a", "./a"]) -def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir): - project = str(datafiles) - generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) - checkoutdir = os.path.join(str(tmpdir), "checkout") - - # Create a local tar - src_tar = os.path.join(str(tmpdir), "a.tar.lz") - _assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar) - - # Track, fetch, build, checkout - result = cli.run(project=project, args=["source", "track", "target-lz.bst"]) - result.assert_success() - result = cli.run(project=project, args=["source", "fetch", "target-lz.bst"]) - result.assert_success() - result = cli.run(project=project, args=["build", "target-lz.bst"]) - result.assert_success() - result = cli.run(project=project, args=["artifact", "checkout", "target-lz.bst", "--directory", checkoutdir]) - result.assert_success() - - # Check that the content of the first directory is checked out (base-dir: '*') - original_dir = os.path.join(str(datafiles), "content", "a") - original_contents = list_dir_contents(original_dir) - checkout_contents = list_dir_contents(checkoutdir) - assert checkout_contents == original_contents - - -# Test that tarballs with read-only files work -# a - contains read-only files in a writable directory -# b - root directory has read-only permission -# c - contains one file that has no read nor write permissions. Base-dir set to '' to extract root of tarball -@pytest.mark.datafiles(os.path.join(DATA_DIR, "read-only")) -@pytest.mark.parametrize("tar_name, base_dir", [("a", "*"), ("b", "*"), ("c", "")]) -def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir): - try: - project = str(datafiles) - generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) - - tar_file = "{}.tar.gz".format(tar_name) - - generate_element( - project, - "target.bst", - { - "kind": "import", - "sources": [{"kind": "tar", "url": "tmpdir:/{}".format(tar_file), "ref": "foo", "base-dir": base_dir}], - }, - ) - - # Get the tarball in tests/sources/tar/read-only/content - # - # NOTE that we need to do this because tarfile.open and tar.add() - # are packing the tar up with writeable files and dirs - tarball = os.path.join(str(datafiles), "content", tar_file) - if not os.path.exists(tarball): - raise FileNotFoundError("{} does not exist".format(tarball)) - copyfile(tarball, os.path.join(str(tmpdir), tar_file)) - - # Because this test can potentially leave directories behind - # which are difficult to remove, ask buildstream to use - # our temp directory, so we can clean up. - tmpdir_str = str(tmpdir) - if not tmpdir_str.endswith(os.path.sep): - tmpdir_str += os.path.sep - env = {"TMP": tmpdir_str} - - # Track, fetch, build, checkout - result = cli.run(project=project, args=["source", "track", "target.bst"], env=env) - result.assert_success() - result = cli.run(project=project, args=["source", "fetch", "target.bst"], env=env) - result.assert_success() - result = cli.run(project=project, args=["build", "target.bst"], env=env) - result.assert_success() - - finally: - utils._force_rmtree(str(tmpdir)) - - -@pytest.mark.parametrize("server_type", ("FTP", "HTTP")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) -def test_use_netrc(cli, datafiles, server_type, tmpdir): - file_server_files = os.path.join(str(tmpdir), "file_server") - fake_home = os.path.join(str(tmpdir), "fake_home") - os.makedirs(file_server_files, exist_ok=True) - os.makedirs(fake_home, exist_ok=True) - project = str(datafiles) - checkoutdir = os.path.join(str(tmpdir), "checkout") - - os.environ["HOME"] = fake_home - with open(os.path.join(fake_home, ".netrc"), "wb") as f: - os.fchmod(f.fileno(), 0o700) - f.write(b"machine 127.0.0.1\n") - f.write(b"login testuser\n") - f.write(b"password 12345\n") - - with create_file_server(server_type) as server: - server.add_user("testuser", "12345", file_server_files) - generate_project(project, config={"aliases": {"tmpdir": server.base_url()}}) - - src_tar = os.path.join(file_server_files, "a.tar.gz") - _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) - - server.start() - - result = cli.run(project=project, args=["source", "track", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["source", "fetch", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["build", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) - result.assert_success() - - original_dir = os.path.join(str(datafiles), "content", "a") - original_contents = list_dir_contents(original_dir) - checkout_contents = list_dir_contents(checkoutdir) - assert checkout_contents == original_contents - - -@pytest.mark.parametrize("server_type", ("FTP", "HTTP")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) -def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir): - file_server_files = os.path.join(str(tmpdir), "file_server") - fake_home = os.path.join(str(tmpdir), "fake_home") - os.makedirs(file_server_files, exist_ok=True) - os.makedirs(fake_home, exist_ok=True) - project = str(datafiles) - - os.environ["HOME"] = fake_home - with open(os.path.join(fake_home, ".netrc"), "wb") as f: - os.fchmod(f.fileno(), 0o700) - f.write(b"machine 127.0.0.1\n") - f.write(b"login testuser\n") - f.write(b"password 12345\n") - - with create_file_server(server_type) as server: - server.add_user("otheruser", "12345", file_server_files) - parts = urllib.parse.urlsplit(server.base_url()) - base_url = urllib.parse.urlunsplit([parts[0], "otheruser@{}".format(parts[1]), *parts[2:]]) - generate_project(project, config={"aliases": {"tmpdir": base_url}}) - - src_tar = os.path.join(file_server_files, "a.tar.gz") - _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) - - server.start() - - result = cli.run(project=project, args=["source", "track", "target.bst"]) - result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.SOURCE, None) - - -# Test that BuildStream doesnt crash if HOME is unset while -# the netrc module is trying to find it's ~/.netrc file. -@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) -def test_homeless_environment(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) - - # Create a local tar - src_tar = os.path.join(str(tmpdir), "a.tar.gz") - _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) - - # Use a track, make sure the plugin tries to find a ~/.netrc - result = cli.run(project=project, args=["source", "track", "target.bst"], env={"HOME": None}) - result.assert_success() - - -@pytest.mark.datafiles(os.path.join(DATA_DIR, "out-of-basedir-hardlinks")) -def test_out_of_basedir_hardlinks(cli, tmpdir, datafiles): - def ensure_link(member): - # By default, python will simply duplicate files - we want - # hardlinks! - if member.path == "contents/to_extract/a": - member.type = tarfile.LNKTYPE - member.linkname = "contents/elsewhere/a" - return member - - project = str(datafiles) - generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) - checkoutdir = os.path.join(str(tmpdir), "checkout") - - # Create a tarball with an odd hardlink - src_tar = os.path.join(str(tmpdir), "contents.tar.gz") - old_dir = os.getcwd() - os.chdir(str(tmpdir)) - with tarfile.open(src_tar, "w:gz") as tar: - tar.add("contents", filter=ensure_link) - os.chdir(old_dir) - - # Make sure our tarfile is actually created with the desired - # attributes set - with tarfile.open(src_tar, "r:gz") as tar: - assert any( - member.islnk() and member.path == "contents/to_extract/a" and member.linkname == "contents/elsewhere/a" - for member in tar.getmembers() - ) - - # Assert that we will actually create a singular copy of the file - result = cli.run(project=project, args=["source", "track", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["source", "fetch", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["build", "target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) - result.assert_success() - - original_dir = os.path.join(str(datafiles), "contents", "to_extract") - original_contents = list_dir_contents(original_dir) - checkout_contents = list_dir_contents(checkoutdir) - assert checkout_contents == original_contents - - -@pytest.mark.datafiles(os.path.join(DATA_DIR, "out-of-basedir-hardlinks")) -def test_malicious_out_of_basedir_hardlinks(cli, tmpdir, datafiles): - project = str(datafiles) - generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) - - # Create a maliciously-hardlinked tarball - def ensure_link(member): - # By default, python will simply duplicate files - we want - # hardlinks! - if member.path == "contents/elsewhere/malicious": - member.type = tarfile.LNKTYPE - # This should not be allowed - member.linkname = "../../../malicious_target.bst" - return member - - src_tar = os.path.join(str(tmpdir), "contents.tar.gz") - old_dir = os.getcwd() - os.chdir(str(tmpdir)) - with tarfile.open(src_tar, "w:gz") as tar: - tar.add("contents", filter=ensure_link) - os.chdir(old_dir) - - # Make sure our tarfile is actually created with the desired - # attributes set - with tarfile.open(src_tar, "r:gz") as tar: - assert any( - member.islnk() - and member.path == "contents/elsewhere/malicious" - and member.linkname == "../../../malicious_target.bst" - for member in tar.getmembers() - ) - - # Try to execute the exploit - result = cli.run(project=project, args=["source", "track", "malicious_target.bst"]) - result.assert_success() - result = cli.run(project=project, args=["source", "fetch", "malicious_target.bst"]) - result.assert_main_error(ErrorDomain.STREAM, None) diff --git a/tests/sources/tar/contains-links/content/base-directory/subdir1/file.txt b/tests/sources/tar/contains-links/content/base-directory/subdir1/file.txt deleted file mode 100644 index f62144808..000000000 --- a/tests/sources/tar/contains-links/content/base-directory/subdir1/file.txt +++ /dev/null @@ -1 +0,0 @@ -pony diff --git a/tests/sources/tar/contains-links/target.bst b/tests/sources/tar/contains-links/target.bst deleted file mode 100644 index ad413a2f4..000000000 --- a/tests/sources/tar/contains-links/target.bst +++ /dev/null @@ -1,6 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: tar - url: tmpdir:/a.tar.gz - ref: foo diff --git a/tests/sources/tar/explicit-basedir/content/a/b/d b/tests/sources/tar/explicit-basedir/content/a/b/d deleted file mode 100644 index 4bcfe98e6..000000000 --- a/tests/sources/tar/explicit-basedir/content/a/b/d +++ /dev/null @@ -1 +0,0 @@ -d diff --git a/tests/sources/tar/explicit-basedir/content/a/c b/tests/sources/tar/explicit-basedir/content/a/c deleted file mode 100644 index f2ad6c76f..000000000 --- a/tests/sources/tar/explicit-basedir/content/a/c +++ /dev/null @@ -1 +0,0 @@ -c diff --git a/tests/sources/tar/explicit-basedir/target.bst b/tests/sources/tar/explicit-basedir/target.bst deleted file mode 100644 index fb85bdf42..000000000 --- a/tests/sources/tar/explicit-basedir/target.bst +++ /dev/null @@ -1,7 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: tar - url: tmpdir:/a.tar.gz - ref: foo - base-dir: 'a' diff --git a/tests/sources/tar/fetch/content/a/b/d b/tests/sources/tar/fetch/content/a/b/d deleted file mode 100644 index 4bcfe98e6..000000000 --- a/tests/sources/tar/fetch/content/a/b/d +++ /dev/null @@ -1 +0,0 @@ -d diff --git a/tests/sources/tar/fetch/content/a/c b/tests/sources/tar/fetch/content/a/c deleted file mode 100644 index f2ad6c76f..000000000 --- a/tests/sources/tar/fetch/content/a/c +++ /dev/null @@ -1 +0,0 @@ -c diff --git a/tests/sources/tar/fetch/target-lz.bst b/tests/sources/tar/fetch/target-lz.bst deleted file mode 100644 index b0569129e..000000000 --- a/tests/sources/tar/fetch/target-lz.bst +++ /dev/null @@ -1,6 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: tar - url: tmpdir:/a.tar.lz - ref: foo diff --git a/tests/sources/tar/fetch/target.bst b/tests/sources/tar/fetch/target.bst deleted file mode 100644 index ad413a2f4..000000000 --- a/tests/sources/tar/fetch/target.bst +++ /dev/null @@ -1,6 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: tar - url: tmpdir:/a.tar.gz - ref: foo diff --git a/tests/sources/tar/no-basedir/content/a/b/d b/tests/sources/tar/no-basedir/content/a/b/d deleted file mode 100644 index 4bcfe98e6..000000000 --- a/tests/sources/tar/no-basedir/content/a/b/d +++ /dev/null @@ -1 +0,0 @@ -d diff --git a/tests/sources/tar/no-basedir/content/a/c b/tests/sources/tar/no-basedir/content/a/c deleted file mode 100644 index f2ad6c76f..000000000 --- a/tests/sources/tar/no-basedir/content/a/c +++ /dev/null @@ -1 +0,0 @@ -c diff --git a/tests/sources/tar/no-basedir/target.bst b/tests/sources/tar/no-basedir/target.bst deleted file mode 100644 index 5a34d87a3..000000000 --- a/tests/sources/tar/no-basedir/target.bst +++ /dev/null @@ -1,7 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: tar - url: tmpdir:/a.tar.gz - ref: foo - base-dir: '' diff --git a/tests/sources/tar/no-ref/a/b/d b/tests/sources/tar/no-ref/a/b/d deleted file mode 100644 index 4bcfe98e6..000000000 --- a/tests/sources/tar/no-ref/a/b/d +++ /dev/null @@ -1 +0,0 @@ -d diff --git a/tests/sources/tar/no-ref/a/c b/tests/sources/tar/no-ref/a/c deleted file mode 100644 index f2ad6c76f..000000000 --- a/tests/sources/tar/no-ref/a/c +++ /dev/null @@ -1 +0,0 @@ -c diff --git a/tests/sources/tar/no-ref/target.bst b/tests/sources/tar/no-ref/target.bst deleted file mode 100644 index 5a73fa25b..000000000 --- a/tests/sources/tar/no-ref/target.bst +++ /dev/null @@ -1,5 +0,0 @@ -kind: autotools -description: The kind of this element is irrelevant. -sources: -- kind: tar - url: tmpdir:/a.tar.gz diff --git a/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/a b/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/a deleted file mode 100644 index ce0136250..000000000 --- a/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/a +++ /dev/null @@ -1 +0,0 @@ -hello diff --git a/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/malicious b/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/malicious deleted file mode 100644 index e69de29bb..000000000 --- a/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/malicious +++ /dev/null diff --git a/tests/sources/tar/out-of-basedir-hardlinks/contents/to_extract/a b/tests/sources/tar/out-of-basedir-hardlinks/contents/to_extract/a deleted file mode 100644 index ce0136250..000000000 --- a/tests/sources/tar/out-of-basedir-hardlinks/contents/to_extract/a +++ /dev/null @@ -1 +0,0 @@ -hello diff --git a/tests/sources/tar/out-of-basedir-hardlinks/malicious_target.bst b/tests/sources/tar/out-of-basedir-hardlinks/malicious_target.bst deleted file mode 100644 index b9debe961..000000000 --- a/tests/sources/tar/out-of-basedir-hardlinks/malicious_target.bst +++ /dev/null @@ -1,5 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: tar - url: tmpdir:/contents.tar.gz diff --git a/tests/sources/tar/out-of-basedir-hardlinks/target.bst b/tests/sources/tar/out-of-basedir-hardlinks/target.bst deleted file mode 100644 index 4f07e2e4c..000000000 --- a/tests/sources/tar/out-of-basedir-hardlinks/target.bst +++ /dev/null @@ -1,6 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: tar - url: tmpdir:/contents.tar.gz - base-dir: contents/to_extract diff --git a/tests/sources/tar/read-only/content/a.tar.gz b/tests/sources/tar/read-only/content/a.tar.gz Binary files differdeleted file mode 100644 index 70922627b..000000000 --- a/tests/sources/tar/read-only/content/a.tar.gz +++ /dev/null diff --git a/tests/sources/tar/read-only/content/b.tar.gz b/tests/sources/tar/read-only/content/b.tar.gz Binary files differdeleted file mode 100644 index 79fd1e864..000000000 --- a/tests/sources/tar/read-only/content/b.tar.gz +++ /dev/null diff --git a/tests/sources/tar/read-only/content/c.tar.gz b/tests/sources/tar/read-only/content/c.tar.gz Binary files differdeleted file mode 100644 index 016a5a218..000000000 --- a/tests/sources/tar/read-only/content/c.tar.gz +++ /dev/null diff --git a/tests/sources/tar/read-only/target.bst b/tests/sources/tar/read-only/target.bst deleted file mode 100644 index ad413a2f4..000000000 --- a/tests/sources/tar/read-only/target.bst +++ /dev/null @@ -1,6 +0,0 @@ -kind: import -description: The kind of this element is irrelevant. -sources: -- kind: tar - url: tmpdir:/a.tar.gz - ref: foo diff --git a/tests/testutils/repo/tar.py b/tests/testutils/repo/tar.py deleted file mode 100644 index 3eb9d896b..000000000 --- a/tests/testutils/repo/tar.py +++ /dev/null @@ -1,27 +0,0 @@ -import os -import tarfile - -from buildstream.utils import sha256sum - -from buildstream.testing import Repo - - -class Tar(Repo): - def create(self, directory): - tarball = os.path.join(self.repo, "file.tar.gz") - - old_dir = os.getcwd() - os.chdir(directory) - with tarfile.open(tarball, "w:gz") as tar: - tar.add(".") - os.chdir(old_dir) - - return sha256sum(tarball) - - def source_config(self, ref=None): - tarball = os.path.join(self.repo, "file.tar.gz") - config = {"kind": "tar", "url": "file://" + tarball, "directory": "", "base-dir": ""} - if ref is not None: - config["ref"] = ref - - return config @@ -37,7 +37,7 @@ deps = py{35,36,37,38}: -rrequirements/dev-requirements.txt # Install external plugins for plugin tests - py{35,36,37,38}-plugins: git+https://gitlab.com/buildstream/bst-plugins-experimental.git@{env:BST_PLUGINS_EXPERIMENTAL_VERSION:{[config]BST_PLUGINS_EXPERIMENTAL_VERSION}}#egg=bst_plugins_experimental[ostree,deb] + py{35,36,37,38}-plugins: git+https://gitlab.com/buildstream/bst-plugins-experimental.git@{env:BST_PLUGINS_EXPERIMENTAL_VERSION:{[config]BST_PLUGINS_EXPERIMENTAL_VERSION}}#egg=bst_plugins_experimental[ostree,deb,tar] # Only require coverage and pytest-cov when using it !nocover: -rrequirements/cov-requirements.txt @@ -173,7 +173,7 @@ deps = sphinx_rtd_theme >= 0.4.2 pytest -rrequirements/requirements.txt - git+https://gitlab.com/BuildStream/bst-plugins-experimental.git@be5ac19e5062bc23a46ed8ce7aa2958a2653c917#egg=bst_plugins_experimental[ostree] + git+https://gitlab.com/buildstream/bst-plugins-experimental.git@{env:BST_PLUGINS_EXPERIMENTAL_VERSION:{[config]BST_PLUGINS_EXPERIMENTAL_VERSION}}#egg=bst_plugins_experimental[ostree,deb,tar] passenv = BST_FORCE_SESSION_REBUILD BST_SOURCE_CACHE |