diff options
| -rw-r--r-- | buildstream/plugins/sources/zip.py | 147 | ||||
| -rw-r--r-- | tests/cachekey/project/sources/zip1.bst | 5 | ||||
| -rw-r--r-- | tests/cachekey/project/sources/zip1.expected | 1 | ||||
| -rw-r--r-- | tests/cachekey/project/sources/zip2.bst | 6 | ||||
| -rw-r--r-- | tests/cachekey/project/sources/zip2.expected | 1 | ||||
| -rw-r--r-- | tests/cachekey/project/target.bst | 2 | ||||
| -rw-r--r-- | tests/cachekey/project/target.expected | 2 | ||||
| -rw-r--r-- | tests/testutils/repo/__init__.py | 2 | ||||
| -rw-r--r-- | tests/testutils/repo/zip.py | 40 |
9 files changed, 205 insertions, 1 deletions
diff --git a/buildstream/plugins/sources/zip.py b/buildstream/plugins/sources/zip.py new file mode 100644 index 000000000..e2c5ee38c --- /dev/null +++ b/buildstream/plugins/sources/zip.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Mathieu Bridon +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Mathieu Bridon <bochecha@daitauha.fr> + +"""A source implementation for staging zip files + +**Usage:** + +.. code:: yaml + + # Specify the zip source kind + kind: zip + + # Specify the zip url. Using an alias defined in your project + # configuration is encouraged. 'bst track' will update the + # sha256sum in 'ref' to the downloaded file's sha256sum. + url: upstream:foo.zip + + # Specify the ref. It's a sha256sum of the file you download. + ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b + + # Specify a glob pattern to indicate the base directory to extract + # from the archive. The first matching directory will be used. + # + # Note that this is '*' by default since most standard release + # archives contain a self named subdirectory at the root which + # contains the files one normally wants to extract to build. + # + # To extract the root of the archive directly, this can be set + # to an empty string. + base-dir: '*' +""" + +import os +import zipfile + +from buildstream import Source, SourceError +from buildstream import utils + +from ._downloadablefilesource import DownloadableFileSource + + +class ZipSource(DownloadableFileSource): + + def configure(self, node): + super().configure(node) + + self.base_dir = self.node_get_member(node, str, 'base-dir', '*') or None + + self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir']) + + def get_unique_key(self): + return super().get_unique_key() + [self.base_dir] + + def stage(self, directory): + try: + with zipfile.ZipFile(self._get_mirror_file()) as archive: + base_dir = None + if self.base_dir: + base_dir = self._find_base_dir(archive, self.base_dir) + + if base_dir: + archive.extractall(path=directory, members=self._extract_members(archive, base_dir)) + else: + archive.extractall(path=directory) + + except (zipfile.BadZipFile, zipfile.LargeZipFile, OSError) as e: + raise SourceError("{}: Error staging source: {}".format(self, e)) from e + + # Override and translate which filenames to extract + def _extract_members(self, archive, base_dir): + if not base_dir.endswith(os.sep): + base_dir = base_dir + os.sep + + l = len(base_dir) + for member in archive.infolist(): + if member.filename == base_dir: + continue + + if member.filename.startswith(base_dir): + member.filename = member.filename[l:] + yield member + + # We want to iterate over all paths of an archive, but namelist() + # is not enough because some archives simply do not contain the leading + # directory paths for the archived files. + def _list_archive_paths(self, archive, dirs_only=False): + + visited = {} + for member in archive.infolist(): + if not member.is_dir(): + + # Loop over the components of a path, for a path of a/b/c/d + # we will first visit 'a', then 'a/b' and then 'a/b/c', excluding + # the final component + components = member.filename.split('/') + for i in range(len(components) - 1): + dir_component = '/'.join([components[j] for j in range(i + 1)]) + if dir_component not in visited: + visited[dir_component] = True + try: + # Dont yield directory members which actually do + # exist in the archive + _ = archive.getinfo(dir_component) + except KeyError: + yield dir_component + + continue + + # Avoid considering the '.' directory, if any is included in the archive + # this is to avoid the default 'base-dir: *' value behaving differently + # depending on whether the archive was encoded with a leading '.' or not + elif member.filename == './': + continue + + if dirs_only and not member.is_dir(): + continue + + yield member.filename + + def _find_base_dir(self, archive, pattern): + paths = self._list_archive_paths(archive, dirs_only=True) + matches = sorted(list(utils.glob(paths, pattern))) + if not matches: + raise SourceError("{}: Could not find base directory matching pattern: {}".format(self, pattern)) + + return matches[0] + + +def setup(): + return ZipSource diff --git a/tests/cachekey/project/sources/zip1.bst b/tests/cachekey/project/sources/zip1.bst new file mode 100644 index 000000000..a55eeb969 --- /dev/null +++ b/tests/cachekey/project/sources/zip1.bst @@ -0,0 +1,5 @@ +kind: import +sources: +- kind: zip + url: https://example.com/releases/1.4/foo-1.4.5.zip + ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected new file mode 100644 index 000000000..6672e86ce --- /dev/null +++ b/tests/cachekey/project/sources/zip1.expected @@ -0,0 +1 @@ +61b75fc80ab68a1aadc62afbbbc617d94ba561849fe31b4aab943b739191f96c
\ No newline at end of file diff --git a/tests/cachekey/project/sources/zip2.bst b/tests/cachekey/project/sources/zip2.bst new file mode 100644 index 000000000..1f3961a6b --- /dev/null +++ b/tests/cachekey/project/sources/zip2.bst @@ -0,0 +1,6 @@ +kind: import +sources: +- kind: zip + url: https://example.com/releases/1.4/foo-1.4.5.zip + ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b + base-dir: src diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected new file mode 100644 index 000000000..092f688c5 --- /dev/null +++ b/tests/cachekey/project/sources/zip2.expected @@ -0,0 +1 @@ +692cb18cb28ab55aaaab97a92de07c924791cb8b89e1094771ab035f051ad4c4
\ No newline at end of file diff --git a/tests/cachekey/project/target.bst b/tests/cachekey/project/target.bst index 1107bcf0a..d49abc8ca 100644 --- a/tests/cachekey/project/target.bst +++ b/tests/cachekey/project/target.bst @@ -15,6 +15,8 @@ depends: - sources/patch3.bst - sources/tar1.bst - sources/tar2.bst +- sources/zip1.bst +- sources/zip2.bst - elements/build1.bst - elements/compose1.bst - elements/compose2.bst diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected index 75d811eca..2260f4bee 100644 --- a/tests/cachekey/project/target.expected +++ b/tests/cachekey/project/target.expected @@ -1 +1 @@ -2926d3718d1430a90e18dd66135a89f87de3acd400cd6444bc4e613caaac944d
\ No newline at end of file +345ceb4f27d21143002f369d57a34cb2a3a95715f85c4ea4360df7f5cf14a42b
\ No newline at end of file diff --git a/tests/testutils/repo/__init__.py b/tests/testutils/repo/__init__.py index 3516679aa..1123d6f5e 100644 --- a/tests/testutils/repo/__init__.py +++ b/tests/testutils/repo/__init__.py @@ -6,12 +6,14 @@ from .git import Git from .bzr import Bzr from .ostree import OSTree from .tar import Tar +from .zip import Zip ALL_REPO_KINDS = OrderedDict() ALL_REPO_KINDS['git'] = Git ALL_REPO_KINDS['bzr'] = Bzr ALL_REPO_KINDS['ostree'] = OSTree ALL_REPO_KINDS['tar'] = Tar +ALL_REPO_KINDS['zip'] = Zip # create_repo() diff --git a/tests/testutils/repo/zip.py b/tests/testutils/repo/zip.py new file mode 100644 index 000000000..47c402421 --- /dev/null +++ b/tests/testutils/repo/zip.py @@ -0,0 +1,40 @@ +import hashlib +import os +import zipfile + +from buildstream.utils import sha256sum + +from .repo import Repo + + +class Zip(Repo): + + def create(self, directory): + archive = os.path.join(self.repo, 'file.zip') + + old_dir = os.getcwd() + os.chdir(directory) + with zipfile.ZipFile(archive, "w") as zip: + for root, dirs, files in os.walk('.'): + names = dirs + files + names = [os.path.join(root, name) for name in names] + + for name in names: + zip.write(name) + + os.chdir(old_dir) + + return sha256sum(archive) + + def source_config(self, ref=None): + archive = os.path.join(self.repo, 'file.zip') + config = { + 'kind': 'zip', + 'url': 'file://' + archive, + 'directory': '', + 'base-dir': '' + } + if ref is not None: + config['ref'] = ref + + return config |