Created deb Source plugin for Issue #10

added tests

25 files changed, 296 insertions, 0 deletions

diff --git a/buildstream/plugins/sources/deb.py b/buildstream/plugins/sources/deb.py
new file mode 100644
index 000000000..eaf24d9a8
--- /dev/null
+++ b/buildstream/plugins/sources/deb.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Phillip Smyth <phillip.smyth@codethink.co.uk>
+#        Jonathan Maw <jonathan.maw@codethink.co.uk>
+#        Richard Maw <richard.maw@codethink.co.uk>
+
+"""A source implementation for staging deb files
+
+**Usage:**
+
+.. code:: yaml
+
+   # Specify the deb source kind
+   kind: deb
+
+   # Optionally specify a relative staging directory
+   # directory: path/to/stage
+
+   # Specify the deb url. Using an alias defined in your project
+   # configuration is encouraged. 'bst track' will update the
+   # sha256sum in 'ref' to the downloaded file's sha256sum.
+   url: upstream:foo.deb
+
+   # Specify the ref. It's a sha256sum of the file you download.
+   ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
+
+   # Specify the basedir to return only the specified dir and it's children
+   base-dir: ''
+
+"""
+
+import tarfile
+from contextlib import contextmanager, ExitStack
+import arpy
+
+from .tar import TarSource
+
+
+class DebSource(TarSource):
+
+    def configure(self, node):
+        super().configure(node)
+
+        self.base_dir = self.node_get_member(node, str, 'base-dir', '') or None
+
+    def preflight(self):
+        return
+
+    @contextmanager
+    def _get_tar(self):
+        with ExitStack() as context:
+            deb_file = context.enter_context(open(self._get_mirror_file(), 'rb'))
+            arpy_archive = arpy.Archive(fileobj=deb_file)
+            arpy_archive.read_all_headers()
+            data_tar_arpy = [v for k, v in arpy_archive.archived_files.items() if b"data.tar" in k][0]
+            # ArchiveFileData is not enough like a file object for tarfile to use.
+            # Monkey-patching a seekable method makes it close enough for TarFile to open.
+            data_tar_arpy.seekable = lambda *args: True
+            tar = tarfile.open(fileobj=data_tar_arpy, mode="r:*")
+            yield tar
+
+
+def setup():
+    return DebSource
diff --git a/tests/sources/deb.py b/tests/sources/deb.py
new file mode 100644
index 000000000..d44076b2d
--- /dev/null
+++ b/tests/sources/deb.py
@@ -0,0 +1,181 @@
+import os
+import pytest
+import tarfile
+import tempfile
+import subprocess
+import shutil
+import arpy
+
+from buildstream._exceptions import ErrorDomain
+from buildstream import _yaml
+from tempfile import TemporaryFile
+from tests.testutils import cli
+
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'deb',
+)
+
+deb_name = "a_deb.deb"
+
+
+def generate_project(project_dir, tmpdir):
+    project_file = os.path.join(project_dir, "project.conf")
+    _yaml.dump({
+        'name': 'foo',
+        'aliases': {
+            'tmpdir': "file:///" + str(tmpdir)
+        }
+    }, project_file)
+
+
+def _copy_deb(start_location, tmpdir):
+    source = os.path.join(start_location, deb_name)
+    destination = os.path.join(str(tmpdir), deb_name)
+    shutil.copyfile(source, destination)
+
+
+def _list_dir_contents(srcdir):
+    contents = set()
+    for _, dirs, files in os.walk(srcdir):
+        for d in dirs:
+            contents.add(d)
+        for f in files:
+            contents.add(f)
+    return contents
+
+
+# Test that without ref, consistency is set appropriately.
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
+def test_no_ref(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    assert cli.get_element_state(project, 'target.bst') == 'no reference'
+
+
+# Test that when I fetch a nonexistent URL, errors are handled gracefully.
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_fetch_bad_url(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+
+    # Try to fetch it
+    result = cli.run(project=project, args=[
+        'fetch', 'target.bst'
+    ])
+    result.assert_main_error(ErrorDomain.PIPELINE, None)
+    result.assert_task_error(ErrorDomain.SOURCE, None)
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_fetch_bad_ref(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+
+    # Copy test deb to tmpdir
+    _copy_deb(DATA_DIR, tmpdir)
+
+    # Try to fetch it
+    result = cli.run(project=project, args=[
+        'fetch', 'target.bst'
+    ])
+    result.assert_main_error(ErrorDomain.PIPELINE, None)
+    result.assert_task_error(ErrorDomain.SOURCE, None)
+
+
+# Test that when tracking with a ref set, there is a warning
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_track_warning(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+
+    # Copy test deb to tmpdir
+    _copy_deb(DATA_DIR, tmpdir)
+
+    # Track it
+    result = cli.run(project=project, args=[
+        'track', 'target.bst'
+    ])
+    result.assert_success()
+    assert "Potential man-in-the-middle attack!" in result.stderr
+
+
+# Test that a staged checkout matches what was tarred up, with the default first subdir
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_stage_default_basedir(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    checkoutdir = os.path.join(str(tmpdir), "checkout")
+
+    # Copy test deb to tmpdir
+    _copy_deb(DATA_DIR, tmpdir)
+
+    # Track, fetch, build, checkout
+    result = cli.run(project=project, args=['track', 'target.bst'])
+    result.assert_success()
+    result = cli.run(project=project, args=['fetch', 'target.bst'])
+    result.assert_success()
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    result.assert_success()
+    result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
+    result.assert_success()
+
+    # Check that the content of the first directory is checked out (base-dir: '')
+    original_dir = os.path.join(str(datafiles), "content")
+    original_contents = _list_dir_contents(original_dir)
+    checkout_contents = _list_dir_contents(checkoutdir)
+    assert(checkout_contents == original_contents)
+
+
+# Test that a staged checkout matches what was tarred up, with an empty base-dir
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-basedir'))
+def test_stage_no_basedir(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    checkoutdir = os.path.join(str(tmpdir), "checkout")
+
+    # Copy test deb to tmpdir
+    _copy_deb(DATA_DIR, tmpdir)
+
+    # Track, fetch, build, checkout
+    result = cli.run(project=project, args=['track', 'target.bst'])
+    result.assert_success()
+    result = cli.run(project=project, args=['fetch', 'target.bst'])
+    result.assert_success()
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    result.assert_success()
+    result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
+    result.assert_success()
+
+    # Check that the full content of the tarball is checked out (base-dir: '')
+    original_dir = os.path.join(str(datafiles), "content")
+    original_contents = _list_dir_contents(original_dir)
+    checkout_contents = _list_dir_contents(checkoutdir)
+    assert(checkout_contents == original_contents)
+
+
+# Test that a staged checkout matches what was tarred up, with an explicit basedir
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'explicit-basedir'))
+def test_stage_explicit_basedir(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    checkoutdir = os.path.join(str(tmpdir), "checkout")
+
+    # Copy test deb to tmpdir
+    _copy_deb(DATA_DIR, tmpdir)
+
+    # Track, fetch, build, checkout
+    result = cli.run(project=project, args=['track', 'target.bst'])
+    result.assert_success()
+    result = cli.run(project=project, args=['fetch', 'target.bst'])
+    result.assert_success()
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    result.assert_success()
+    result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
+    result.assert_success()
+
+    # Check that the content of the first directory is checked out (base-dir: '')
+    original_dir = os.path.join(str(datafiles), "content")
+    original_contents = _list_dir_contents(original_dir)
+    checkout_contents = _list_dir_contents(checkoutdir)
+    assert(checkout_contents == original_contents)
diff --git a/tests/sources/deb/a_deb.deb b/tests/sources/deb/a_deb.deb
new file mode 100644
index 000000000..c8fef9117
--- /dev/null
+++ b/tests/sources/deb/a_deb.deb
diff --git a/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/README b/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/README
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/README
diff --git a/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/changelog.Debian.gz b/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/changelog.Debian.gz
new file mode 100644
index 000000000..a9090a51c
--- /dev/null
+++ b/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/changelog.Debian.gz
diff --git a/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/copyright b/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/copyright
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/sources/deb/explicit-basedir/content/share/doc/lua-clod/copyright
diff --git a/tests/sources/deb/explicit-basedir/content/share/lua/5.1/clod.lua b/tests/sources/deb/explicit-basedir/content/share/lua/5.1/clod.lua
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/sources/deb/explicit-basedir/content/share/lua/5.1/clod.lua
diff --git a/tests/sources/deb/explicit-basedir/content/share/lua/5.2/clod.lua b/tests/sources/deb/explicit-basedir/content/share/lua/5.2/clod.lua
new file mode 120000
index 000000000..79531e49d
--- /dev/null
+++ b/tests/sources/deb/explicit-basedir/content/share/lua/5.2/clod.lua
@@ -0,0 +1 @@
+../5.1/clod.lua
\ No newline at end of file
diff --git a/tests/sources/deb/explicit-basedir/target.bst b/tests/sources/deb/explicit-basedir/target.bst
new file mode 100644
index 000000000..a75881151
--- /dev/null
+++ b/tests/sources/deb/explicit-basedir/target.bst
@@ -0,0 +1,7 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: deb
+  url: tmpdir:/a_deb.deb
+  ref: foo
+  base-dir: 'usr'
diff --git a/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/README b/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/README
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/README
diff --git a/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/changelog.Debian.gz b/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/changelog.Debian.gz
new file mode 100644
index 000000000..040dd15a7
--- /dev/null
+++ b/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/changelog.Debian.gz
diff --git a/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/copyright b/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/copyright
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/sources/deb/fetch/content/usr/share/doc/lua-clod/copyright
diff --git a/tests/sources/deb/fetch/content/usr/share/lua/5.1/clod.lua b/tests/sources/deb/fetch/content/usr/share/lua/5.1/clod.lua
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/sources/deb/fetch/content/usr/share/lua/5.1/clod.lua
diff --git a/tests/sources/deb/fetch/content/usr/share/lua/5.2/clod.lua b/tests/sources/deb/fetch/content/usr/share/lua/5.2/clod.lua
new file mode 120000
index 000000000..79531e49d
--- /dev/null
+++ b/tests/sources/deb/fetch/content/usr/share/lua/5.2/clod.lua
@@ -0,0 +1 @@
+../5.1/clod.lua
\ No newline at end of file
diff --git a/tests/sources/deb/fetch/target-lz.bst b/tests/sources/deb/fetch/target-lz.bst
new file mode 100644
index 000000000..b0569129e
--- /dev/null
+++ b/tests/sources/deb/fetch/target-lz.bst
@@ -0,0 +1,6 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+  url: tmpdir:/a.tar.lz
+  ref: foo
diff --git a/tests/sources/deb/fetch/target.bst b/tests/sources/deb/fetch/target.bst
new file mode 100644
index 000000000..919e66a37
--- /dev/null
+++ b/tests/sources/deb/fetch/target.bst
@@ -0,0 +1,6 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: deb
+  url: tmpdir:/a_deb.deb
+  ref: foo
diff --git a/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/README b/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/README
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/README
diff --git a/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/changelog.Debian.gz b/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/changelog.Debian.gz
new file mode 100644
index 000000000..be777e65f
--- /dev/null
+++ b/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/changelog.Debian.gz
diff --git a/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/copyright b/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/copyright
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/sources/deb/no-basedir/content/usr/share/doc/lua-clod/copyright
diff --git a/tests/sources/deb/no-basedir/content/usr/share/lua/5.1/clod.lua b/tests/sources/deb/no-basedir/content/usr/share/lua/5.1/clod.lua
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/sources/deb/no-basedir/content/usr/share/lua/5.1/clod.lua
diff --git a/tests/sources/deb/no-basedir/content/usr/share/lua/5.2/clod.lua b/tests/sources/deb/no-basedir/content/usr/share/lua/5.2/clod.lua
new file mode 120000
index 000000000..79531e49d
--- /dev/null
+++ b/tests/sources/deb/no-basedir/content/usr/share/lua/5.2/clod.lua
@@ -0,0 +1 @@
+../5.1/clod.lua
\ No newline at end of file
diff --git a/tests/sources/deb/no-basedir/target.bst b/tests/sources/deb/no-basedir/target.bst
new file mode 100644
index 000000000..69fcd700c
--- /dev/null
+++ b/tests/sources/deb/no-basedir/target.bst
@@ -0,0 +1,7 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: deb
+  url: tmpdir:/a_deb.deb
+  ref: foo
+  base-dir: ''
diff --git a/tests/sources/deb/no-ref/a/b/d b/tests/sources/deb/no-ref/a/b/d
new file mode 100644
index 000000000..4bcfe98e6
--- /dev/null
+++ b/tests/sources/deb/no-ref/a/b/d
@@ -0,0 +1 @@
+d
diff --git a/tests/sources/deb/no-ref/a/c b/tests/sources/deb/no-ref/a/c
new file mode 100644
index 000000000..f2ad6c76f
--- /dev/null
+++ b/tests/sources/deb/no-ref/a/c
@@ -0,0 +1 @@
+c
diff --git a/tests/sources/deb/no-ref/target.bst b/tests/sources/deb/no-ref/target.bst
new file mode 100644
index 000000000..89b639662
--- /dev/null
+++ b/tests/sources/deb/no-ref/target.bst
@@ -0,0 +1,5 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: deb
+  url: tmpdir:/a_deb.deb