Fix the test suite0.1.0

2 files changed, 142 insertions, 77 deletions

diff --git a/tests/programs.c b/tests/programs.c
new file mode 100644
index 0000000..340bc28
--- /dev/null
+++ b/tests/programs.c
@@ -0,0 +1,109 @@
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Test programs for 'sandboxlib' functional tests.
+
+The tests need to create clean, reproducible sandboxes in order for the tests
+to behave the same on all machines. This means not depending on the host OS.
+We need some programs to actually run inside the sandbox and try to break them.
+There are two approaches: either build / download a small OS from somewhere
+that will run in a chroot and will work the same on all platforms, or build
+minimal, self-contained tester programs using tools in the host OS.
+
+I picked the second approach: to test the sandboxes using statically linked C
+programs. Each C program below should be small, self-contained and should test
+one thing.
+
+'''
+
+
+def build_c_program(source_code, output_path, compiler_args=None):
+    '''Compile a temporary C program.'''
+    compiler_args = compiler_args or []
+    with tempfile.NamedTemporaryFile(suffix='.c') as f:
+        f.write(WRITEABLE_PATHS_TEST_CODE.encode('utf-8'))
+        f.flush()
+
+        subprocess.check_call(
+            ['gcc', '-static', f.name, '-o', str(output_path)])
+
+
+# Test if a file or directory exists.
+FILE_OR_DIRECTORY_EXISTS_TEST_PROGRAM = """
+#include <stdio.h>
+#include <sys/stat.h>
+
+int main(int argc, char *argv[]) {
+    struct stat stat_data;
+
+    if (argc != 2) {
+        fprintf(stderr, "Expected 1 argument: filename to try to read from.");
+        return 1;
+    }
+
+    if (stat(argv[1], &stat_data) != 0) {
+        printf("Did not find %s.", argv[1]);
+        return 2;
+    }
+
+    printf("%s exists", argv[1]);
+    return 0;
+};
+"""
+
+@pytest.fixture(scope='module')
+def file_exists_test_program(self, tmpdir):
+    program_path = tmpdir.join('writable-paths-tester')
+    build_c_program(
+        FILE_OR_DIRECTORY_EXISTS_TEST_PROGRAM, program_path, compiler_args=['-static'])
+    return program_path
+
+# Test if a file can be written to.
+FILE_IS_WRITABLE_TEST_PROGRAM = """
+#include <stdio.h>
+
+int main(int argc, char *argv[]) {
+    FILE *file;
+
+    if (argc != 2) {
+        fprintf(stderr, "Expected 1 argument: filename to try to write to.");
+        return 1;
+    }
+
+    file = fopen(argv[0], "w");
+
+    if (file == NULL) {
+        printf("Couldn't open %s for writing.", argv[1]);
+        return 2;
+    }
+
+    if (fputc('!', file) != '!') {
+        printf("Couldn't write to %s.", argv[1]);
+        fclose(file);
+        return 3;
+    }
+
+    fclose(file);
+    printf("Wrote data to %s.", argv[1]);
+    return 0;
+};
+"""
+
+@pytest.fixture(scope='module')
+def file_is_writable_test_program(self, tmpdir):
+    program_path = tmpdir.join('writable-paths-tester')
+    build_c_program(
+        FILE_IS_WRITEABLE_TEST_PROGRAM, program_path, compiler_args=['-static'])
+    return program_path
diff --git a/tests/test_all.py b/tests/test_all.py
index d5ff47c..3d3d398 100644
--- a/tests/test_all.py
+++ b/tests/test_all.py
@@ -13,37 +13,17 @@
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-'''Functional ('black-box') tests for all 'sandboxlib' backends.
-
-FIXME: right now this is incomplete! Needs to introspect more!
-
-'''
+'''Functional ('black-box') tests for all 'sandboxlib' backends.'''
 
 
 import pytest
 
 import os
-import subprocess
-import tempfile
 
 import sandboxlib
-
-
-def build_c_program(source_code, output_path, compiler_args=None):
-    '''Compile a temporary C program.
-
-    In order that the test suite be self-contained, we test the sandboxes
-    using statically linked C programs. The alternative would be to depend on
-    some operating system image that can run in a container.
-
-    '''
-    compiler_args = compiler_args or []
-    with tempfile.NamedTemporaryFile(suffix='.c') as f:
-        f.write(WRITEABLE_PATHS_TEST_CODE.encode('utf-8'))
-        f.flush()
-
-        subprocess.check_call(
-            ['gcc', '-static', f.name, '-o', str(output_path)])
+from programs import (
+    file_is_writable_test_program, file_or_directory_exists_test_program,
+    session_tmpdir)
 
 
 @pytest.fixture(params=['chroot', 'linux_user_chroot'])
@@ -73,64 +53,40 @@ def test_current_working_directory(sandboxlib_executor, tmpdir):
     assert err.decode('unicode-escape') == ''
 
 
-def test_mounts(sandboxlib_executor, tmpdir):
-    # FIXME: This test will fail because we try to run a command in an empty
-    # chroot. Need some kind of statically linked C program to run in there.
-    exit, out, err = sandboxlib_executor.run_sandbox(
-        ['/bin/ls', '/proc'],
-        filesystem_root=str(tmpdir),
-        extra_mounts=[(None, '/proc', 'proc')])
-
-
-# The simplest way to test these sandboxes is with a statically linked C
-# program.
-WRITEABLE_PATHS_TEST_CODE = """
-#include <stdio.h>
-
-int main(int argc, char *argv[]) {
-    FILE *file;
+class TestMounts(object):
+    @pytest.fixture()
+    def mounts_test_sandbox(self, tmpdir,
+                            file_or_directory_exists_test_program):
+        sandbox_path = tmpdir.mkdir('sandbox')
 
-    if (argc != 2) {
-        fprintf(stderr, "Expected 1 argument: filename to try to write to.");
-        return 1;
-    }
+        bin_path = sandbox_path.mkdir('bin')
 
-    file = fopen(argv[0], "w");
+        file_or_directory_exists_test_program.copy(bin_path)
+        bin_path.join('test-file-or-directory-exists').chmod(0o755)
 
-    if (file == NULL) {
-        printf("Couldn't open %s for writing.", argv[1]);
-        return 2;
-    }
+        return sandbox_path
 
-    if (fputc('!', file) != '!') {
-        printf("Couldn't write to %s.", argv[1]);
-        fclose(file);
-        return 3;
-    }
+    def test_mount_proc(self, sandboxlib_executor, mounts_test_sandbox):
+        exit, out, err = sandboxlib_executor.run_sandbox(
+            ['test-file-or-directory-exists', '/proc'],
+            filesystem_root=str(mounts_test_sandbox),
+            extra_mounts=[(None, '/proc', 'proc')])
 
-    fclose(file);
-    printf("Wrote data to %s.", argv[1]);
-    return 0;
-};
-"""
+        assert err.decode('unicode-escape') == ''
+        assert out.decode('unicode-escape') == "/proc exists"
+        assert exit == 0
 
 
 class TestWriteablePaths(object):
-    @pytest.fixture(scope='module')
-    def writable_paths_test_program(self, tmpdir):
-        program_path = tmpdir.join('writable-paths-tester')
-        build_c_program(
-            WRITEABLE_PATHS_TEST_CODE, program_path, compiler_args=['-static'])
-        return program_path
-
     @pytest.fixture()
-    def writable_paths_test_sandbox(self, tmpdir, writable_paths_test_program):
+    def writable_paths_test_sandbox(self, tmpdir,
+                                    file_is_writable_test_program):
         sandbox_path = tmpdir.mkdir('sandbox')
 
         bin_path = sandbox_path.mkdir('bin')
 
-        writable_paths_test_program.copy(bin_path)
-        bin_path.join('writable-paths-tester').chmod(0o755)
+        file_is_writable_test_program.copy(bin_path)
+        bin_path.join('test-file-is-writable').chmod(0o755)
 
         data_path = sandbox_path.mkdir('data')
         data_path.mkdir('1')
@@ -139,19 +95,19 @@ class TestWriteablePaths(object):
         return sandbox_path
 
     def test_none_writable(self, sandboxlib_executor,
-                            writable_paths_test_sandbox):
+                           writable_paths_test_sandbox):
         if sandboxlib_executor == sandboxlib.chroot:
             pytest.xfail("chroot backend doesn't support read-only paths.")
 
         exit, out, err = sandboxlib_executor.run_sandbox(
-            ['writable-paths-tester', '/data/1/canary'],
+            ['test-file-is-writable', '/data/1/canary'],
             filesystem_root=str(writable_paths_test_sandbox),
             filesystem_writable_paths='none')
 
         assert err.decode('unicode-escape') == ''
         assert out.decode('unicode-escape') == \
             "Couldn't open /data/1/canary for writing."
-        assert exit == 2
+        assert exit == 1
 
     def test_some_writable(self, sandboxlib_executor,
                            writable_paths_test_sandbox):
@@ -159,7 +115,7 @@ class TestWriteablePaths(object):
             pytest.xfail("chroot backend doesn't support read-only paths.")
 
         exit, out, err = sandboxlib_executor.run_sandbox(
-            ['writable-paths-tester', '/data/1/canary'],
+            ['test-file-is-writable', '/data/1/canary'],
             filesystem_root=str(writable_paths_test_sandbox),
             filesystem_writable_paths=['/data/1'])
 
@@ -171,7 +127,7 @@ class TestWriteablePaths(object):
     def test_all_writable(self, sandboxlib_executor,
                           writable_paths_test_sandbox):
         exit, out, err = sandboxlib_executor.run_sandbox(
-            ['writable-paths-tester', '/data/1/canary'],
+            ['test-file-is-writable', '/data/1/canary'],
             filesystem_root=str(writable_paths_test_sandbox),
             filesystem_writable_paths='all')
 
@@ -186,7 +142,7 @@ class TestWriteablePaths(object):
             pytest.xfail("chroot backend doesn't support read-only paths.")
 
         exit, out, err = sandboxlib_executor.run_sandbox(
-            ['writable-paths-tester', '/data/1/canary'],
+            ['test-file-is-writable', '/data/1/canary'],
             filesystem_root=str(writable_paths_test_sandbox),
             filesystem_writable_paths='none',
             extra_mounts=[
@@ -196,7 +152,7 @@ class TestWriteablePaths(object):
         assert err.decode('unicode-escape') == ''
         assert out.decode('unicode-escape') == \
             "Couldn't open /data/1/canary for writing."
-        assert exit == 2
+        assert exit == 1
 
     def test_mount_point_writable(self, sandboxlib_executor,
                                   writable_paths_test_sandbox):
@@ -204,7 +160,7 @@ class TestWriteablePaths(object):
             pytest.xfail("chroot backend doesn't support read-only paths.")
 
         exit, out, err = sandboxlib_executor.run_sandbox(
-            ['writable-paths-tester', '/data/1/canary'],
+            ['test-file-is-writable', '/data/1/canary'],
             filesystem_root=str(writable_paths_test_sandbox),
             filesystem_writable_paths=['/data'],
             extra_mounts=[