tests: Modify all tests using subprocesses to use in_subprocess mark

Additionally, test code that was previous executed by a subfunction (in
the forked process) has been folded into the test function itself, as
separating it is now redundant. This removes some duplicate code for
setting up the context and project, etc.

2 files changed, 29 insertions, 130 deletions

diff --git a/tests/internals/storage.py b/tests/internals/storage.py
index 385162c13..d846e4b1f 100644
--- a/tests/internals/storage.py
+++ b/tests/internals/storage.py
@@ -1,11 +1,8 @@
 from contextlib import contextmanager
-import multiprocessing
 import os
-import signal
 
 import pytest
 
-from buildstream import utils, _signals
 from buildstream._cas import CASCache
 from buildstream.storage._casbaseddirectory import CasBasedDirectory
 from buildstream.storage._filebaseddirectory import FileBasedDirectory
@@ -16,19 +13,6 @@ DATA_DIR = os.path.join(
 )
 
 
-# Since parent processes wait for queue events, we need
-# to put something on it if the called process raises an
-# exception.
-def _queue_wrapper(target, queue, *args):
-    try:
-        target(*args)
-    except Exception as e:
-        queue.put(str(e))
-        raise
-
-    queue.put(None)
-
-
 @contextmanager
 def setup_backend(backend_class, tmpdir):
     if backend_class == FileBasedDirectory:
@@ -41,7 +25,11 @@ def setup_backend(backend_class, tmpdir):
             cas_cache.release_resources()
 
 
-def _test_import_subprocess(tmpdir, datafiles, backend):
+@pytest.mark.in_subprocess
+@pytest.mark.parametrize("backend", [
+    FileBasedDirectory, CasBasedDirectory])
+@pytest.mark.datafiles(DATA_DIR)
+def test_import(tmpdir, datafiles, backend):
     original = os.path.join(str(datafiles), "original")
 
     with setup_backend(backend, str(tmpdir)) as c:
@@ -51,27 +39,11 @@ def _test_import_subprocess(tmpdir, datafiles, backend):
         assert "bin/hello" in c.list_relative_paths()
 
 
+@pytest.mark.in_subprocess
 @pytest.mark.parametrize("backend", [
     FileBasedDirectory, CasBasedDirectory])
 @pytest.mark.datafiles(DATA_DIR)
-def test_import(tmpdir, datafiles, backend):
-    queue = multiprocessing.Queue()
-    process = multiprocessing.Process(target=_queue_wrapper,
-                                      args=(_test_import_subprocess, queue,
-                                            tmpdir, datafiles, backend))
-    try:
-        with _signals.blocked([signal.SIGINT], ignore=False):
-            process.start()
-        error = queue.get()
-        process.join()
-    except KeyboardInterrupt:
-        utils._kill_process_tree(process.pid)
-        raise
-
-    assert not error
-
-
-def _test_modified_file_list_subprocess(tmpdir, datafiles, backend):
+def test_modified_file_list(tmpdir, datafiles, backend):
     original = os.path.join(str(datafiles), "original")
     overlay = os.path.join(str(datafiles), "overlay")
 
@@ -86,23 +58,3 @@ def _test_modified_file_list_subprocess(tmpdir, datafiles, backend):
         assert "bin/bash" in c.list_relative_paths()
         assert "bin/bash" in c.list_modified_paths()
         assert "bin/hello" not in c.list_modified_paths()
-
-
-@pytest.mark.parametrize("backend", [
-    FileBasedDirectory, CasBasedDirectory])
-@pytest.mark.datafiles(DATA_DIR)
-def test_modified_file_list(tmpdir, datafiles, backend):
-    queue = multiprocessing.Queue()
-    process = multiprocessing.Process(target=_queue_wrapper,
-                                      args=(_test_modified_file_list_subprocess, queue,
-                                            tmpdir, datafiles, backend))
-    try:
-        with _signals.blocked([signal.SIGINT], ignore=False):
-            process.start()
-        error = queue.get()
-        process.join()
-    except KeyboardInterrupt:
-        utils._kill_process_tree(process.pid)
-        raise
-
-    assert not error
diff --git a/tests/internals/storage_vdir_import.py b/tests/internals/storage_vdir_import.py
index e0165fc13..4599d4d5d 100644
--- a/tests/internals/storage_vdir_import.py
+++ b/tests/internals/storage_vdir_import.py
@@ -14,14 +14,11 @@
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 from hashlib import sha256
-import multiprocessing
 import os
 import random
-import signal
 
 import pytest
 
-from buildstream import utils, _signals
 from buildstream.storage._casbaseddirectory import CasBasedDirectory
 from buildstream.storage._filebaseddirectory import FileBasedDirectory
 from buildstream._cas import CASCache
@@ -58,34 +55,6 @@ RANDOM_SEED = 69105
 NUM_RANDOM_TESTS = 10
 
 
-# Since parent processes wait for queue events, we need
-# to put something on it if the called process raises an
-# exception.
-def _queue_wrapper(target, queue, *args):
-    try:
-        target(*args)
-    except Exception as e:
-        queue.put(str(e))
-        raise
-
-    queue.put(None)
-
-
-def _run_test_in_subprocess(func, *args):
-    queue = multiprocessing.Queue()
-    process = multiprocessing.Process(target=_queue_wrapper, args=(func, queue, *args))
-    try:
-        with _signals.blocked([signal.SIGINT], ignore=False):
-            process.start()
-        error = queue.get()
-        process.join()
-    except KeyboardInterrupt:
-        utils._kill_process_tree(process.pid)
-        raise
-
-    assert not error
-
-
 def generate_import_roots(rootno, directory):
     rootname = "root{}".format(rootno)
     rootdir = os.path.join(directory, "content", rootname)
@@ -213,7 +182,7 @@ def directory_not_empty(path):
     return os.listdir(path)
 
 
-def _import_test_subprocess(tmpdir, original, overlay, generator_function, verify_contents=False):
+def _import_test(tmpdir, original, overlay, generator_function, verify_contents=False):
     cas_cache = CASCache(tmpdir)
     try:
         # Create some fake content
@@ -269,26 +238,21 @@ def _import_test_subprocess(tmpdir, original, overlay, generator_function, verif
         cas_cache.release_resources()
 
 
-def _import_test(tmpdir, original, overlay, generator_function, verify_contents=False):
-    _run_test_in_subprocess(_import_test_subprocess, tmpdir, original, overlay, generator_function, verify_contents)
-
-
-# It's possible to parameterize on both original and overlay values,
-# but this leads to more tests being listed in the output than are
-# comfortable.
+@pytest.mark.in_subprocess
 @pytest.mark.parametrize("original", range(1, len(root_filesets) + 1))
-def test_fixed_cas_import(tmpdir, original):
-    for overlay in range(1, len(root_filesets) + 1):
-        _import_test(str(tmpdir), original, overlay, generate_import_roots, verify_contents=True)
+@pytest.mark.parametrize("overlay", range(1, len(root_filesets) + 1))
+def test_fixed_cas_import(tmpdir, original, overlay):
+    _import_test(str(tmpdir), original, overlay, generate_import_roots, verify_contents=True)
 
 
+@pytest.mark.in_subprocess
 @pytest.mark.parametrize("original", range(1, NUM_RANDOM_TESTS + 1))
-def test_random_cas_import(tmpdir, original):
-    for overlay in range(1, NUM_RANDOM_TESTS + 1):
-        _import_test(str(tmpdir), original, overlay, generate_random_root, verify_contents=False)
+@pytest.mark.parametrize("overlay", range(1, NUM_RANDOM_TESTS + 1))
+def test_random_cas_import(tmpdir, original, overlay):
+    _import_test(str(tmpdir), original, overlay, generate_random_root, verify_contents=False)
 
 
-def _listing_test_subprocess(tmpdir, root, generator_function):
+def _listing_test(tmpdir, root, generator_function):
     cas_cache = CASCache(tmpdir)
     try:
         # Create some fake content
@@ -305,22 +269,21 @@ def _listing_test_subprocess(tmpdir, root, generator_function):
         cas_cache.release_resources()
 
 
-def _listing_test(tmpdir, root, generator_function):
-    _run_test_in_subprocess(_listing_test_subprocess, tmpdir, root, generator_function)
-
-
 @pytest.mark.parametrize("root", range(1, 11))
+@pytest.mark.in_subprocess
 def test_random_directory_listing(tmpdir, root):
     _listing_test(str(tmpdir), root, generate_random_root)
 
 
 @pytest.mark.parametrize("root", [1, 2, 3, 4, 5])
+@pytest.mark.in_subprocess
 def test_fixed_directory_listing(tmpdir, root):
     _listing_test(str(tmpdir), root, generate_import_roots)
 
 
 # Check that the vdir is decending and readable
-def _test_descend_subprocess(tmpdir):
+@pytest.mark.in_subprocess
+def test_descend(tmpdir):
     cas_dir = os.path.join(str(tmpdir), 'cas')
     cas_cache = CASCache(cas_dir)
     try:
@@ -343,14 +306,11 @@ def _test_descend_subprocess(tmpdir):
         cas_cache.release_resources()
 
 
-def test_descend(tmpdir):
-    _run_test_in_subprocess(_test_descend_subprocess, tmpdir)
-
-
 # Check symlink logic for edgecases
 # Make sure the correct erros are raised when trying
 # to decend in to files or links to files
-def _test_bad_symlinks_subprocess(tmpdir):
+@pytest.mark.in_subprocess
+def test_bad_symlinks(tmpdir):
     cas_dir = os.path.join(str(tmpdir), 'cas')
     cas_cache = CASCache(cas_dir)
     try:
@@ -381,13 +341,10 @@ def _test_bad_symlinks_subprocess(tmpdir):
         cas_cache.release_resources()
 
 
-def test_bad_symlinks(tmpdir):
-    _run_test_in_subprocess(_test_bad_symlinks_subprocess, tmpdir)
-
-
 # Check symlink logic for edgecases
 # Check decend accross relitive link
-def _test_relative_symlink_subprocess(tmpdir):
+@pytest.mark.in_subprocess
+def test_relative_symlink(tmpdir):
     cas_dir = os.path.join(str(tmpdir), 'cas')
     cas_cache = CASCache(cas_dir)
     try:
@@ -410,13 +367,10 @@ def _test_relative_symlink_subprocess(tmpdir):
         cas_cache.release_resources()
 
 
-def test_relative_symlink(tmpdir):
-    _run_test_in_subprocess(_test_relative_symlink_subprocess, tmpdir)
-
-
 # Check symlink logic for edgecases
 # Check deccend accross abs link
-def _test_abs_symlink_subprocess(tmpdir):
+@pytest.mark.in_subprocess
+def test_abs_symlink(tmpdir):
     cas_dir = os.path.join(str(tmpdir), 'cas')
     cas_cache = CASCache(cas_dir)
     try:
@@ -440,13 +394,10 @@ def _test_abs_symlink_subprocess(tmpdir):
         cas_cache.release_resources()
 
 
-def test_abs_symlink(tmpdir):
-    _run_test_in_subprocess(_test_abs_symlink_subprocess, tmpdir)
-
-
 # Check symlink logic for edgecases
 # Check symlink can not escape root
-def _test_bad_sym_escape_subprocess(tmpdir):
+@pytest.mark.in_subprocess
+def test_bad_sym_escape(tmpdir):
     cas_dir = os.path.join(str(tmpdir), 'cas')
     cas_cache = CASCache(cas_dir)
     try:
@@ -468,7 +419,3 @@ def _test_bad_sym_escape_subprocess(tmpdir):
             assert error.reason == "directory-not-found"
     finally:
         cas_cache.release_resources()
-
-
-def test_bad_sym_escape(tmpdir):
-    _run_test_in_subprocess(_test_bad_sym_escape_subprocess, tmpdir)