summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPhil Dawson <phildawson.0807@gmail.com>2019-02-08 16:28:05 +0000
committerPhil Dawson <phildawson.0807@gmail.com>2019-02-08 16:28:05 +0000
commit77ce0c1607d060eda098779228d5b0ab62ec04ef (patch)
treeff2b7c958ba18ab3a018d1917a7b9369bc82b8a9
parente61f471376d6d3ef2691abf3eee75d30999e7f05 (diff)
parentb4d4c4f59c12c9b500f83d8cd4381418e1124f67 (diff)
downloadbuildstream-77ce0c1607d060eda098779228d5b0ab62ec04ef.tar.gz
Merge branch 'phil/plugin-testing-api' into 'master'
Expose basic api for testing external plugins. Closes #847 See merge request BuildStream/buildstream!1075
-rw-r--r--buildstream/plugintestutils/__init__.py30
-rw-r--r--buildstream/plugintestutils/integration.py51
-rw-r--r--buildstream/plugintestutils/runcli.py (renamed from tests/testutils/runcli.py)38
-rw-r--r--tests/artifactcache/cache_size.py3
-rw-r--r--tests/artifactcache/config.py2
-rw-r--r--tests/artifactcache/expiry.py3
-rw-r--r--tests/artifactcache/junctions.py4
-rw-r--r--tests/artifactcache/pull.py3
-rw-r--r--tests/artifactcache/push.py4
-rw-r--r--tests/cachekey/cachekey.py2
-rwxr-xr-xtests/cachekey/update.py2
-rw-r--r--tests/elements/filter.py3
-rw-r--r--tests/examples/autotools.py4
-rw-r--r--tests/examples/developing.py4
-rw-r--r--tests/examples/first-project.py4
-rw-r--r--tests/examples/flatpak-autotools.py4
-rw-r--r--tests/examples/integration-commands.py4
-rw-r--r--tests/examples/junctions.py4
-rw-r--r--tests/examples/running-commands.py4
-rw-r--r--tests/format/assertion.py2
-rw-r--r--tests/format/dependencies.py2
-rw-r--r--tests/format/include.py4
-rw-r--r--tests/format/junctions.py3
-rw-r--r--tests/format/listdirectiveerrors.py2
-rw-r--r--tests/format/optionarch.py2
-rw-r--r--tests/format/optionbool.py2
-rw-r--r--tests/format/optioneltmask.py2
-rw-r--r--tests/format/optionenum.py2
-rw-r--r--tests/format/optionexports.py2
-rw-r--r--tests/format/optionflags.py2
-rw-r--r--tests/format/optionos.py2
-rw-r--r--tests/format/optionoverrides.py2
-rw-r--r--tests/format/options.py2
-rw-r--r--tests/format/project.py4
-rw-r--r--tests/format/projectoverrides.py2
-rw-r--r--tests/format/variables.py2
-rw-r--r--tests/frontend/buildcheckout.py3
-rw-r--r--tests/frontend/buildtrack.py4
-rw-r--r--tests/frontend/completions.py2
-rw-r--r--tests/frontend/compose_splits.py2
-rw-r--r--tests/frontend/configurable_warnings.py2
-rw-r--r--tests/frontend/cross_junction_workspace.py4
-rw-r--r--tests/frontend/fetch.py4
-rw-r--r--tests/frontend/help.py2
-rw-r--r--tests/frontend/init.py2
-rw-r--r--tests/frontend/logging.py3
-rw-r--r--tests/frontend/mirror.py3
-rw-r--r--tests/frontend/order.py4
-rw-r--r--tests/frontend/overlaps.py2
-rw-r--r--tests/frontend/pull.py3
-rw-r--r--tests/frontend/push.py4
-rw-r--r--tests/frontend/rebuild.py2
-rw-r--r--tests/frontend/show.py3
-rw-r--r--tests/frontend/source_checkout.py2
-rw-r--r--tests/frontend/track.py3
-rw-r--r--tests/frontend/track_cross_junction.py4
-rw-r--r--tests/frontend/version.py2
-rw-r--r--tests/frontend/workspace.py3
-rw-r--r--tests/frontend/yamlcache.py3
-rw-r--r--tests/integration/artifact.py2
-rw-r--r--tests/integration/autotools.py4
-rw-r--r--tests/integration/build-tree.py3
-rw-r--r--tests/integration/build-uid.py4
-rw-r--r--tests/integration/cachedfail.py4
-rw-r--r--tests/integration/cmake.py4
-rw-r--r--tests/integration/compose-symlinks.py4
-rw-r--r--tests/integration/compose.py4
-rw-r--r--tests/integration/import.py4
-rw-r--r--tests/integration/make.py4
-rw-r--r--tests/integration/manual.py2
-rw-r--r--tests/integration/messages.py3
-rw-r--r--tests/integration/pip_element.py4
-rw-r--r--tests/integration/pip_source.py4
-rw-r--r--tests/integration/pullbuildtrees.py7
-rw-r--r--tests/integration/sandbox-bwrap.py4
-rw-r--r--tests/integration/script.py3
-rw-r--r--tests/integration/shell.py3
-rw-r--r--tests/integration/sockets.py5
-rw-r--r--tests/integration/source-determinism.py2
-rw-r--r--tests/integration/stack.py3
-rw-r--r--tests/integration/symlinks.py4
-rw-r--r--tests/integration/workspace.py4
-rw-r--r--tests/internals/storage_vdir_import.py2
-rw-r--r--tests/sandboxes/missing-command.py2
-rw-r--r--tests/sandboxes/missing_dependencies.py2
-rw-r--r--tests/sandboxes/remote-exec-config.py2
-rw-r--r--tests/sources/bzr.py3
-rw-r--r--tests/sources/deb.py2
-rw-r--r--tests/sources/git.py4
-rw-r--r--tests/sources/local.py3
-rw-r--r--tests/sources/no_fetch_cached.py3
-rw-r--r--tests/sources/ostree.py3
-rw-r--r--tests/sources/patch.py3
-rw-r--r--tests/sources/pip.py2
-rw-r--r--tests/sources/previous_source_access.py2
-rw-r--r--tests/sources/remote.py2
-rw-r--r--tests/sources/tar.py2
-rw-r--r--tests/sources/zip.py2
-rw-r--r--tests/testutils/__init__.py1
-rw-r--r--tests/testutils/integration.py29
-rw-r--r--tox.ini1
101 files changed, 274 insertions, 159 deletions
diff --git a/buildstream/plugintestutils/__init__.py b/buildstream/plugintestutils/__init__.py
new file mode 100644
index 000000000..c7238a29c
--- /dev/null
+++ b/buildstream/plugintestutils/__init__.py
@@ -0,0 +1,30 @@
+#
+# Copyright (C) 2019 Codethink Limited
+# Copyright (C) 2019 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+
+from .runcli import cli, cli_integration
+
+# To make use of these test utilities it is necessary to have pytest
+# available. However, we don't want to have a hard dependency on
+# pytest.
+try:
+ import pytest
+except ImportError:
+ module_name = globals()['__name__']
+ msg = "Could not import pytest:\n" \
+ "To use the {} module, you must have pytest installed.".format(module_name)
+ raise ImportError(msg)
diff --git a/buildstream/plugintestutils/integration.py b/buildstream/plugintestutils/integration.py
new file mode 100644
index 000000000..e29f480ea
--- /dev/null
+++ b/buildstream/plugintestutils/integration.py
@@ -0,0 +1,51 @@
+#
+# Copyright (C) 2017 Codethink Limited
+# Copyright (C) 2018 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+"""
+Integration - tools for inspecting the output of plugin integration tests
+=========================================================================
+
+This module contains utilities for inspecting the artifacts produced during
+integration tests.
+"""
+
+import os
+
+
+# Return a list of files relative to the given directory
+def walk_dir(root):
+ for dirname, dirnames, filenames in os.walk(root):
+ # ensure consistent traversal order, needed for consistent
+ # handling of symlinks.
+ dirnames.sort()
+ filenames.sort()
+
+ # print path to all subdirectories first.
+ for subdirname in dirnames:
+ yield os.path.join(dirname, subdirname)[len(root):]
+
+ # print path to all filenames.
+ for filename in filenames:
+ yield os.path.join(dirname, filename)[len(root):]
+
+
+# Ensure that a directory contains the given filenames.
+def assert_contains(directory, expected):
+ missing = set(expected)
+ missing.difference_update(walk_dir(directory))
+ if missing:
+ raise AssertionError("Missing {} expected elements from list: {}"
+ .format(len(missing), missing))
diff --git a/tests/testutils/runcli.py b/buildstream/plugintestutils/runcli.py
index b051dec21..fb7c23c6c 100644
--- a/tests/testutils/runcli.py
+++ b/buildstream/plugintestutils/runcli.py
@@ -1,3 +1,34 @@
+#
+# Copyright (C) 2017 Codethink Limited
+# Copyright (C) 2018 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+"""
+runcli - Test fixtures used for running BuildStream commands
+============================================================
+
+:function:'cli' Use result = cli.run([arg1, arg2]) to run buildstream commands
+
+:function:'cli_integration' A variant of the main fixture that keeps persistent
+ artifact and source caches. It also does not use
+ the click test runner to avoid deadlock issues when
+ running `bst shell`, but unfortunately cannot produce
+ nice stacktraces.
+
+"""
+
+
import os
import re
import sys
@@ -5,7 +36,6 @@ import shutil
import tempfile
import itertools
import traceback
-import subprocess
from contextlib import contextmanager, ExitStack
from ruamel import yaml
import pytest
@@ -333,7 +363,7 @@ class Cli():
return result
- def invoke(self, cli, args=None, color=False, binary_capture=False, **extra):
+ def invoke(self, cli_object, args=None, color=False, binary_capture=False, **extra):
exc_info = None
exception = None
exit_code = 0
@@ -348,7 +378,7 @@ class Cli():
capture.start_capturing()
try:
- cli.main(args=args or (), prog_name=cli.name, **extra)
+ cli_object.main(args=args or (), prog_name=cli_object.name, **extra)
except SystemExit as e:
if e.code != 0:
exception = e
@@ -361,7 +391,7 @@ class Cli():
sys.stdout.write(str(exit_code))
sys.stdout.write('\n')
exit_code = 1
- except Exception as e:
+ except Exception as e: # pylint: disable=broad-except
exception = e
exit_code = -1
exc_info = sys.exc_info()
diff --git a/tests/artifactcache/cache_size.py b/tests/artifactcache/cache_size.py
index 63ab9ad07..88f8eaddf 100644
--- a/tests/artifactcache/cache_size.py
+++ b/tests/artifactcache/cache_size.py
@@ -5,8 +5,9 @@ from unittest import mock
from buildstream import _yaml
from buildstream._artifactcache import CACHE_SIZE_FILE
from buildstream._exceptions import ErrorDomain
+from buildstream.plugintestutils import cli
-from tests.testutils import cli, create_element_size
+from tests.testutils import create_element_size
# XXX: Currently lacking:
# * A way to check whether it's faster to read cache size on
diff --git a/tests/artifactcache/config.py b/tests/artifactcache/config.py
index fecb3dd2e..fda309725 100644
--- a/tests/artifactcache/config.py
+++ b/tests/artifactcache/config.py
@@ -10,7 +10,7 @@ from buildstream.utils import _deduplicate
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py
index 2cc59e03c..d92e68f0b 100644
--- a/tests/artifactcache/expiry.py
+++ b/tests/artifactcache/expiry.py
@@ -25,8 +25,9 @@ import pytest
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
+from buildstream.plugintestutils import cli
-from tests.testutils import cli, create_element_size, update_element_size, wait_for_cache_granularity
+from tests.testutils import create_element_size, update_element_size, wait_for_cache_granularity
DATA_DIR = os.path.join(
diff --git a/tests/artifactcache/junctions.py b/tests/artifactcache/junctions.py
index c6d6921c8..d5de16282 100644
--- a/tests/artifactcache/junctions.py
+++ b/tests/artifactcache/junctions.py
@@ -1,9 +1,11 @@
import os
import shutil
import pytest
-from tests.testutils import cli, create_artifact_share
from buildstream import _yaml
+from buildstream.plugintestutils import cli
+
+from tests.testutils import create_artifact_share
DATA_DIR = os.path.join(
diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py
index 4c332bf36..edd5a93ba 100644
--- a/tests/artifactcache/pull.py
+++ b/tests/artifactcache/pull.py
@@ -9,8 +9,9 @@ from buildstream import _yaml, _signals, utils
from buildstream._context import Context
from buildstream._project import Project
from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
+from buildstream.plugintestutils import cli
-from tests.testutils import cli, create_artifact_share
+from tests.testutils import create_artifact_share
# Project directory
diff --git a/tests/artifactcache/push.py b/tests/artifactcache/push.py
index 116fa7865..ed2a140e7 100644
--- a/tests/artifactcache/push.py
+++ b/tests/artifactcache/push.py
@@ -10,8 +10,8 @@ from buildstream._context import Context
from buildstream._project import Project
from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
from buildstream.storage._casbaseddirectory import CasBasedDirectory
-
-from tests.testutils import cli, create_artifact_share
+from buildstream.plugintestutils import cli
+from tests.testutils import create_artifact_share
# Project directory
diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py
index 761ff0c76..4a6896888 100644
--- a/tests/cachekey/cachekey.py
+++ b/tests/cachekey/cachekey.py
@@ -35,7 +35,7 @@
# run over to the corresponding .expected source files and commit
# the result.
#
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
from tests.testutils.site import HAVE_BZR, HAVE_GIT, HAVE_OSTREE, IS_LINUX, MACHINE_ARCH
from buildstream.plugin import CoreWarnings
from buildstream import _yaml
diff --git a/tests/cachekey/update.py b/tests/cachekey/update.py
index d574d07b3..3e0924925 100755
--- a/tests/cachekey/update.py
+++ b/tests/cachekey/update.py
@@ -13,7 +13,7 @@
#
import os
import tempfile
-from tests.testutils.runcli import Cli
+from buildstream.plugintestutils.runcli import Cli
# This weird try / except is needed, because this will be imported differently
# when pytest runner imports them vs when you run the updater directly from
diff --git a/tests/elements/filter.py b/tests/elements/filter.py
index d40a8bdd1..c0ef2568a 100644
--- a/tests/elements/filter.py
+++ b/tests/elements/filter.py
@@ -1,7 +1,8 @@
import os
import pytest
import shutil
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS
+from tests.testutils import create_repo, ALL_REPO_KINDS
+from buildstream.plugintestutils import cli
from buildstream._exceptions import ErrorDomain
from buildstream import _yaml
diff --git a/tests/examples/autotools.py b/tests/examples/autotools.py
index 30f50768b..96827ff4c 100644
--- a/tests/examples/autotools.py
+++ b/tests/examples/autotools.py
@@ -1,8 +1,8 @@
import os
import pytest
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import HAVE_BWRAP, IS_LINUX, MACHINE_ARCH
pytestmark = pytest.mark.integration
diff --git a/tests/examples/developing.py b/tests/examples/developing.py
index 166fcf374..3ef78fd36 100644
--- a/tests/examples/developing.py
+++ b/tests/examples/developing.py
@@ -2,8 +2,8 @@ import os
import pytest
import tests.testutils.patch as patch
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import HAVE_BWRAP, IS_LINUX, MACHINE_ARCH
pytestmark = pytest.mark.integration
diff --git a/tests/examples/first-project.py b/tests/examples/first-project.py
index 821d2c190..35839770c 100644
--- a/tests/examples/first-project.py
+++ b/tests/examples/first-project.py
@@ -1,8 +1,8 @@
import os
import pytest
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import IS_LINUX
diff --git a/tests/examples/flatpak-autotools.py b/tests/examples/flatpak-autotools.py
index 4153a9563..454b8d0bb 100644
--- a/tests/examples/flatpak-autotools.py
+++ b/tests/examples/flatpak-autotools.py
@@ -1,8 +1,8 @@
import os
import pytest
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import HAVE_OSTREE, IS_LINUX, MACHINE_ARCH
diff --git a/tests/examples/integration-commands.py b/tests/examples/integration-commands.py
index 71e383008..abc64d951 100644
--- a/tests/examples/integration-commands.py
+++ b/tests/examples/integration-commands.py
@@ -1,8 +1,8 @@
import os
import pytest
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import HAVE_BWRAP, IS_LINUX, MACHINE_ARCH
diff --git a/tests/examples/junctions.py b/tests/examples/junctions.py
index 753fa2dc0..3992b1520 100644
--- a/tests/examples/junctions.py
+++ b/tests/examples/junctions.py
@@ -1,8 +1,8 @@
import os
import pytest
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import IS_LINUX, HAVE_BWRAP, MACHINE_ARCH
pytestmark = pytest.mark.integration
diff --git a/tests/examples/running-commands.py b/tests/examples/running-commands.py
index 61e23fd9c..6290204a9 100644
--- a/tests/examples/running-commands.py
+++ b/tests/examples/running-commands.py
@@ -1,8 +1,8 @@
import os
import pytest
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import IS_LINUX, HAVE_BWRAP, MACHINE_ARCH
diff --git a/tests/format/assertion.py b/tests/format/assertion.py
index 42f69b759..b787e07fc 100644
--- a/tests/format/assertion.py
+++ b/tests/format/assertion.py
@@ -1,7 +1,7 @@
import os
import pytest
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# Project directory
DATA_DIR = os.path.join(
diff --git a/tests/format/dependencies.py b/tests/format/dependencies.py
index a6b417a66..5513077ee 100644
--- a/tests/format/dependencies.py
+++ b/tests/format/dependencies.py
@@ -2,7 +2,7 @@ import os
import pytest
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/include.py b/tests/format/include.py
index 36e723ed0..1db37083e 100644
--- a/tests/format/include.py
+++ b/tests/format/include.py
@@ -2,7 +2,9 @@ import os
import pytest
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import cli, generate_junction, create_repo
+from buildstream.plugintestutils import cli
+
+from tests.testutils import generate_junction, create_repo
# Project directory
diff --git a/tests/format/junctions.py b/tests/format/junctions.py
index 90608d0fc..7f27b5982 100644
--- a/tests/format/junctions.py
+++ b/tests/format/junctions.py
@@ -4,7 +4,8 @@ import shutil
from buildstream import _yaml, ElementError
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import cli, create_repo
+from buildstream.plugintestutils import cli
+from tests.testutils import create_repo
from tests.testutils.site import HAVE_GIT
diff --git a/tests/format/listdirectiveerrors.py b/tests/format/listdirectiveerrors.py
index 4a2de86fc..8c397252c 100644
--- a/tests/format/listdirectiveerrors.py
+++ b/tests/format/listdirectiveerrors.py
@@ -1,7 +1,7 @@
import os
import pytest
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# Project directory
DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optionarch.py b/tests/format/optionarch.py
index 09f9c07c9..a1237002d 100644
--- a/tests/format/optionarch.py
+++ b/tests/format/optionarch.py
@@ -3,7 +3,7 @@ import pytest
from contextlib import contextmanager
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# Project directory
DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optionbool.py b/tests/format/optionbool.py
index f02f0eee3..598496cd3 100644
--- a/tests/format/optionbool.py
+++ b/tests/format/optionbool.py
@@ -2,7 +2,7 @@ import os
import pytest
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# Project directory
DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optioneltmask.py b/tests/format/optioneltmask.py
index b49eee88a..7cbbd88f4 100644
--- a/tests/format/optioneltmask.py
+++ b/tests/format/optioneltmask.py
@@ -2,7 +2,7 @@ import os
import pytest
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# Project directory
DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optionenum.py b/tests/format/optionenum.py
index 6990ad915..936126dfc 100644
--- a/tests/format/optionenum.py
+++ b/tests/format/optionenum.py
@@ -2,7 +2,7 @@ import os
import pytest
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# Project directory
DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optionexports.py b/tests/format/optionexports.py
index 26eadc677..5e65e9721 100644
--- a/tests/format/optionexports.py
+++ b/tests/format/optionexports.py
@@ -1,7 +1,7 @@
import os
import pytest
from buildstream import _yaml
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# Project directory
DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optionflags.py b/tests/format/optionflags.py
index 3585f5ea5..341bb8c80 100644
--- a/tests/format/optionflags.py
+++ b/tests/format/optionflags.py
@@ -2,7 +2,7 @@ import os
import pytest
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# Project directory
DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optionos.py b/tests/format/optionos.py
index b05d7e49e..ba6ab2a48 100644
--- a/tests/format/optionos.py
+++ b/tests/format/optionos.py
@@ -4,7 +4,7 @@ from contextlib import contextmanager
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optionoverrides.py b/tests/format/optionoverrides.py
index e5c37b3a5..256562edf 100644
--- a/tests/format/optionoverrides.py
+++ b/tests/format/optionoverrides.py
@@ -1,7 +1,7 @@
import os
import pytest
from buildstream import _yaml
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# Project directory
DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/options.py b/tests/format/options.py
index 2076106cb..3a1b12877 100644
--- a/tests/format/options.py
+++ b/tests/format/options.py
@@ -2,7 +2,7 @@ import os
import pytest
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# Project directory
DATA_DIR = os.path.join(
diff --git a/tests/format/project.py b/tests/format/project.py
index 0433df99e..db6604409 100644
--- a/tests/format/project.py
+++ b/tests/format/project.py
@@ -2,7 +2,9 @@ import os
import pytest
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import cli, filetypegenerator
+from buildstream.plugintestutils import cli
+
+from tests.testutils import filetypegenerator
# Project directory
diff --git a/tests/format/projectoverrides.py b/tests/format/projectoverrides.py
index d63b0a23d..cf9a1d35e 100644
--- a/tests/format/projectoverrides.py
+++ b/tests/format/projectoverrides.py
@@ -2,7 +2,7 @@
import os
import pytest
from buildstream import _yaml
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# Project directory
DATA_DIR = os.path.join(
diff --git a/tests/format/variables.py b/tests/format/variables.py
index be7faefc5..5a0d21206 100644
--- a/tests/format/variables.py
+++ b/tests/format/variables.py
@@ -3,7 +3,7 @@ import pytest
import sys
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# Project directory
diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py
index b35b14820..80d710f6f 100644
--- a/tests/frontend/buildcheckout.py
+++ b/tests/frontend/buildcheckout.py
@@ -3,9 +3,10 @@ import tarfile
import hashlib
import pytest
import subprocess
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS, generate_junction
from tests.testutils.site import IS_WINDOWS
+from tests.testutils import create_repo, ALL_REPO_KINDS, generate_junction
+from buildstream.plugintestutils import cli
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
diff --git a/tests/frontend/buildtrack.py b/tests/frontend/buildtrack.py
index 720ab7efc..9c56fb4a0 100644
--- a/tests/frontend/buildtrack.py
+++ b/tests/frontend/buildtrack.py
@@ -4,9 +4,11 @@ import shutil
import itertools
import pytest
-from tests.testutils import cli, create_repo
+
+from tests.testutils import create_repo
from buildstream import _yaml
+from buildstream.plugintestutils import cli
from buildstream._exceptions import ErrorDomain
from . import configure_project
diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py
index 93b908f60..cb151c284 100644
--- a/tests/frontend/completions.py
+++ b/tests/frontend/completions.py
@@ -1,6 +1,6 @@
import os
import pytest
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
# Project directory
DATA_DIR = os.path.join(
diff --git a/tests/frontend/compose_splits.py b/tests/frontend/compose_splits.py
index 97558b653..3d2db44fb 100644
--- a/tests/frontend/compose_splits.py
+++ b/tests/frontend/compose_splits.py
@@ -1,6 +1,6 @@
import os
import pytest
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# Project directory
DATA_DIR = os.path.join(
diff --git a/tests/frontend/configurable_warnings.py b/tests/frontend/configurable_warnings.py
index e8a7b6ac8..042874529 100644
--- a/tests/frontend/configurable_warnings.py
+++ b/tests/frontend/configurable_warnings.py
@@ -4,7 +4,7 @@ import os
from buildstream.plugin import CoreWarnings
from buildstream._exceptions import ErrorDomain, LoadErrorReason
from buildstream import _yaml
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
TOP_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/frontend/cross_junction_workspace.py b/tests/frontend/cross_junction_workspace.py
index a10eb7421..837a09f8a 100644
--- a/tests/frontend/cross_junction_workspace.py
+++ b/tests/frontend/cross_junction_workspace.py
@@ -1,7 +1,9 @@
import os
-from tests.testutils import cli, create_repo
+from buildstream.plugintestutils import cli
from buildstream import _yaml
+from tests.testutils import create_repo
+
def prepare_junction_project(cli, tmpdir):
main_project = tmpdir.join("main")
diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py
index 24d9a36a6..9edfad94b 100644
--- a/tests/frontend/fetch.py
+++ b/tests/frontend/fetch.py
@@ -1,7 +1,9 @@
import os
import pytest
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS, generate_junction
+from tests.testutils import create_repo, ALL_REPO_KINDS, generate_junction
+
+from buildstream.plugintestutils import cli
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
diff --git a/tests/frontend/help.py b/tests/frontend/help.py
index 331eb43c0..540a3a60f 100644
--- a/tests/frontend/help.py
+++ b/tests/frontend/help.py
@@ -1,5 +1,5 @@
import pytest
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
def assert_help(cli_output):
diff --git a/tests/frontend/init.py b/tests/frontend/init.py
index f6e7dd68a..b323a01c8 100644
--- a/tests/frontend/init.py
+++ b/tests/frontend/init.py
@@ -1,6 +1,6 @@
import os
import pytest
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
from buildstream import _yaml
from buildstream._frontend.app import App
diff --git a/tests/frontend/logging.py b/tests/frontend/logging.py
index ddaadfa26..17d925cd1 100644
--- a/tests/frontend/logging.py
+++ b/tests/frontend/logging.py
@@ -1,10 +1,11 @@
import os
import pytest
import re
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS
+from tests.testutils import create_repo, ALL_REPO_KINDS
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain
+from buildstream.plugintestutils import cli
# Project directory
DATA_DIR = os.path.join(
diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
index 3cadd15f3..c0c2e7059 100644
--- a/tests/frontend/mirror.py
+++ b/tests/frontend/mirror.py
@@ -1,10 +1,11 @@
import os
import pytest
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS, generate_junction
+from tests.testutils import create_repo, ALL_REPO_KINDS, generate_junction
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain
+from buildstream.plugintestutils import cli
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/frontend/order.py b/tests/frontend/order.py
index 97ebc2bf6..422fce260 100644
--- a/tests/frontend/order.py
+++ b/tests/frontend/order.py
@@ -1,8 +1,8 @@
import os
import pytest
-from tests.testutils import cli, create_repo
-
+from tests.testutils import create_repo
+from buildstream.plugintestutils import cli
from buildstream import _yaml
# Project directory
diff --git a/tests/frontend/overlaps.py b/tests/frontend/overlaps.py
index 27be8deac..22365e442 100644
--- a/tests/frontend/overlaps.py
+++ b/tests/frontend/overlaps.py
@@ -1,6 +1,6 @@
import os
import pytest
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
from buildstream._exceptions import ErrorDomain
from buildstream import _yaml
from buildstream.plugin import CoreWarnings
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index a2c710fb8..9579d9f27 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -1,7 +1,8 @@
import os
import shutil
import pytest
-from tests.testutils import cli, create_artifact_share, generate_junction
+from buildstream.plugintestutils import cli
+from tests.testutils import create_artifact_share, generate_junction
# Project directory
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index 729df9ae9..7a4cb3c0d 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -24,8 +24,8 @@ import os
import pytest
from buildstream._exceptions import ErrorDomain
-from tests.testutils import cli, create_artifact_share, create_element_size
-from tests.testutils import generate_junction, wait_for_cache_granularity
+from buildstream.plugintestutils import cli
+from tests.testutils import create_artifact_share, create_element_size, generate_junction, wait_for_cache_granularity
from . import configure_project
diff --git a/tests/frontend/rebuild.py b/tests/frontend/rebuild.py
index d93aac0dc..6a46e855e 100644
--- a/tests/frontend/rebuild.py
+++ b/tests/frontend/rebuild.py
@@ -1,6 +1,6 @@
import os
import pytest
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
# Project directory
DATA_DIR = os.path.join(
diff --git a/tests/frontend/show.py b/tests/frontend/show.py
index 88f38dd6a..ad3ae3591 100644
--- a/tests/frontend/show.py
+++ b/tests/frontend/show.py
@@ -3,7 +3,8 @@ import sys
import shutil
import itertools
import pytest
-from tests.testutils import cli, generate_junction
+from tests.testutils import generate_junction
+from buildstream.plugintestutils import cli
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
diff --git a/tests/frontend/source_checkout.py b/tests/frontend/source_checkout.py
index d7ff86d70..b41d32491 100644
--- a/tests/frontend/source_checkout.py
+++ b/tests/frontend/source_checkout.py
@@ -3,7 +3,7 @@ import pytest
import tarfile
from pathlib import Path
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
from buildstream import utils, _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
diff --git a/tests/frontend/track.py b/tests/frontend/track.py
index a4ace92b6..486a4123c 100644
--- a/tests/frontend/track.py
+++ b/tests/frontend/track.py
@@ -1,8 +1,9 @@
import stat
import os
import pytest
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS, generate_junction
+from tests.testutils import create_repo, ALL_REPO_KINDS, generate_junction
+from buildstream.plugintestutils import cli
from buildstream._exceptions import ErrorDomain, LoadErrorReason
from buildstream import _yaml
diff --git a/tests/frontend/track_cross_junction.py b/tests/frontend/track_cross_junction.py
index 4bbf2db18..7d9d468ae 100644
--- a/tests/frontend/track_cross_junction.py
+++ b/tests/frontend/track_cross_junction.py
@@ -1,6 +1,8 @@
import os
import pytest
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS, generate_junction
+from tests.testutils import create_repo, ALL_REPO_KINDS, generate_junction
+
+from buildstream.plugintestutils import cli
from buildstream import _yaml
diff --git a/tests/frontend/version.py b/tests/frontend/version.py
index 997eb2340..1d36a33b5 100644
--- a/tests/frontend/version.py
+++ b/tests/frontend/version.py
@@ -1,4 +1,4 @@
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
# For utils.get_bst_version()
from buildstream import utils
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index 6d4270e11..f6d12e8bf 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -30,9 +30,10 @@ import pytest
import shutil
import subprocess
from ruamel.yaml.comments import CommentedSet
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
+from tests.testutils import create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
from tests.testutils import create_artifact_share, create_element_size
+from buildstream.plugintestutils import cli
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
from buildstream._workspaces import BST_WORKSPACE_FORMAT_VERSION
diff --git a/tests/frontend/yamlcache.py b/tests/frontend/yamlcache.py
index 20388593e..99b5d71c3 100644
--- a/tests/frontend/yamlcache.py
+++ b/tests/frontend/yamlcache.py
@@ -4,7 +4,8 @@ import hashlib
import tempfile
from ruamel import yaml
-from tests.testutils import cli, generate_junction, create_element_size, create_repo
+from tests.testutils import generate_junction, create_element_size, create_repo
+from buildstream.plugintestutils import cli
from buildstream import _yaml
from buildstream._yamlcache import YamlCache
from buildstream._project import Project
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index 2e12e712c..459241209 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -21,7 +21,7 @@
import os
import pytest
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
pytestmark = pytest.mark.integration
diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py
index b1d33f497..3ed0533cf 100644
--- a/tests/integration/autotools.py
+++ b/tests/integration/autotools.py
@@ -1,8 +1,8 @@
import os
import pytest
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/build-tree.py b/tests/integration/build-tree.py
index bacbf1cc7..98bb5b1e8 100644
--- a/tests/integration/build-tree.py
+++ b/tests/integration/build-tree.py
@@ -2,8 +2,9 @@ import os
import pytest
import shutil
-from tests.testutils import cli, cli_integration, create_artifact_share
+from tests.testutils import create_artifact_share
from tests.testutils.site import HAVE_SANDBOX
+from buildstream.plugintestutils import cli, cli_integration
from buildstream._exceptions import ErrorDomain
diff --git a/tests/integration/build-uid.py b/tests/integration/build-uid.py
index 06779ee20..88b887b5e 100644
--- a/tests/integration/build-uid.py
+++ b/tests/integration/build-uid.py
@@ -3,8 +3,8 @@ import pytest
from buildstream import _yaml
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import HAVE_BWRAP, IS_LINUX, HAVE_SANDBOX
diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py
index 08dbf81e1..5335ff5ed 100644
--- a/tests/integration/cachedfail.py
+++ b/tests/integration/cachedfail.py
@@ -3,10 +3,10 @@ import pytest
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain
+from buildstream.plugintestutils import cli_integration as cli
from conftest import clean_platform_cache
-
-from tests.testutils import cli_integration as cli, create_artifact_share
+from tests.testutils import create_artifact_share
from tests.testutils.site import HAVE_BWRAP, IS_LINUX, HAVE_SANDBOX
diff --git a/tests/integration/cmake.py b/tests/integration/cmake.py
index 7d03e291d..51e215655 100644
--- a/tests/integration/cmake.py
+++ b/tests/integration/cmake.py
@@ -1,8 +1,8 @@
import os
import pytest
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/compose-symlinks.py b/tests/integration/compose-symlinks.py
index 2599d8bcd..c6027bf2b 100644
--- a/tests/integration/compose-symlinks.py
+++ b/tests/integration/compose-symlinks.py
@@ -5,8 +5,8 @@ import pytest
from buildstream import _yaml
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import walk_dir
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import walk_dir
pytestmark = pytest.mark.integration
diff --git a/tests/integration/compose.py b/tests/integration/compose.py
index 077d5eea0..386e08438 100644
--- a/tests/integration/compose.py
+++ b/tests/integration/compose.py
@@ -5,8 +5,8 @@ import pytest
from buildstream import _yaml
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import walk_dir
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import walk_dir
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/import.py b/tests/integration/import.py
index 842640515..deef66bc1 100644
--- a/tests/integration/import.py
+++ b/tests/integration/import.py
@@ -3,8 +3,8 @@ import pytest
from buildstream import _yaml
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import walk_dir
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import walk_dir
pytestmark = pytest.mark.integration
diff --git a/tests/integration/make.py b/tests/integration/make.py
index 12c027b61..a76fe9a41 100644
--- a/tests/integration/make.py
+++ b/tests/integration/make.py
@@ -1,8 +1,8 @@
import os
import pytest
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/manual.py b/tests/integration/manual.py
index b9f09e25c..97cbec537 100644
--- a/tests/integration/manual.py
+++ b/tests/integration/manual.py
@@ -3,7 +3,7 @@ import pytest
from buildstream import _yaml
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/messages.py b/tests/integration/messages.py
index 03c922472..7f4194e0f 100644
--- a/tests/integration/messages.py
+++ b/tests/integration/messages.py
@@ -22,8 +22,7 @@ import pytest
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain
-
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/pip_element.py b/tests/integration/pip_element.py
index 807123620..48ca13bf5 100644
--- a/tests/integration/pip_element.py
+++ b/tests/integration/pip_element.py
@@ -4,8 +4,8 @@ import pytest
from buildstream import _yaml
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py
index d43b380aa..993b3a182 100644
--- a/tests/integration/pip_source.py
+++ b/tests/integration/pip_source.py
@@ -3,9 +3,9 @@ import pytest
from buildstream import _yaml
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.python_repo import setup_pypi_repo
-from tests.testutils.integration import assert_contains
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/pullbuildtrees.py b/tests/integration/pullbuildtrees.py
index ca6c40f94..24fac7e00 100644
--- a/tests/integration/pullbuildtrees.py
+++ b/tests/integration/pullbuildtrees.py
@@ -2,9 +2,12 @@ import os
import shutil
import pytest
-from tests.testutils import cli, cli_integration as cli2, create_artifact_share
-from tests.testutils.integration import assert_contains
+from tests.testutils import create_artifact_share
from tests.testutils.site import HAVE_SANDBOX
+
+from buildstream.plugintestutils.integration import assert_contains
+from buildstream.plugintestutils import cli, cli_integration as cli2
+from buildstream.plugintestutils.integration import assert_contains
from buildstream._exceptions import ErrorDomain, LoadErrorReason
diff --git a/tests/integration/sandbox-bwrap.py b/tests/integration/sandbox-bwrap.py
index 66e9f5b57..2c939e9be 100644
--- a/tests/integration/sandbox-bwrap.py
+++ b/tests/integration/sandbox-bwrap.py
@@ -3,8 +3,8 @@ import pytest
from buildstream._exceptions import ErrorDomain
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import HAVE_BWRAP, HAVE_BWRAP_JSON_STATUS
diff --git a/tests/integration/script.py b/tests/integration/script.py
index 446a63f3d..0297a3f0a 100644
--- a/tests/integration/script.py
+++ b/tests/integration/script.py
@@ -2,8 +2,7 @@ import os
import pytest
from buildstream import _yaml
-
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/shell.py b/tests/integration/shell.py
index 9b35db883..d1a551f53 100644
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -3,8 +3,7 @@ import pytest
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain
-
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/sockets.py b/tests/integration/sockets.py
index 30cf622cf..fc2fb2b92 100644
--- a/tests/integration/sockets.py
+++ b/tests/integration/sockets.py
@@ -2,9 +2,8 @@ import os
import pytest
from buildstream import _yaml
-
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/source-determinism.py b/tests/integration/source-determinism.py
index 6be6a4d4a..f7b2bf2ac 100644
--- a/tests/integration/source-determinism.py
+++ b/tests/integration/source-determinism.py
@@ -2,8 +2,8 @@ import os
import pytest
from buildstream import _yaml, utils
+from buildstream.plugintestutils import cli_integration as cli
from tests.testutils import create_repo, ALL_REPO_KINDS
-from tests.testutils import cli_integration as cli
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/stack.py b/tests/integration/stack.py
index 1b8d38198..d5208d7be 100644
--- a/tests/integration/stack.py
+++ b/tests/integration/stack.py
@@ -2,8 +2,7 @@ import os
import pytest
from buildstream import _yaml
-
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/symlinks.py b/tests/integration/symlinks.py
index c45c81177..22ff527f8 100644
--- a/tests/integration/symlinks.py
+++ b/tests/integration/symlinks.py
@@ -4,8 +4,8 @@ import pytest
from buildstream import _yaml
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
from tests.testutils.site import HAVE_SANDBOX
diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py
index ee1d575bf..f1d8d6fcd 100644
--- a/tests/integration/workspace.py
+++ b/tests/integration/workspace.py
@@ -2,9 +2,9 @@ import os
import pytest
from buildstream import _yaml
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
from tests.testutils.site import HAVE_SANDBOX
-from tests.testutils.integration import walk_dir
+from buildstream.plugintestutils.integration import walk_dir
pytestmark = pytest.mark.integration
diff --git a/tests/internals/storage_vdir_import.py b/tests/internals/storage_vdir_import.py
index 0bb47e3cd..1d61a6e5f 100644
--- a/tests/internals/storage_vdir_import.py
+++ b/tests/internals/storage_vdir_import.py
@@ -3,7 +3,7 @@ import os
import pytest
import random
import tempfile
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
from buildstream.storage._casbaseddirectory import CasBasedDirectory
from buildstream.storage._filebaseddirectory import FileBasedDirectory
diff --git a/tests/sandboxes/missing-command.py b/tests/sandboxes/missing-command.py
index 8f210bcec..ddf1c487f 100644
--- a/tests/sandboxes/missing-command.py
+++ b/tests/sandboxes/missing-command.py
@@ -3,7 +3,7 @@ import pytest
from buildstream._exceptions import ErrorDomain
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
DATA_DIR = os.path.join(
diff --git a/tests/sandboxes/missing_dependencies.py b/tests/sandboxes/missing_dependencies.py
index d77674c64..19515a187 100644
--- a/tests/sandboxes/missing_dependencies.py
+++ b/tests/sandboxes/missing_dependencies.py
@@ -1,6 +1,6 @@
import os
import pytest
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
from tests.testutils.site import IS_LINUX
from buildstream import _yaml
diff --git a/tests/sandboxes/remote-exec-config.py b/tests/sandboxes/remote-exec-config.py
index 9cf0a8f08..9864f2591 100644
--- a/tests/sandboxes/remote-exec-config.py
+++ b/tests/sandboxes/remote-exec-config.py
@@ -6,7 +6,7 @@ import os
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/sources/bzr.py b/tests/sources/bzr.py
index a56005f5e..bf7d1e844 100644
--- a/tests/sources/bzr.py
+++ b/tests/sources/bzr.py
@@ -4,7 +4,8 @@ import pytest
from buildstream._pipeline import PipelineError
from buildstream import _yaml
-from tests.testutils import cli, create_repo
+from buildstream.plugintestutils import cli
+from tests.testutils import create_repo
from tests.testutils.site import HAVE_BZR
DATA_DIR = os.path.join(
diff --git a/tests/sources/deb.py b/tests/sources/deb.py
index b40358eaa..480bea6f4 100644
--- a/tests/sources/deb.py
+++ b/tests/sources/deb.py
@@ -8,7 +8,7 @@ import shutil
from buildstream._exceptions import ErrorDomain
from buildstream import _yaml
from tempfile import TemporaryFile
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
from tests.testutils.site import HAVE_ARPY
from . import list_dir_contents
diff --git a/tests/sources/git.py b/tests/sources/git.py
index 795908329..f3d2e2809 100644
--- a/tests/sources/git.py
+++ b/tests/sources/git.py
@@ -28,8 +28,10 @@ import shutil
from buildstream._exceptions import ErrorDomain
from buildstream import _yaml
from buildstream.plugin import CoreWarnings
+from buildstream.plugintestutils import cli
-from tests.testutils import cli, create_repo
+from tests.testutils.site import HAVE_GIT, HAVE_OLD_GIT
+from tests.testutils import create_repo
from tests.testutils.site import HAVE_GIT, HAVE_OLD_GIT
DATA_DIR = os.path.join(
diff --git a/tests/sources/local.py b/tests/sources/local.py
index 4a0851d72..f7c1f4bd2 100644
--- a/tests/sources/local.py
+++ b/tests/sources/local.py
@@ -2,7 +2,8 @@ import os
import pytest
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import cli, filetypegenerator
+from buildstream.plugintestutils import cli
+from tests.testutils import filetypegenerator
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/sources/no_fetch_cached.py b/tests/sources/no_fetch_cached.py
index d2880f628..9ef838cf4 100644
--- a/tests/sources/no_fetch_cached.py
+++ b/tests/sources/no_fetch_cached.py
@@ -3,7 +3,8 @@ import pytest
from buildstream import _yaml
-from tests.testutils import cli, create_repo
+from buildstream.plugintestutils import cli
+from tests.testutils import create_repo
from tests.testutils.site import HAVE_GIT
DATA_DIR = os.path.join(
diff --git a/tests/sources/ostree.py b/tests/sources/ostree.py
index e059a882f..7ddea77f6 100644
--- a/tests/sources/ostree.py
+++ b/tests/sources/ostree.py
@@ -22,8 +22,9 @@ import pytest
from buildstream._exceptions import ErrorDomain
from buildstream import _yaml
+from buildstream.plugintestutils import cli
-from tests.testutils import cli, create_repo
+from tests.testutils import create_repo
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/sources/patch.py b/tests/sources/patch.py
index 51ae69021..2a82bf184 100644
--- a/tests/sources/patch.py
+++ b/tests/sources/patch.py
@@ -2,7 +2,8 @@ import os
import pytest
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import cli, filetypegenerator
+from buildstream.plugintestutils import cli
+from tests.testutils import filetypegenerator
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/sources/pip.py b/tests/sources/pip.py
index 6e1a347d9..3114a824e 100644
--- a/tests/sources/pip.py
+++ b/tests/sources/pip.py
@@ -4,7 +4,7 @@ import pytest
from buildstream._exceptions import ErrorDomain
from buildstream import _yaml
from buildstream.plugins.sources.pip import _match_package_name
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/sources/previous_source_access.py b/tests/sources/previous_source_access.py
index 26640d180..9d831781a 100644
--- a/tests/sources/previous_source_access.py
+++ b/tests/sources/previous_source_access.py
@@ -1,7 +1,7 @@
import os
import pytest
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/sources/remote.py b/tests/sources/remote.py
index 1d7727302..30b65b4cc 100644
--- a/tests/sources/remote.py
+++ b/tests/sources/remote.py
@@ -4,7 +4,7 @@ import pytest
from buildstream._exceptions import ErrorDomain
from buildstream import _yaml
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
from tests.testutils.file_server import create_file_server
DATA_DIR = os.path.join(
diff --git a/tests/sources/tar.py b/tests/sources/tar.py
index 406d6702b..959ff890c 100644
--- a/tests/sources/tar.py
+++ b/tests/sources/tar.py
@@ -8,7 +8,7 @@ from shutil import copyfile, rmtree
from buildstream._exceptions import ErrorDomain
from buildstream import _yaml
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
from tests.testutils.file_server import create_file_server
from tests.testutils.site import HAVE_LZIP
from . import list_dir_contents
diff --git a/tests/sources/zip.py b/tests/sources/zip.py
index 009862e1c..dd72dd579 100644
--- a/tests/sources/zip.py
+++ b/tests/sources/zip.py
@@ -4,7 +4,7 @@ import zipfile
from buildstream._exceptions import ErrorDomain
from buildstream import _yaml
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
from tests.testutils.file_server import create_file_server
from . import list_dir_contents
diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py
index eb7211ea8..173cc7c3e 100644
--- a/tests/testutils/__init__.py
+++ b/tests/testutils/__init__.py
@@ -23,7 +23,6 @@
# William Salmon <will.salmon@codethink.co.uk>
#
-from .runcli import cli, cli_integration
from .repo import create_repo, ALL_REPO_KINDS
from .artifactshare import create_artifact_share
from .element_generators import create_element_size, update_element_size
diff --git a/tests/testutils/integration.py b/tests/testutils/integration.py
deleted file mode 100644
index b2cf9fba4..000000000
--- a/tests/testutils/integration.py
+++ /dev/null
@@ -1,29 +0,0 @@
-import os
-
-from buildstream import _yaml
-
-
-# Return a list of files relative to the given directory
-def walk_dir(root):
- for dirname, dirnames, filenames in os.walk(root):
- # ensure consistent traversal order, needed for consistent
- # handling of symlinks.
- dirnames.sort()
- filenames.sort()
-
- # print path to all subdirectories first.
- for subdirname in dirnames:
- yield os.path.join(dirname, subdirname)[len(root):]
-
- # print path to all filenames.
- for filename in filenames:
- yield os.path.join(dirname, filename)[len(root):]
-
-
-# Ensure that a directory contains the given filenames.
-def assert_contains(directory, expected):
- missing = set(expected)
- missing.difference_update(walk_dir(directory))
- if len(missing) > 0:
- raise AssertionError("Missing {} expected elements from list: {}"
- .format(len(missing), missing))
diff --git a/tox.ini b/tox.ini
index 6683df796..9183e023f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -70,6 +70,7 @@ deps =
sphinx
sphinx-click
sphinx_rtd_theme >= 0.4.2
+ pytest
-rrequirements/requirements.txt
-rrequirements/plugin-requirements.txt
passenv =