summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTom Pollard <tom.pollard@codethink.co.uk>2019-03-21 12:59:56 +0000
committerbst-marge-bot <marge-bot@buildstream.build>2019-03-29 10:38:53 +0000
commitd93763daa96c159fbf891219b71a7e5e9a31a971 (patch)
tree8298ed673c395104bbf311cb12b5801be6a5583d
parent35616512815cc3edc22b7aab128744a81b21c32f (diff)
downloadbuildstream-d93763daa96c159fbf891219b71a7e5e9a31a971.tar.gz
plugintestutils/runcli.py: Add TestArtifact() 'abstraction' class
This adds the initial Artifact 'abstractions', taking existing test methods into a dedicated Artifact class. The class is accessed via an instance attribute of the generic Cli() fixture. This is in preperation for AaaP, to ensure existing tests pass whilst creating an entry point for further abstractions when necessary.
-rw-r--r--buildstream/plugintestutils/runcli.py112
-rw-r--r--tests/artifactcache/pull.py12
-rw-r--r--tests/artifactcache/push.py5
-rw-r--r--tests/integration/artifact.py22
-rw-r--r--tests/integration/pullbuildtrees.py20
5 files changed, 123 insertions, 48 deletions
diff --git a/buildstream/plugintestutils/runcli.py b/buildstream/plugintestutils/runcli.py
index 1ddb95745..72bdce09e 100644
--- a/buildstream/plugintestutils/runcli.py
+++ b/buildstream/plugintestutils/runcli.py
@@ -52,7 +52,7 @@ from _pytest.capture import MultiCapture, FDCapture, FDCaptureBinary
# Import the main cli entrypoint
from buildstream._frontend import cli as bst_cli
from buildstream import _yaml
-
+from buildstream._cas import CASCache
# Special private exception accessor, for test case purposes
from buildstream._exceptions import BstError, get_last_exception, get_last_task_error
@@ -253,6 +253,7 @@ class Cli():
self.directory = directory
self.config = None
self.verbose = verbose
+ self.artifact = TestArtifact()
if default_options is None:
default_options = []
@@ -274,6 +275,15 @@ class Cli():
for key, val in config.items():
self.config[key] = val
+ # remove_artifact_from_cache():
+ #
+ # Remove given element artifact from artifact cache
+ #
+ # Args:
+ # project (str): The project path under test
+ # element_name (str): The name of the element artifact
+ # cache_dir (str): Specific cache dir to remove artifact from
+ #
def remove_artifact_from_cache(self, project, element_name,
*, cache_dir=None):
# Read configuration to figure out where artifacts are stored
@@ -285,10 +295,7 @@ class Cli():
else:
cache_dir = default
- cache_dir = os.path.join(cache_dir, 'cas', 'refs', 'heads')
-
- cache_dir = os.path.splitext(os.path.join(cache_dir, 'test', element_name))[0]
- shutil.rmtree(cache_dir)
+ self.artifact.remove_artifact_from_cache(cache_dir, element_name)
# run():
#
@@ -617,6 +624,101 @@ class CliRemote(CliIntegration):
return configured_services
+class TestArtifact():
+
+ # remove_artifact_from_cache():
+ #
+ # Remove given element artifact from artifact cache
+ #
+ # Args:
+ # cache_dir (str): Specific cache dir to remove artifact from
+ # element_name (str): The name of the element artifact
+ #
+ def remove_artifact_from_cache(self, cache_dir, element_name):
+
+ cache_dir = os.path.join(cache_dir, 'cas', 'refs', 'heads')
+
+ cache_dir = os.path.splitext(os.path.join(cache_dir, 'test', element_name))[0]
+ shutil.rmtree(cache_dir)
+
+ # is_cached():
+ #
+ # Check if given element has a cached artifact
+ #
+ # Args:
+ # cache_dir (str): Specific cache dir to check
+ # element (Element): The element object
+ # element_key (str): The element's cache key
+ #
+ # Returns:
+ # (bool): If the cache contains the element's artifact
+ #
+ def is_cached(self, cache_dir, element, element_key):
+
+ cas = CASCache(str(cache_dir))
+ artifact_ref = element.get_artifact_name(element_key)
+ return cas.contains(artifact_ref)
+
+ # get_digest():
+ #
+ # Get the digest for a given element's artifact
+ #
+ # Args:
+ # cache_dir (str): Specific cache dir to check
+ # element (Element): The element object
+ # element_key (str): The element's cache key
+ #
+ # Returns:
+ # (Digest): The digest stored in the ref
+ #
+ def get_digest(self, cache_dir, element, element_key):
+
+ cas = CASCache(str(cache_dir))
+ artifact_ref = element.get_artifact_name(element_key)
+ digest = cas.resolve_ref(artifact_ref)
+ return digest
+
+ # extract_buildtree():
+ #
+ # Context manager for extracting an elements artifact buildtree for
+ # inspection.
+ #
+ # Args:
+ # tmpdir (LocalPath): pytest fixture for the tests tmp dir
+ # digest (Digest): The element directory digest to extract
+ #
+ # Yields:
+ # (str): path to extracted buildtree directory, does not guarantee
+ # existence.
+ @contextmanager
+ def extract_buildtree(self, tmpdir, digest):
+ with self._extract_subdirectory(tmpdir, digest, 'buildtree') as extract:
+ yield extract
+
+ # _extract_subdirectory():
+ #
+ # Context manager for extracting an element artifact for inspection,
+ # providing an expected path for a given subdirectory
+ #
+ # Args:
+ # tmpdir (LocalPath): pytest fixture for the tests tmp dir
+ # digest (Digest): The element directory digest to extract
+ # subdir (str): Subdirectory to path
+ #
+ # Yields:
+ # (str): path to extracted subdir directory, does not guarantee
+ # existence.
+ @contextmanager
+ def _extract_subdirectory(self, tmpdir, digest, subdir):
+ with tempfile.TemporaryDirectory() as extractdir:
+ try:
+ cas = CASCache(str(tmpdir))
+ cas.checkout(extractdir, digest)
+ yield os.path.join(extractdir, subdir)
+ except FileNotFoundError:
+ yield None
+
+
# Main fixture
#
# Use result = cli.run([arg1, arg2]) to run buildstream commands
diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py
index 3c10c256c..d75b6d3c7 100644
--- a/tests/artifactcache/pull.py
+++ b/tests/artifactcache/pull.py
@@ -95,15 +95,14 @@ def test_pull(cli, tmpdir, datafiles):
context.load(config=user_config_file)
context.set_message_handler(message_handler)
- # Load the project and CAS cache
+ # Load the project
project = Project(project_dir, context)
project.ensure_fully_loaded()
- cas = context.artifactcache
# Assert that the element's artifact is **not** cached
element = project.load_elements(['target.bst'])[0]
element_key = cli.get_element_key(project_dir, 'target.bst')
- assert not cas.contains(element, element_key)
+ assert not cli.artifact.is_cached(cache_dir, element, element_key)
queue = multiprocessing.Queue()
# Use subprocess to avoid creation of gRPC threads in main BuildStream process
@@ -124,7 +123,7 @@ def test_pull(cli, tmpdir, datafiles):
raise
assert not error
- assert cas.contains(element, element_key)
+ assert cli.artifact.is_cached(cache_dir, element, element_key)
def _test_pull(user_config_file, project_dir, cache_dir,
@@ -209,11 +208,10 @@ def test_pull_tree(cli, tmpdir, datafiles):
# Assert that the element's artifact is cached
element = project.load_elements(['target.bst'])[0]
element_key = cli.get_element_key(project_dir, 'target.bst')
- assert artifactcache.contains(element, element_key)
+ assert cli.artifact.is_cached(rootcache_dir, element, element_key)
# Retrieve the Directory object from the cached artifact
- artifact_ref = element.get_artifact_name(element_key)
- artifact_digest = cas.resolve_ref(artifact_ref)
+ artifact_digest = cli.artifact.get_digest(rootcache_dir, element, element_key)
queue = multiprocessing.Queue()
# Use subprocess to avoid creation of gRPC threads in main BuildStream process
diff --git a/tests/artifactcache/push.py b/tests/artifactcache/push.py
index 56af50a0d..426d22f24 100644
--- a/tests/artifactcache/push.py
+++ b/tests/artifactcache/push.py
@@ -75,13 +75,10 @@ def test_push(cli, tmpdir, datafiles):
project = Project(project_dir, context)
project.ensure_fully_loaded()
- # Create a local CAS cache handle
- cas = context.artifactcache
-
# Assert that the element's artifact is cached
element = project.load_elements(['target.bst'])[0]
element_key = cli.get_element_key(project_dir, 'target.bst')
- assert cas.contains(element, element_key)
+ assert cli.artifact.is_cached(rootcache_dir, element, element_key)
queue = multiprocessing.Queue()
# Use subprocess to avoid creation of gRPC threads in main BuildStream process
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index f9470ed6b..0718de893 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -58,16 +58,6 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
'cachedir': str(tmpdir)
})
- @contextmanager
- def cas_extract_buildtree(digest):
- extractdir = tempfile.mkdtemp(prefix="tmp", dir=str(tmpdir))
- try:
- cas = CASCache(str(tmpdir))
- cas.checkout(extractdir, digest)
- yield os.path.join(extractdir, 'buildtree')
- finally:
- utils._force_rmtree(extractdir)
-
# Build autotools element with the default behavior of caching buildtrees
# only when necessary. The artifact should be successfully pushed to the share1 remote
# and cached locally with an 'empty' buildtree digest, as it's not a
@@ -80,7 +70,7 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
# The buildtree dir should not exist, as we set the config to not cache buildtrees.
cache_key = cli.get_element_key(project, element_name)
elementdigest = share1.has_artifact('test', element_name, cache_key)
- with cas_extract_buildtree(elementdigest) as buildtreedir:
+ with cli.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
assert not os.path.isdir(buildtreedir)
# Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees
@@ -89,7 +79,7 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
assert cli.get_element_state(project, element_name) != 'cached'
result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
- with cas_extract_buildtree(elementdigest) as buildtreedir:
+ with cli.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
assert not os.path.isdir(buildtreedir)
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
@@ -99,7 +89,7 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
# leading to no buildtreedir being extracted
result = cli.run(project=project, args=['artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
- with cas_extract_buildtree(elementdigest) as buildtreedir:
+ with cli.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
assert not os.path.isdir(buildtreedir)
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
@@ -116,7 +106,7 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
# Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths
elementdigest = share2.has_artifact('test', element_name, cache_key)
- with cas_extract_buildtree(elementdigest) as buildtreedir:
+ with cli.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
@@ -126,7 +116,7 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
assert cli.get_element_state(project, element_name) != 'cached'
result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
- with cas_extract_buildtree(elementdigest) as buildtreedir:
+ with cli.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
@@ -144,6 +134,6 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
assert cli.get_element_state(project, element_name) == 'cached'
cache_key = cli.get_element_key(project, element_name)
elementdigest = share3.has_artifact('test', element_name, cache_key)
- with cas_extract_buildtree(elementdigest) as buildtreedir:
+ with cli.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
diff --git a/tests/integration/pullbuildtrees.py b/tests/integration/pullbuildtrees.py
index e8f84d50d..1a89f776a 100644
--- a/tests/integration/pullbuildtrees.py
+++ b/tests/integration/pullbuildtrees.py
@@ -51,18 +51,6 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
'cache': {'cache-buildtrees': 'always'},
})
- @contextmanager
- def cas_extract_buildtree(digest):
- extractdir = tempfile.mkdtemp(prefix="tmp", dir=str(tmpdir))
- try:
- cas = CASCache(str(tmpdir))
- cas.checkout(extractdir, digest)
- yield os.path.join(extractdir, 'buildtree')
- except FileNotFoundError:
- yield None
- finally:
- utils._force_rmtree(extractdir)
-
# Build autotools element, checked pushed, delete local
result = cli2.run(project=project, args=['build', element_name])
assert result.exit_code == 0
@@ -88,11 +76,11 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
result = cli2.run(project=project, args=['artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
elementdigest = share1.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
- with cas_extract_buildtree(elementdigest) as buildtreedir:
+ with cli2.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
assert not buildtreedir
result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
- with cas_extract_buildtree(elementdigest) as buildtreedir:
+ with cli2.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
assert os.path.isdir(buildtreedir)
default_state(cli2, tmpdir, share1)
@@ -151,7 +139,7 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'push', element_name])
assert "Attempting to fetch missing artifact buildtrees" in result.stderr
assert element_name not in result.get_pulled_elements()
- with cas_extract_buildtree(elementdigest) as buildtreedir:
+ with cli2.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
assert not buildtreedir
assert element_name not in result.get_pushed_elements()
assert not share3.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
@@ -164,7 +152,7 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'push', element_name])
assert "Attempting to fetch missing artifact buildtrees" in result.stderr
assert element_name in result.get_pulled_elements()
- with cas_extract_buildtree(elementdigest) as buildtreedir:
+ with cli2.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert element_name in result.get_pushed_elements()
assert share3.has_artifact('test', element_name, cli2.get_element_key(project, element_name))