From 805baf7d96ade33c8bbe3f38122283e705000d81 Mon Sep 17 00:00:00 2001 From: Tom Pollard Date: Thu, 10 Jan 2019 11:09:40 +0000 Subject: Download buildtrees on demand for bst shell --use-buildtree Provide bst shell --use-buildtree the ability to attempt to acquire missing buildtrees, given respective option, user pull-buildtree context and remote availability. _frontend/cli.py: Refactor logic for determining --use-buildtree option with given opportunity to attempt pulling a non-local buildtree. Element loaded with artifact_config to allow remote querying. _stream.py: With given user option and element state, construct PullQueue to fetch remote buildtree. Continue or Error without buildtree if cannot be attained. tests/integration/build-tree.py: Update to support new usecases --- tests/integration/build-tree.py | 48 +++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 11 deletions(-) (limited to 'tests') diff --git a/tests/integration/build-tree.py b/tests/integration/build-tree.py index b50d84152..b1a41aefe 100644 --- a/tests/integration/build-tree.py +++ b/tests/integration/build-tree.py @@ -101,7 +101,7 @@ def test_buildtree_from_failure(cli_integration, tmpdir, datafiles): 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test' ]) res.assert_success() - assert "Warning: using a buildtree from a failed build" in res.output + assert "WARNING: using a buildtree from a failed build" in res.stderr assert 'Hi' in res.output @@ -141,7 +141,7 @@ def test_buildtree_pulled(cli, tmpdir, datafiles): res.assert_success() -# This test checks for correct behaviour if a buildtree is not present. +# This test checks for correct behaviour if a buildtree is not present in the local cache. @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux') def test_buildtree_options(cli, tmpdir, datafiles): @@ -156,6 +156,7 @@ def test_buildtree_options(cli, tmpdir, datafiles): result = cli.run(project=project, args=['build', element_name]) result.assert_success() assert cli.get_element_state(project, element_name) == 'cached' + assert share.has_artifact('test', element_name, cli.get_element_key(project, element_name)) # Discard the cache cli.configure({ @@ -168,8 +169,6 @@ def test_buildtree_options(cli, tmpdir, datafiles): result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', element_name]) result.assert_success() - # The above is the simplest way I know to create a local cache without any buildtrees. - # Check it's not using the cached build tree res = cli.run(project=project, args=[ 'shell', '--build', element_name, '--use-buildtree', 'never', '--', 'cat', 'test' @@ -177,24 +176,51 @@ def test_buildtree_options(cli, tmpdir, datafiles): res.assert_shell_error() assert 'Hi' not in res.output - # Check it's not correctly handling the lack of buildtree + # Check it's not using the cached build tree, default is to ask, and fall back to not + # for non interactive behavior res = cli.run(project=project, args=[ - 'shell', '--build', element_name, '--use-buildtree', 'try', '--', 'cat', 'test' + 'shell', '--build', element_name, '--', 'cat', 'test' ]) res.assert_shell_error() assert 'Hi' not in res.output - # Check it's not using the cached build tree, default is to ask, and fall back to not - # for non interactive behavior + # Check correctly handling the lack of buildtree, with 'try' not attempting to + # pull the buildtree as the user context is by default set to not pull them res = cli.run(project=project, args=[ - 'shell', '--build', element_name, '--', 'cat', 'test' + 'shell', '--build', element_name, '--use-buildtree', 'try', '--', 'cat', 'test' ]) - res.assert_shell_error() assert 'Hi' not in res.output + assert 'Attempting to fetch missing artifact buildtrees' not in res.stderr + assert """Buildtree is not cached locally or in available remotes, + shell will be loaded without it""" - # Check it's using the cached build tree + # Check correctly handling the lack of buildtree, with 'try' attempting and succeeding + # to pull the buildtree as the user context allow the pulling of buildtrees and it is + # available in the remote + res = cli.run(project=project, args=[ + '--pull-buildtrees', 'shell', '--build', element_name, '--use-buildtree', 'try', '--', 'cat', 'test' + ]) + assert 'Attempting to fetch missing artifact buildtree' in res.stderr + assert 'Hi' in res.output + shutil.rmtree(os.path.join(os.path.join(cli.directory, 'artifacts2'))) + assert cli.get_element_state(project, element_name) != 'cached' + + # Check it's not loading the shell at all with always set for the buildtree, when the + # user context does not allow for buildtree pulling + result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', element_name]) + result.assert_success() res = cli.run(project=project, args=[ 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test' ]) res.assert_main_error(ErrorDomain.PROG_NOT_FOUND, None) + assert 'Buildtree is not cached locally or in available remotes' in res.stderr assert 'Hi' not in res.output + assert 'Attempting to fetch missing artifact buildtree' not in res.stderr + + # Check that when user context is set to pull buildtrees and a remote has the buildtree, + # 'always' will attempt and succeed at pulling the missing buildtree. + res = cli.run(project=project, args=[ + '--pull-buildtrees', 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test' + ]) + assert 'Hi' in res.output + assert 'Attempting to fetch missing artifact buildtree' in res.stderr -- cgit v1.2.1