tests/frontend: Updating tests to use new error checks

This also fixes #177 - the problem here was solved simply
by passing the project directory to `cli.run(project=project...)`

12 files changed, 50 insertions, 57 deletions

diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py
index b5b294512..1a5b08caa 100644
--- a/tests/frontend/buildcheckout.py
+++ b/tests/frontend/buildcheckout.py
@@ -3,7 +3,7 @@ import pytest
 from tests.testutils import cli, create_repo, ALL_REPO_KINDS
 
 from buildstream import _yaml
-from buildstream._pipeline import PipelineError
+from buildstream._exceptions import ErrorDomain
 
 # Project directory
 DATA_DIR = os.path.join(
@@ -31,7 +31,7 @@ def test_build_checkout(datafiles, cli, strict, hardlinks):
 
     # First build it
     result = cli.run(project=project, args=strict_args(['build', 'target.bst'], strict))
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert that after a successful build, the builddir is empty
     builddir = os.path.join(cli.directory, 'build')
@@ -46,7 +46,7 @@ def test_build_checkout(datafiles, cli, strict, hardlinks):
 
     # Now check it out
     result = cli.run(project=project, args=checkout_args)
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Check that the executable hello file is found in the checkout
     filename = os.path.join(checkout, 'usr', 'bin', 'hello')
@@ -66,7 +66,7 @@ def test_build_checkout_nonempty(datafiles, cli, hardlinks):
 
     # First build it
     result = cli.run(project=project, args=['build', 'target.bst'])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert that after a successful build, the builddir is empty
     builddir = os.path.join(cli.directory, 'build')
@@ -86,8 +86,7 @@ def test_build_checkout_nonempty(datafiles, cli, hardlinks):
 
     # Now check it out
     result = cli.run(project=project, args=checkout_args)
-    assert result.exit_code != 0
-    assert isinstance(result.exception, PipelineError)
+    result.assert_main_error(ErrorDomain.PIPELINE, None)
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -99,7 +98,7 @@ def test_build_checkout_force(datafiles, cli, hardlinks):
 
     # First build it
     result = cli.run(project=project, args=['build', 'target.bst'])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert that after a successful build, the builddir is empty
     builddir = os.path.join(cli.directory, 'build')
@@ -119,7 +118,7 @@ def test_build_checkout_force(datafiles, cli, hardlinks):
 
     # Now check it out
     result = cli.run(project=project, args=checkout_args)
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Check that the file we added is still there
     filename = os.path.join(checkout, 'file.txt')
@@ -167,14 +166,14 @@ def test_fetch_build_checkout(cli, tmpdir, datafiles, strict, kind):
 
     assert cli.get_element_state(project, element_name) == 'fetch needed'
     result = cli.run(project=project, args=strict_args(['build', element_name], strict))
-    assert result.exit_code == 0
+    result.assert_success()
     assert cli.get_element_state(project, element_name) == 'cached'
 
     # Now check it out
     result = cli.run(project=project, args=strict_args([
         'checkout', element_name, checkout
     ], strict))
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Check that the pony.h include from files/dev-files exists
     filename = os.path.join(checkout, 'usr', 'include', 'pony.h')
@@ -190,10 +189,6 @@ def test_install_to_build(cli, tmpdir, datafiles):
     # We expect this to throw an ElementError, since the element will
     # attempt to stage into /buildstream/build, which is not allowed.
     result = cli.run(project=project, args=strict_args(['build', element], True))
-    assert result.exit_code != 0
 
-    # While the error thrown is an ElementError, the pipeline picks up
-    # the error and raises it as a PipelineError. Since we are testing
-    # through the cli runner, we can't be more specific.
-    assert result.exception
-    assert isinstance(result.exception, PipelineError)
+    result.assert_main_error(ErrorDomain.PIPELINE, None)
+    result.assert_task_error(ErrorDomain.ELEMENT, None)
diff --git a/tests/frontend/buildtrack.py b/tests/frontend/buildtrack.py
index 45dd30d82..c351080b9 100644
--- a/tests/frontend/buildtrack.py
+++ b/tests/frontend/buildtrack.py
@@ -7,7 +7,7 @@ import pytest
 from tests.testutils import cli, create_repo
 
 from buildstream import _yaml
-from buildstream._exceptions import LoadError
+from buildstream._exceptions import ErrorDomain
 
 
 # Project directory
@@ -254,10 +254,8 @@ def test_build_track_inconsistent(cli, datafiles, tmpdir,
     args += itertools.chain.from_iterable(zip(itertools.repeat('--track-except'), exceptions))
     args += ['0.bst']
 
-    result = cli.run(args=args, silent=True)
-
-    assert result.exit_code != 0
-    assert isinstance(result.exception, LoadError)
+    result = cli.run(project=project, args=args, silent=True)
+    result.assert_main_error(ErrorDomain.PIPELINE, "inconsistent-pipeline")
 
 
 # Assert that if a build element has a dependency in the tracking
@@ -290,14 +288,14 @@ def test_build_track_track_first(cli, datafiles, tmpdir, strict):
     # Build 1.bst and 2.bst first so we have an artifact for them
     args = [strict, 'build', '2.bst']
     result = cli.run(args=args, project=project, silent=True)
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Test building 0.bst while tracking 1.bst
     cli.remove_artifact_from_cache(project, '0.bst')
 
     args = [strict, 'build', '--track', '1.bst', '2.bst']
     result = cli.run(args=args, project=project, silent=True)
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert that 1.bst successfully tracks before 0.bst builds
     track_messages = re.finditer(r'\[track:1.bst\s*]', result.stderr)
diff --git a/tests/frontend/compose_splits.py b/tests/frontend/compose_splits.py
index fd65493bb..b5a2e760d 100644
--- a/tests/frontend/compose_splits.py
+++ b/tests/frontend/compose_splits.py
@@ -20,13 +20,13 @@ def test_compose_splits(datafiles, cli, target):
 
     # First build it
     result = cli.run(project=project, args=['build', target])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Now check it out
     result = cli.run(project=project, args=[
         'checkout', target, checkout
     ])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Check that the executable hello file is found in the checkout
     filename = os.path.join(checkout, 'usr', 'bin', 'hello')
diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py
index e8c22c6f7..6f497fdab 100644
--- a/tests/frontend/fetch.py
+++ b/tests/frontend/fetch.py
@@ -41,7 +41,7 @@ def test_fetch(cli, tmpdir, datafiles, kind):
 
     # Now try to fetch it
     result = cli.run(project=project, args=['fetch', element_name])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert that we are now buildable because the source is
     # now cached.
diff --git a/tests/frontend/help.py b/tests/frontend/help.py
index 7eb017c2e..bdafe851b 100644
--- a/tests/frontend/help.py
+++ b/tests/frontend/help.py
@@ -13,7 +13,7 @@ def assert_help(cli_output):
 
 def test_help_main(cli):
     result = cli.run(args=['--help'])
-    assert result.exit_code == 0
+    result.assert_success()
     assert_help(result.output)
 
 
@@ -31,5 +31,5 @@ def test_help_main(cli):
 ])
 def test_help(cli, command):
     result = cli.run(args=[command, '--help'])
-    assert result.exit_code == 0
+    result.assert_success()
     assert_help(result.output)
diff --git a/tests/frontend/overlaps.py b/tests/frontend/overlaps.py
index 4ef5657fb..38ebe1f5f 100644
--- a/tests/frontend/overlaps.py
+++ b/tests/frontend/overlaps.py
@@ -15,7 +15,7 @@ def test_overlaps(cli, datafiles):
     result = cli.run(project=project, silent=True, args=[
         'build', 'collect.bst'])
 
-    assert result.exit_code == 0
+    result.assert_success()
     print(result.output)
     assert "/file1: three.bst above one.bst" in result.stderr
     assert "/file2: two.bst above three.bst above one.bst" in result.stderr
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index 96e1c68a5..530c1e047 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -40,7 +40,7 @@ def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url):
 
     # First build it without the artifact cache configured
     result = cli.run(project=project, args=['build', 'import-bin.bst'])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert that we are now cached locally
     state = cli.get_element_state(project, 'import-bin.bst')
@@ -76,7 +76,7 @@ def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url):
 
     # Now try bst push
     result = cli.run(project=project, args=['push', 'import-bin.bst'])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # And finally assert that the artifact is in the share
     assert_shared(cli, share, project, 'import-bin.bst')
@@ -99,7 +99,7 @@ def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url):
 
     # Now try bst pull
     result = cli.run(project=project, args=['pull', 'import-bin.bst'])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # And assert that it's again in the local cache, without having built
     state = cli.get_element_state(project, 'import-bin.bst')
@@ -114,7 +114,7 @@ def test_push_pull_all(cli, tmpdir, datafiles):
 
     # First build it without the artifact cache configured
     result = cli.run(project=project, args=['build', 'target.bst'])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert that we are now cached locally
     state = cli.get_element_state(project, 'target.bst')
@@ -140,7 +140,7 @@ def test_push_pull_all(cli, tmpdir, datafiles):
 
     # Now try bst push
     result = cli.run(project=project, args=['push', '--deps', 'all', 'target.bst'])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # And finally assert that the artifact is in the share
     all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst']
@@ -166,7 +166,7 @@ def test_push_pull_all(cli, tmpdir, datafiles):
 
     # Now try bst pull
     result = cli.run(project=project, args=['pull', '--deps', 'all', 'target.bst'])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # And assert that it's again in the local cache, without having built
     for element_name in all_elements:
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index ced42acf3..9f8bc5b62 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -39,7 +39,7 @@ def test_push(cli, tmpdir, datafiles, override_url, user_url, project_url):
 
     # First build it without the artifact cache configured
     result = cli.run(project=project, args=['build', 'target.bst'])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert that we are now cached locally
     state = cli.get_element_state(project, 'target.bst')
@@ -75,7 +75,7 @@ def test_push(cli, tmpdir, datafiles, override_url, user_url, project_url):
 
     # Now try bst push
     result = cli.run(project=project, args=['push', 'target.bst'])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # And finally assert that the artifact is in the share
     assert_shared(cli, share, project, 'target.bst')
@@ -89,7 +89,7 @@ def test_push_all(cli, tmpdir, datafiles):
 
     # First build it without the artifact cache configured
     result = cli.run(project=project, args=['build', 'target.bst'])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert that we are now cached locally
     state = cli.get_element_state(project, 'target.bst')
@@ -118,7 +118,7 @@ def test_push_all(cli, tmpdir, datafiles):
         'push', 'target.bst',
         '--deps', 'all'
     ])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # And finally assert that all the artifacts are in the share
     assert_shared(cli, share, project, 'target.bst')
diff --git a/tests/frontend/show.py b/tests/frontend/show.py
index 2c2f03d96..fda5214bf 100644
--- a/tests/frontend/show.py
+++ b/tests/frontend/show.py
@@ -24,7 +24,7 @@ def test_show(cli, datafiles, target, format, expected):
         '--deps', 'none',
         '--format', format,
         target])
-    assert result.exit_code == 0
+    result.assert_success()
 
     if result.output.strip() != expected:
         raise AssertionError("Expected output:\n{}\nInstead received output:\n{}"
@@ -48,7 +48,7 @@ def test_show_except(cli, datafiles, target, except_, expected):
         '--except', except_,
         target])
 
-    assert result.exit_code == 0
+    result.assert_success()
 
     results = result.output.strip().splitlines()
     if results != expected:
@@ -68,7 +68,7 @@ def test_parallel_order(cli, tmpdir, datafiles):
     args = ['show', '-d', 'plan', '-f', '%{name}'] + elements
     result = cli.run(project=project, args=args)
 
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Get the planned order
     names = result.output.splitlines()
@@ -97,7 +97,7 @@ def test_target_is_dependency(cli, tmpdir, datafiles):
     args = ['show', '-d', 'plan', '-f', '%{name}'] + elements
     result = cli.run(project=project, args=args)
 
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Get the planned order
     names = result.output.splitlines()
diff --git a/tests/frontend/track.py b/tests/frontend/track.py
index f83237545..aa8cff6d1 100644
--- a/tests/frontend/track.py
+++ b/tests/frontend/track.py
@@ -46,14 +46,14 @@ def test_track(cli, tmpdir, datafiles, kind):
 
     # Now first try to track it
     result = cli.run(project=project, args=['track', element_name])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # And now fetch it: The Source has probably already cached the
     # latest ref locally, but it is not required to have cached
     # the associated content of the latest ref at track time, that
     # is the job of fetch.
     result = cli.run(project=project, args=['fetch', element_name])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert that we are now buildable because the source is
     # now cached.
@@ -88,7 +88,7 @@ def test_track_recurse(cli, tmpdir, datafiles, kind):
     result = cli.run(project=project, args=[
         'track', '--deps', 'all',
         element_target_name])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # And now fetch it: The Source has probably already cached the
     # latest ref locally, but it is not required to have cached
@@ -97,7 +97,7 @@ def test_track_recurse(cli, tmpdir, datafiles, kind):
     result = cli.run(project=project, args=[
         'fetch', '--deps', 'all',
         element_target_name])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert that the dependency is buildable and the target is waiting
     assert cli.get_element_state(project, element_dep_name) == 'buildable'
@@ -132,7 +132,7 @@ def test_track_recurse_except(cli, tmpdir, datafiles, kind):
     result = cli.run(project=project, args=[
         'track', '--deps', 'all', '--except', element_dep_name,
         element_target_name])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # And now fetch it: The Source has probably already cached the
     # latest ref locally, but it is not required to have cached
@@ -141,7 +141,7 @@ def test_track_recurse_except(cli, tmpdir, datafiles, kind):
     result = cli.run(project=project, args=[
         'fetch', '--deps', 'none',
         element_target_name])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert that the dependency is buildable and the target is waiting
     assert cli.get_element_state(project, element_dep_name) == 'no reference'
diff --git a/tests/frontend/version.py b/tests/frontend/version.py
index 69112d685..8be2e9271 100644
--- a/tests/frontend/version.py
+++ b/tests/frontend/version.py
@@ -16,5 +16,5 @@ def assert_version(cli_version_output):
 
 def test_version(cli):
     result = cli.run(args=['--version'])
-    assert result.exit_code == 0
+    result.assert_success()
     assert_version(result.output)
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index db0414a91..4bfc00e7d 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -55,7 +55,7 @@ def open_workspace(cli, tmpdir, datafiles, kind, track):
     args.extend([element_name, workspace])
 
     result = cli.run(project=project, args=args)
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert that we are now buildable because the source is
     # now cached.
@@ -90,7 +90,7 @@ def test_close(cli, tmpdir, datafiles, kind):
     result = cli.run(project=project, args=[
         'workspace', 'close', '--remove-dir', element_name
     ])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert the workspace dir has been deleted
     assert not os.path.exists(workspace)
@@ -109,7 +109,7 @@ def test_close_removed(cli, tmpdir, datafiles, kind):
     result = cli.run(project=project, args=[
         'workspace', 'close', element_name
     ])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Assert the workspace dir has been deleted
     assert not os.path.exists(workspace)
@@ -132,7 +132,7 @@ def test_reset(cli, tmpdir, datafiles, kind):
     result = cli.run(project=project, args=[
         'workspace', 'reset', element_name
     ])
-    assert result.exit_code == 0
+    result.assert_success()
     assert os.path.exists(os.path.join(workspace, 'usr', 'bin', 'hello'))
     assert not os.path.exists(os.path.join(workspace, 'etc', 'pony.conf'))
 
@@ -147,7 +147,7 @@ def test_list(cli, tmpdir, datafiles, kind):
     result = cli.run(project=project, args=[
         'workspace', 'list'
     ])
-    assert result.exit_code == 0
+    result.assert_success()
 
     loaded = _yaml.load_data(result.output)
     assert isinstance(loaded.get('workspaces'), list)
@@ -174,14 +174,14 @@ def test_build(cli, tmpdir, datafiles, kind):
     # Build modified workspace
     assert cli.get_element_state(project, element_name) == 'buildable'
     result = cli.run(project=project, args=['build', element_name])
-    assert result.exit_code == 0
+    result.assert_success()
     assert cli.get_element_state(project, element_name) == 'cached'
 
     # Checkout the result
     result = cli.run(project=project, args=[
         'checkout', element_name, checkout
     ])
-    assert result.exit_code == 0
+    result.assert_success()
 
     # Check that the pony.conf from the modified workspace exists
     filename = os.path.join(checkout, 'etc', 'pony.conf')