tests: ensure provenance for _get_loader errors

Make the tests that currently cover _get_loader ensure that we are
getting the expected provenance.

Note that for some tests, we must use yaml_file_get_provenance, as the
generated yaml is not stable across versions of ruamel.

In later work we may replace all instances of provenance string tests
with yaml_file_get_provenance, as it will be more robust to future
changes.

5 files changed, 52 insertions, 9 deletions

diff --git a/tests/format/junctions.py b/tests/format/junctions.py
index 210455cb4..fc2dbc155 100644
--- a/tests/format/junctions.py
+++ b/tests/format/junctions.py
@@ -81,6 +81,9 @@ def test_junction_missing_project_conf(cli, datafiles):
     result = cli.run(project=project, args=['build', 'app.bst'])
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION)
 
+    # Assert that we have the expected provenance encoded into the error
+    assert "app.bst [line 6 column 2]" in result.stderr
+
 
 @pytest.mark.datafiles(DATA_DIR)
 def test_workspaced_junction_missing_project_conf(cli, datafiles):
@@ -102,6 +105,9 @@ def test_workspaced_junction_missing_project_conf(cli, datafiles):
     result = cli.run(project=project, args=['build', 'app.bst'])
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION)
 
+    # Assert that we have the expected provenance encoded into the error
+    assert "app.bst [line 6 column 2]" in result.stderr
+
 
 @pytest.mark.datafiles(DATA_DIR)
 def test_build_of_same_junction_used_twice(cli, datafiles):
@@ -203,6 +209,8 @@ def test_nested_conflict(cli, datafiles):
     result = cli.run(project=project, args=['build', 'target.bst'])
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.CONFLICTING_JUNCTION)
 
+    assert "bar.bst:target.bst [line 3 column 2]" in result.stderr
+
 
 # Test that we error correctly when the junction element itself is missing
 @pytest.mark.datafiles(DATA_DIR)
@@ -316,6 +324,9 @@ def test_git_show(cli, tmpdir, datafiles):
     result = cli.run(project=project, args=['show', 'target.bst'])
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_FETCH_NEEDED)
 
+    # Assert that we have the expected provenance encoded into the error
+    assert "target.bst [line 3 column 2]" in result.stderr
+
     # Explicitly fetch subproject
     result = cli.run(project=project, args=['source', 'fetch', 'base.bst'])
     result.assert_success()
@@ -380,6 +391,9 @@ def test_git_missing_project_conf(cli, tmpdir, datafiles):
     result = cli.run(project=project, args=['build', 'app.bst'])
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION)
 
+    # Assert that we have the expected provenance encoded into the error
+    assert "app.bst [line 6 column 2]" in result.stderr
+
 
 @pytest.mark.datafiles(DATA_DIR)
 def test_cross_junction_names(cli, datafiles):
diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py
index dc3c5e4d4..5145b1222 100644
--- a/tests/frontend/buildcheckout.py
+++ b/tests/frontend/buildcheckout.py
@@ -9,7 +9,7 @@ import subprocess
 import pytest
 
 from tests.testutils.site import IS_WINDOWS
-from tests.testutils import generate_junction
+from tests.testutils import generate_junction, yaml_file_get_provenance
 
 from buildstream.plugintestutils import cli  # pylint: disable=unused-import
 from buildstream import _yaml
@@ -457,6 +457,11 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage):
     result = cli.run(project=project, args=['build', 'junction-dep.bst'])
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
 
+    # Assert that we have the expected provenance encoded into the error
+    provenance = yaml_file_get_provenance(
+        element_path, 'junction-dep.bst', key='depends', indices=[0])
+    assert str(provenance) in result.stderr
+
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py
index 1a2c40602..8282e2131 100644
--- a/tests/frontend/fetch.py
+++ b/tests/frontend/fetch.py
@@ -4,7 +4,7 @@
 import os
 import pytest
 
-from tests.testutils import create_repo, generate_junction
+from tests.testutils import create_repo, generate_junction, yaml_file_get_provenance
 
 from buildstream.plugintestutils import cli  # pylint: disable=unused-import
 from buildstream import _yaml
@@ -160,3 +160,8 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage):
     # informing the user to track the junction first
     result = cli.run(project=project, args=['source', 'fetch', 'junction-dep.bst'])
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
+
+    # Assert that we have the expected provenance encoded into the error
+    provenance = yaml_file_get_provenance(
+        element_path, 'junction-dep.bst', key='depends', indices=[0])
+    assert str(provenance) in result.stderr
diff --git a/tests/frontend/show.py b/tests/frontend/show.py
index bb2fc2ee2..7e5ebfb77 100644
--- a/tests/frontend/show.py
+++ b/tests/frontend/show.py
@@ -6,7 +6,7 @@ import sys
 import shutil
 import itertools
 import pytest
-from tests.testutils import generate_junction
+from tests.testutils import generate_junction, yaml_file_get_provenance
 from buildstream.plugintestutils import cli  # pylint: disable=unused-import
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
@@ -277,8 +277,7 @@ def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage, element_name):
 
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
 @pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
-@pytest.mark.parametrize("element_name", ['junction-dep.bst', 'junction.bst:import-etc.bst'])
-def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, element_name):
+def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage):
     project = str(datafiles)
     subproject_path = os.path.join(project, 'files', 'sub-project')
     junction_path = os.path.join(project, 'elements', 'junction.bst')
@@ -305,10 +304,20 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, element_name
     _yaml.dump(element, element_path)
 
     # Assert the correct error when trying to show the pipeline
-    result = cli.run(project=project, silent=True, args=[
-        'show', element_name])
+    dep_result = cli.run(project=project, silent=True, args=[
+        'show', 'junction-dep.bst'])
+
+    # Assert the correct error when trying to show the pipeline
+    etc_result = cli.run(project=project, silent=True, args=[
+        'show', 'junction.bst:import-etc.bst'])
+
+    # Assert that we have the expected provenance encoded into the error
+    provenance = yaml_file_get_provenance(
+        element_path, 'junction-dep.bst', key='depends', indices=[0])
+    assert str(provenance) in dep_result.stderr
 
-    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
+    dep_result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
+    etc_result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
 
 
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
diff --git a/tests/frontend/track.py b/tests/frontend/track.py
index 5a6460d3d..13d1d4646 100644
--- a/tests/frontend/track.py
+++ b/tests/frontend/track.py
@@ -4,7 +4,7 @@
 import stat
 import os
 import pytest
-from tests.testutils import create_repo, generate_junction
+from tests.testutils import create_repo, generate_junction, yaml_file_get_provenance
 
 from buildstream.plugintestutils import cli  # pylint: disable=unused-import
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
@@ -274,6 +274,11 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage):
     result = cli.run(project=project, args=['source', 'track', 'junction-dep.bst'])
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
 
+    # Assert that we have the expected provenance encoded into the error
+    provenance = yaml_file_get_provenance(
+        element_path, 'junction-dep.bst', key='depends', indices=[0])
+    assert str(provenance) in result.stderr
+
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
@@ -307,6 +312,11 @@ def test_junction_element(cli, tmpdir, datafiles, ref_storage):
     result = cli.run(project=project, args=['show', 'junction-dep.bst'])
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
 
+    # Assert that we have the expected provenance encoded into the error
+    provenance = yaml_file_get_provenance(
+        element_path, 'junction-dep.bst', key='depends', indices=[0])
+    assert str(provenance) in result.stderr
+
     # Now track the junction itself
     result = cli.run(project=project, args=['source', 'track', 'junction.bst'])
     result.assert_success()