tests/format/variables.py: Added some new tests

  * Test scenarios where a junction needs to resolve variables
    in order to configure a subproject, but where some other variables
    may be derived from the same subproject.

    In this scenario we allow partial resolution of variables
    for junction elements.

  * Enhanced the undefined variables and circular reference tests
    to also check for the expected provenances.

  * Test for deep variable resolution

    Test variable indirection with 50, 500 and 5000 variables:

    * 50 - tests generally large indirections in the recursive algorithm,
           which is limited to 200 recursions

    * 500 - tests that the non-recursive algorithm works for successful outcomes
            and not only for error resolution

    * 5000 - tests that the iterative algorithm works and ensures it
             is not discarded, as a recursive algorithm cannot be implemented
             to support this depth with python (which limits itself to
             merely 1000 stack frames).

7 files changed, 118 insertions, 18 deletions

diff --git a/tests/format/variables.py b/tests/format/variables.py
index fd6288fb9..a888818e8 100644
--- a/tests/format/variables.py
+++ b/tests/format/variables.py
@@ -65,26 +65,59 @@ def test_overrides(cli, datafiles, tmpdir, target, varname, expected):
     assert result_vars[varname] == expected
 
 
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'missing_variables'))
-def test_missing_variable(cli, datafiles, tmpdir):
-    project = os.path.join(datafiles.dirname, datafiles.basename)
-    result = cli.run(project=project, silent=True, args=[
-        'show', '--deps', 'none', '--format', '%{config}', 'manual.bst'
-    ])
-    result.assert_main_error(ErrorDomain.LOAD,
-                             LoadErrorReason.UNRESOLVED_VARIABLE)
+@pytest.mark.parametrize(
+    "element,provenance",
+    [
+        # This test makes a reference to an undefined variable in a build command
+        ("manual.bst", "manual.bst [line 5 column 6]"),
+        # This test makes a reference to an undefined variable by another variable,
+        # ensuring that we validate variables even when they are unused
+        ("manual2.bst", "manual2.bst [line 4 column 8]"),
+        # This test uses a build command to refer to some variables which ultimately
+        # refer to an undefined variable, testing a more complex case.
+        ("manual3.bst", "manual3.bst [line 6 column 8]"),
+    ],
+    ids=["build-command", "variables", "complex"],
+)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "missing_variables"))
+def test_undefined(cli, datafiles, element, provenance):
+    project = str(datafiles)
+    result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{config}", element])
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNRESOLVED_VARIABLE)
+    assert provenance in result.stderr
 
 
-@pytest.mark.timeout(3, method="signal")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'cyclic_variables'))
-def test_cyclic_variables(cli, datafiles):
-    print_warning("Performing cyclic test, if this test times out it will " +
-                  "exit the test sequence")
-    project = os.path.join(datafiles.dirname, datafiles.basename)
-    result = cli.run(project=project, silent=True, args=[
-        "build", "cyclic.bst"
-    ])
+@pytest.mark.parametrize(
+    "element,provenances",
+    [
+        # Test a simple a -> b and b -> a reference
+        ("simple-cyclic.bst", ["simple-cyclic.bst [line 4 column 5]", "simple-cyclic.bst [line 5 column 5]"]),
+        # Test a simple a -> b and b -> a reference with some text involved
+        ("cyclic.bst", ["cyclic.bst [line 5 column 10]", "cyclic.bst [line 4 column 5]"]),
+        # Test an indirect circular dependency
+        (
+            "indirect-cyclic.bst",
+            [
+                "indirect-cyclic.bst [line 5 column 5]",
+                "indirect-cyclic.bst [line 6 column 5]",
+                "indirect-cyclic.bst [line 7 column 5]",
+                "indirect-cyclic.bst [line 8 column 5]",
+            ],
+        ),
+        # Test an indirect circular dependency
+        ("self-reference.bst", ["self-reference.bst [line 4 column 5]"]),
+    ],
+    ids=["simple", "simple-text", "indirect", "self-reference"],
+)
+@pytest.mark.timeout(15, method="signal")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "cyclic_variables"))
+def test_circular_reference(cli, datafiles, element, provenances):
+    print_warning("Performing cyclic test, if this test times out it will exit the test sequence")
+    project = str(datafiles)
+    result = cli.run(project=project, silent=True, args=["build", element])
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.CIRCULAR_REFERENCE_VARIABLE)
+    for provenance in provenances:
+        assert provenance in result.stderr
 
 
 def print_warning(msg):
@@ -92,6 +125,42 @@ def print_warning(msg):
     print(("\n{}{}{}").format(RED, msg, END), file=sys.stderr)
 
 
+# Test that variables which refer to eachother very deeply are
+# still resolved correctly, this ensures that we are not relying
+# on a recursive algorithm limited by stack depth.
+#
+@pytest.mark.parametrize(
+    "maxvars", [50, 500, 5000],
+)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "defaults"))
+def test_deep_references(cli, datafiles, maxvars):
+    project = str(datafiles)
+
+    # Generate an element with very, very many variables to resolve,
+    # each which expand to the value of the previous variable.
+    #
+    # The bottom variable defines a test value which we check for
+    # in the top variable in `bst show` output.
+    #
+    topvar = "var{}".format(maxvars)
+    bottomvar = "var0"
+    testvalue = "testvalue {}".format(maxvars)
+
+    # Generate
+    variables = {"var{}".format(idx + 1): "%{var" + str(idx) + "}" for idx in range(maxvars)}
+    variables[bottomvar] = testvalue
+    element = {"kind": "manual", "variables": variables}
+    _yaml.dump(element, os.path.join(project, "test.bst"))
+
+    # Run `bst show`
+    result = cli.run(project=project, args=["show", "--format", "%{vars}", "test.bst"])
+    result.assert_success()
+
+    # Test results
+    result_vars = _yaml.load_data(result.output)
+    assert result_vars[topvar] == testvalue
+
+
 @pytest.mark.datafiles(os.path.join(DATA_DIR, "partial_context"))
 def test_partial_context_junctions(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/format/variables/cyclic_variables/cyclic.bst b/tests/format/variables/cyclic_variables/cyclic.bst
index a05a40b27..38832fa86 100644
--- a/tests/format/variables/cyclic_variables/cyclic.bst
+++ b/tests/format/variables/cyclic_variables/cyclic.bst
@@ -2,4 +2,4 @@ kind: manual
 
 variables:
   a: "%{prefix}/a"
-  prefix: "%{a}/some_prefix/"
\ No newline at end of file
+  prefix: "%{a}/some_prefix/"
diff --git a/tests/format/variables/cyclic_variables/indirect-cyclic.bst b/tests/format/variables/cyclic_variables/indirect-cyclic.bst
new file mode 100644
index 000000000..fb06fb008
--- /dev/null
+++ b/tests/format/variables/cyclic_variables/indirect-cyclic.bst
@@ -0,0 +1,8 @@
+kind: manual
+
+variables:
+  foo: "%{a}"
+  a: "%{b}"
+  b: "%{c}"
+  c: "%{d}"
+  d: "%{a}"
diff --git a/tests/format/variables/cyclic_variables/self-reference.bst b/tests/format/variables/cyclic_variables/self-reference.bst
new file mode 100644
index 000000000..2e9829d03
--- /dev/null
+++ b/tests/format/variables/cyclic_variables/self-reference.bst
@@ -0,0 +1,4 @@
+kind: manual
+
+variables:
+  a: "Referencing itself with %{a}"
diff --git a/tests/format/variables/cyclic_variables/simple-cyclic.bst b/tests/format/variables/cyclic_variables/simple-cyclic.bst
new file mode 100644
index 000000000..806e1f390
--- /dev/null
+++ b/tests/format/variables/cyclic_variables/simple-cyclic.bst
@@ -0,0 +1,5 @@
+kind: manual
+
+variables:
+  a: "%{b}"
+  b: "%{a}"
diff --git a/tests/format/variables/missing_variables/manual2.bst b/tests/format/variables/missing_variables/manual2.bst
new file mode 100644
index 000000000..bd8e2baf7
--- /dev/null
+++ b/tests/format/variables/missing_variables/manual2.bst
@@ -0,0 +1,4 @@
+kind: manual
+
+variables:
+  test: hello %{missing}
diff --git a/tests/format/variables/missing_variables/manual3.bst b/tests/format/variables/missing_variables/manual3.bst
new file mode 100644
index 000000000..ff3c8d583
--- /dev/null
+++ b/tests/format/variables/missing_variables/manual3.bst
@@ -0,0 +1,10 @@
+kind: manual
+
+variables:
+  hello: "Hello mister %{pony}"
+  greeting: "The %{hello} string twice: %{hello} again"
+  pony: "The pony is %{undefined}"
+  
+config:
+  build-commands:
+  - Some indirectly undefined variable %{greeting}