From a0517233e23fe1c9645c5099d4644b775290040d Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Sun, 28 Jun 2020 23:59:42 +0900 Subject: tests/format/variables.py: Added some new tests * Test scenarios where a junction needs to resolve variables in order to configure a subproject, but where some other variables may be derived from the same subproject. In this scenario we allow partial resolution of variables for junction elements. * Enhanced the undefined variables and circular reference tests to also check for the expected provenances. * Test for deep variable resolution Test variable indirection with 50, 500 and 5000 variables: * 50 - tests generally large indirections in the recursive algorithm, which is limited to 200 recursions * 500 - tests that the non-recursive algorithm works for successful outcomes and not only for error resolution * 5000 - tests that the iterative algorithm works and ensures it is not discarded, as a recursive algorithm cannot be implemented to support this depth with python (which limits itself to merely 1000 stack frames). --- tests/format/variables.py | 103 +++++++++++++++++---- tests/format/variables/cyclic_variables/cyclic.bst | 2 +- .../variables/cyclic_variables/indirect-cyclic.bst | 8 ++ .../variables/cyclic_variables/self-reference.bst | 4 + .../variables/cyclic_variables/simple-cyclic.bst | 5 + .../format/variables/missing_variables/manual2.bst | 4 + .../format/variables/missing_variables/manual3.bst | 10 ++ 7 files changed, 118 insertions(+), 18 deletions(-) create mode 100644 tests/format/variables/cyclic_variables/indirect-cyclic.bst create mode 100644 tests/format/variables/cyclic_variables/self-reference.bst create mode 100644 tests/format/variables/cyclic_variables/simple-cyclic.bst create mode 100644 tests/format/variables/missing_variables/manual2.bst create mode 100644 tests/format/variables/missing_variables/manual3.bst diff --git a/tests/format/variables.py b/tests/format/variables.py index fd6288fb9..a888818e8 100644 --- a/tests/format/variables.py +++ b/tests/format/variables.py @@ -65,26 +65,59 @@ def test_overrides(cli, datafiles, tmpdir, target, varname, expected): assert result_vars[varname] == expected -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'missing_variables')) -def test_missing_variable(cli, datafiles, tmpdir): - project = os.path.join(datafiles.dirname, datafiles.basename) - result = cli.run(project=project, silent=True, args=[ - 'show', '--deps', 'none', '--format', '%{config}', 'manual.bst' - ]) - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.UNRESOLVED_VARIABLE) +@pytest.mark.parametrize( + "element,provenance", + [ + # This test makes a reference to an undefined variable in a build command + ("manual.bst", "manual.bst [line 5 column 6]"), + # This test makes a reference to an undefined variable by another variable, + # ensuring that we validate variables even when they are unused + ("manual2.bst", "manual2.bst [line 4 column 8]"), + # This test uses a build command to refer to some variables which ultimately + # refer to an undefined variable, testing a more complex case. + ("manual3.bst", "manual3.bst [line 6 column 8]"), + ], + ids=["build-command", "variables", "complex"], +) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "missing_variables")) +def test_undefined(cli, datafiles, element, provenance): + project = str(datafiles) + result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{config}", element]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNRESOLVED_VARIABLE) + assert provenance in result.stderr -@pytest.mark.timeout(3, method="signal") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'cyclic_variables')) -def test_cyclic_variables(cli, datafiles): - print_warning("Performing cyclic test, if this test times out it will " + - "exit the test sequence") - project = os.path.join(datafiles.dirname, datafiles.basename) - result = cli.run(project=project, silent=True, args=[ - "build", "cyclic.bst" - ]) +@pytest.mark.parametrize( + "element,provenances", + [ + # Test a simple a -> b and b -> a reference + ("simple-cyclic.bst", ["simple-cyclic.bst [line 4 column 5]", "simple-cyclic.bst [line 5 column 5]"]), + # Test a simple a -> b and b -> a reference with some text involved + ("cyclic.bst", ["cyclic.bst [line 5 column 10]", "cyclic.bst [line 4 column 5]"]), + # Test an indirect circular dependency + ( + "indirect-cyclic.bst", + [ + "indirect-cyclic.bst [line 5 column 5]", + "indirect-cyclic.bst [line 6 column 5]", + "indirect-cyclic.bst [line 7 column 5]", + "indirect-cyclic.bst [line 8 column 5]", + ], + ), + # Test an indirect circular dependency + ("self-reference.bst", ["self-reference.bst [line 4 column 5]"]), + ], + ids=["simple", "simple-text", "indirect", "self-reference"], +) +@pytest.mark.timeout(15, method="signal") +@pytest.mark.datafiles(os.path.join(DATA_DIR, "cyclic_variables")) +def test_circular_reference(cli, datafiles, element, provenances): + print_warning("Performing cyclic test, if this test times out it will exit the test sequence") + project = str(datafiles) + result = cli.run(project=project, silent=True, args=["build", element]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.CIRCULAR_REFERENCE_VARIABLE) + for provenance in provenances: + assert provenance in result.stderr def print_warning(msg): @@ -92,6 +125,42 @@ def print_warning(msg): print(("\n{}{}{}").format(RED, msg, END), file=sys.stderr) +# Test that variables which refer to eachother very deeply are +# still resolved correctly, this ensures that we are not relying +# on a recursive algorithm limited by stack depth. +# +@pytest.mark.parametrize( + "maxvars", [50, 500, 5000], +) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "defaults")) +def test_deep_references(cli, datafiles, maxvars): + project = str(datafiles) + + # Generate an element with very, very many variables to resolve, + # each which expand to the value of the previous variable. + # + # The bottom variable defines a test value which we check for + # in the top variable in `bst show` output. + # + topvar = "var{}".format(maxvars) + bottomvar = "var0" + testvalue = "testvalue {}".format(maxvars) + + # Generate + variables = {"var{}".format(idx + 1): "%{var" + str(idx) + "}" for idx in range(maxvars)} + variables[bottomvar] = testvalue + element = {"kind": "manual", "variables": variables} + _yaml.dump(element, os.path.join(project, "test.bst")) + + # Run `bst show` + result = cli.run(project=project, args=["show", "--format", "%{vars}", "test.bst"]) + result.assert_success() + + # Test results + result_vars = _yaml.load_data(result.output) + assert result_vars[topvar] == testvalue + + @pytest.mark.datafiles(os.path.join(DATA_DIR, "partial_context")) def test_partial_context_junctions(cli, datafiles): project = str(datafiles) diff --git a/tests/format/variables/cyclic_variables/cyclic.bst b/tests/format/variables/cyclic_variables/cyclic.bst index a05a40b27..38832fa86 100644 --- a/tests/format/variables/cyclic_variables/cyclic.bst +++ b/tests/format/variables/cyclic_variables/cyclic.bst @@ -2,4 +2,4 @@ kind: manual variables: a: "%{prefix}/a" - prefix: "%{a}/some_prefix/" \ No newline at end of file + prefix: "%{a}/some_prefix/" diff --git a/tests/format/variables/cyclic_variables/indirect-cyclic.bst b/tests/format/variables/cyclic_variables/indirect-cyclic.bst new file mode 100644 index 000000000..fb06fb008 --- /dev/null +++ b/tests/format/variables/cyclic_variables/indirect-cyclic.bst @@ -0,0 +1,8 @@ +kind: manual + +variables: + foo: "%{a}" + a: "%{b}" + b: "%{c}" + c: "%{d}" + d: "%{a}" diff --git a/tests/format/variables/cyclic_variables/self-reference.bst b/tests/format/variables/cyclic_variables/self-reference.bst new file mode 100644 index 000000000..2e9829d03 --- /dev/null +++ b/tests/format/variables/cyclic_variables/self-reference.bst @@ -0,0 +1,4 @@ +kind: manual + +variables: + a: "Referencing itself with %{a}" diff --git a/tests/format/variables/cyclic_variables/simple-cyclic.bst b/tests/format/variables/cyclic_variables/simple-cyclic.bst new file mode 100644 index 000000000..806e1f390 --- /dev/null +++ b/tests/format/variables/cyclic_variables/simple-cyclic.bst @@ -0,0 +1,5 @@ +kind: manual + +variables: + a: "%{b}" + b: "%{a}" diff --git a/tests/format/variables/missing_variables/manual2.bst b/tests/format/variables/missing_variables/manual2.bst new file mode 100644 index 000000000..bd8e2baf7 --- /dev/null +++ b/tests/format/variables/missing_variables/manual2.bst @@ -0,0 +1,4 @@ +kind: manual + +variables: + test: hello %{missing} diff --git a/tests/format/variables/missing_variables/manual3.bst b/tests/format/variables/missing_variables/manual3.bst new file mode 100644 index 000000000..ff3c8d583 --- /dev/null +++ b/tests/format/variables/missing_variables/manual3.bst @@ -0,0 +1,10 @@ +kind: manual + +variables: + hello: "Hello mister %{pony}" + greeting: "The %{hello} string twice: %{hello} again" + pony: "The pony is %{undefined}" + +config: + build-commands: + - Some indirectly undefined variable %{greeting} -- cgit v1.2.1