From f9c82abb547aa0155ab07b7744800790ff3f13a3 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Fri, 24 Jul 2020 18:21:49 +0900 Subject: _yaml.py: Support merging of conditional lists During composition of project.conf, it can happen that we are including and composing node hierarchies before options are resolved and before we are ready to evaluate conditional statements. Without this patch, conditional statements are treated like regular lists, causing existing conditional statements to be overwritten by subsequently composited conditional statements without ever having been resolved. This patch introduces a special case for composing lists, when composing a sequence that is a conditional statement, the sequence is appended instead of overwriting the underlying list. Note that precedence of declarations in including vs included yaml fragments did raise some concern, and a test in the following commit is added to cover this case. Composition of conditional lists on top of other conditional lists are unconditionally appended, however the semantics of includes ensures that the including fragment's conditionals is always composited *on top* over the included fragment, ensuring that we have the correct pecedence even when compositing yet to be resolved conditional directives. --- buildstream/_yaml.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py index 4f2f88329..405a10ec8 100644 --- a/buildstream/_yaml.py +++ b/buildstream/_yaml.py @@ -738,7 +738,20 @@ def composite_list(target_node, source_node, key): "{}: List cannot overwrite value at: {}" .format(source_key_provenance, target_key_provenance)) - composite_list_overwrite(target_node, key, source_node, key) + # Special case: The `project.conf` in some cases needs to composite + # include files before having resolved options, so there can be + # conditionals that need to be merged at this point. + # + # This unconditionally appends conditional statements to a matching + # conditional in the target so as to preserve them. The precedence + # of include files is preserved regardless due to the order in which + # included dictionaries are composited. + # + if key == "(?)": + composite_list_append(target_node, key, source_node, key) + else: + composite_list_overwrite(target_node, key, source_node, key) + return True # When a composite list is encountered in the source, then -- cgit v1.2.1 From 840c32d61eb934ad94e0ee8d0a7b69bceaf27156 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Tue, 21 Jul 2020 17:44:57 +0900 Subject: tests/format/include.py: Test preservation of conditionals at include time Added tests to ensure that conditional statements don't get overwritten when performing composition of one dictionary on top of another due to include processing. --- tests/format/include.py | 24 ++++++++++++++++++++++ .../conditional-conflicts-complex/element.bst | 1 + .../enable_work_around.yml | 5 +++++ .../conditional-conflicts-complex/extra_conf.yml | 5 +++++ .../extra_conf_i586.yml | 4 ++++ .../extra_conf_x86_64.yml | 4 ++++ .../conditional-conflicts-complex/options.yml | 9 ++++++++ .../conditional-conflicts-complex/project.conf | 6 ++++++ .../conditional-conflicts-complex/work_around.yml | 3 +++ .../conditional-conflicts-element/element.bst | 5 +++++ .../conditional-conflicts-element/extra_conf.yml | 6 ++++++ .../conditional-conflicts-element/project.conf | 10 +++++++++ .../conditional-conflicts-element/work_around.yml | 5 +++++ .../element.bst | 1 + .../extra_conf.yml | 6 ++++++ .../options.yml | 9 ++++++++ .../project.conf | 6 ++++++ .../work_around.yml | 5 +++++ .../conditional-conflicts-project/element.bst | 1 + .../conditional-conflicts-project/extra_conf.yml | 6 ++++++ .../conditional-conflicts-project/project.conf | 14 +++++++++++++ .../conditional-conflicts-project/work_around.yml | 5 +++++ .../element.bst | 1 + .../extra_conf.yml | 6 ++++++ .../project.conf | 22 ++++++++++++++++++++ .../work_around.yml | 5 +++++ 26 files changed, 174 insertions(+) create mode 100644 tests/format/include/conditional-conflicts-complex/element.bst create mode 100644 tests/format/include/conditional-conflicts-complex/enable_work_around.yml create mode 100644 tests/format/include/conditional-conflicts-complex/extra_conf.yml create mode 100644 tests/format/include/conditional-conflicts-complex/extra_conf_i586.yml create mode 100644 tests/format/include/conditional-conflicts-complex/extra_conf_x86_64.yml create mode 100644 tests/format/include/conditional-conflicts-complex/options.yml create mode 100644 tests/format/include/conditional-conflicts-complex/project.conf create mode 100644 tests/format/include/conditional-conflicts-complex/work_around.yml create mode 100644 tests/format/include/conditional-conflicts-element/element.bst create mode 100644 tests/format/include/conditional-conflicts-element/extra_conf.yml create mode 100644 tests/format/include/conditional-conflicts-element/project.conf create mode 100644 tests/format/include/conditional-conflicts-element/work_around.yml create mode 100644 tests/format/include/conditional-conflicts-options-included/element.bst create mode 100644 tests/format/include/conditional-conflicts-options-included/extra_conf.yml create mode 100644 tests/format/include/conditional-conflicts-options-included/options.yml create mode 100644 tests/format/include/conditional-conflicts-options-included/project.conf create mode 100644 tests/format/include/conditional-conflicts-options-included/work_around.yml create mode 100644 tests/format/include/conditional-conflicts-project/element.bst create mode 100644 tests/format/include/conditional-conflicts-project/extra_conf.yml create mode 100644 tests/format/include/conditional-conflicts-project/project.conf create mode 100644 tests/format/include/conditional-conflicts-project/work_around.yml create mode 100644 tests/format/include/conditional-conflicts-toplevel-precedence/element.bst create mode 100644 tests/format/include/conditional-conflicts-toplevel-precedence/extra_conf.yml create mode 100644 tests/format/include/conditional-conflicts-toplevel-precedence/project.conf create mode 100644 tests/format/include/conditional-conflicts-toplevel-precedence/work_around.yml diff --git a/tests/format/include.py b/tests/format/include.py index cfbfb66e3..f844eadc5 100644 --- a/tests/format/include.py +++ b/tests/format/include.py @@ -205,6 +205,30 @@ def test_conditional_in_fragment(cli, tmpdir, datafiles): assert loaded['size'] == '8' +@pytest.mark.parametrize( + "project_dir", + [ + "conditional-conflicts-project", + "conditional-conflicts-element", + "conditional-conflicts-options-included", + "conditional-conflicts-complex", + "conditional-conflicts-toplevel-precedence", + ], +) +@pytest.mark.datafiles(DATA_DIR) +def test_preserve_conditionals(cli, datafiles, project_dir): + project = os.path.join(str(datafiles), project_dir) + + result = cli.run( + project=project, + args=["-o", "build_arch", "i586", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], + ) + result.assert_success() + loaded = _yaml.load_data(result.output) + assert loaded["enable-work-around"] == "true" + assert loaded["size"] == "4" + + @pytest.mark.datafiles(DATA_DIR) def test_inner(cli, datafiles): project = os.path.join(str(datafiles), 'inner') diff --git a/tests/format/include/conditional-conflicts-complex/element.bst b/tests/format/include/conditional-conflicts-complex/element.bst new file mode 100644 index 000000000..4d7f70266 --- /dev/null +++ b/tests/format/include/conditional-conflicts-complex/element.bst @@ -0,0 +1 @@ +kind: manual diff --git a/tests/format/include/conditional-conflicts-complex/enable_work_around.yml b/tests/format/include/conditional-conflicts-complex/enable_work_around.yml new file mode 100644 index 000000000..7e56ae727 --- /dev/null +++ b/tests/format/include/conditional-conflicts-complex/enable_work_around.yml @@ -0,0 +1,5 @@ +variables: + enable-work-around: "false" + (?): + - build_arch == "i586": + enable-work-around: "true" diff --git a/tests/format/include/conditional-conflicts-complex/extra_conf.yml b/tests/format/include/conditional-conflicts-complex/extra_conf.yml new file mode 100644 index 000000000..a25eabe23 --- /dev/null +++ b/tests/format/include/conditional-conflicts-complex/extra_conf.yml @@ -0,0 +1,5 @@ +(?): +- build_arch == "i586": + (@): extra_conf_i586.yml +- build_arch == "x86_64": + (@): extra_conf_x86_64.yml diff --git a/tests/format/include/conditional-conflicts-complex/extra_conf_i586.yml b/tests/format/include/conditional-conflicts-complex/extra_conf_i586.yml new file mode 100644 index 000000000..caf872b98 --- /dev/null +++ b/tests/format/include/conditional-conflicts-complex/extra_conf_i586.yml @@ -0,0 +1,4 @@ +variables: + (?): + - build_arch == "i586": + size: 4 diff --git a/tests/format/include/conditional-conflicts-complex/extra_conf_x86_64.yml b/tests/format/include/conditional-conflicts-complex/extra_conf_x86_64.yml new file mode 100644 index 000000000..9c5f64630 --- /dev/null +++ b/tests/format/include/conditional-conflicts-complex/extra_conf_x86_64.yml @@ -0,0 +1,4 @@ +variables: + (?): + - build_arch == "x86_64": + size: 8 diff --git a/tests/format/include/conditional-conflicts-complex/options.yml b/tests/format/include/conditional-conflicts-complex/options.yml new file mode 100644 index 000000000..8bb07305a --- /dev/null +++ b/tests/format/include/conditional-conflicts-complex/options.yml @@ -0,0 +1,9 @@ + +options: + build_arch: + type: arch + description: Architecture + variable: build_arch + values: + - i586 + - x86_64 diff --git a/tests/format/include/conditional-conflicts-complex/project.conf b/tests/format/include/conditional-conflicts-complex/project.conf new file mode 100644 index 000000000..2c7ca7044 --- /dev/null +++ b/tests/format/include/conditional-conflicts-complex/project.conf @@ -0,0 +1,6 @@ +name: test + +(@): +- extra_conf.yml +- work_around.yml +- options.yml diff --git a/tests/format/include/conditional-conflicts-complex/work_around.yml b/tests/format/include/conditional-conflicts-complex/work_around.yml new file mode 100644 index 000000000..d7e3ec1dc --- /dev/null +++ b/tests/format/include/conditional-conflicts-complex/work_around.yml @@ -0,0 +1,3 @@ +(?): +- build_arch == "i586": + (@): enable_work_around.yml diff --git a/tests/format/include/conditional-conflicts-element/element.bst b/tests/format/include/conditional-conflicts-element/element.bst new file mode 100644 index 000000000..bb1ba25d4 --- /dev/null +++ b/tests/format/include/conditional-conflicts-element/element.bst @@ -0,0 +1,5 @@ +kind: manual + +(@): + - extra_conf.yml + - work_around.yml diff --git a/tests/format/include/conditional-conflicts-element/extra_conf.yml b/tests/format/include/conditional-conflicts-element/extra_conf.yml new file mode 100644 index 000000000..dd58c9855 --- /dev/null +++ b/tests/format/include/conditional-conflicts-element/extra_conf.yml @@ -0,0 +1,6 @@ +variables: + (?): + - build_arch == "i586": + size: "4" + - build_arch == "x86_64": + size: "8" diff --git a/tests/format/include/conditional-conflicts-element/project.conf b/tests/format/include/conditional-conflicts-element/project.conf new file mode 100644 index 000000000..e6ac8a701 --- /dev/null +++ b/tests/format/include/conditional-conflicts-element/project.conf @@ -0,0 +1,10 @@ +name: test + +options: + build_arch: + type: arch + description: Architecture + variable: build_arch + values: + - i586 + - x86_64 diff --git a/tests/format/include/conditional-conflicts-element/work_around.yml b/tests/format/include/conditional-conflicts-element/work_around.yml new file mode 100644 index 000000000..a527fe124 --- /dev/null +++ b/tests/format/include/conditional-conflicts-element/work_around.yml @@ -0,0 +1,5 @@ +variables: + enable-work-around: "false" + (?): + - build_arch == "i586": + enable-work-around: "true" diff --git a/tests/format/include/conditional-conflicts-options-included/element.bst b/tests/format/include/conditional-conflicts-options-included/element.bst new file mode 100644 index 000000000..4d7f70266 --- /dev/null +++ b/tests/format/include/conditional-conflicts-options-included/element.bst @@ -0,0 +1 @@ +kind: manual diff --git a/tests/format/include/conditional-conflicts-options-included/extra_conf.yml b/tests/format/include/conditional-conflicts-options-included/extra_conf.yml new file mode 100644 index 000000000..dd58c9855 --- /dev/null +++ b/tests/format/include/conditional-conflicts-options-included/extra_conf.yml @@ -0,0 +1,6 @@ +variables: + (?): + - build_arch == "i586": + size: "4" + - build_arch == "x86_64": + size: "8" diff --git a/tests/format/include/conditional-conflicts-options-included/options.yml b/tests/format/include/conditional-conflicts-options-included/options.yml new file mode 100644 index 000000000..8bb07305a --- /dev/null +++ b/tests/format/include/conditional-conflicts-options-included/options.yml @@ -0,0 +1,9 @@ + +options: + build_arch: + type: arch + description: Architecture + variable: build_arch + values: + - i586 + - x86_64 diff --git a/tests/format/include/conditional-conflicts-options-included/project.conf b/tests/format/include/conditional-conflicts-options-included/project.conf new file mode 100644 index 000000000..ac1b06ad7 --- /dev/null +++ b/tests/format/include/conditional-conflicts-options-included/project.conf @@ -0,0 +1,6 @@ +name: test + +(@): +- options.yml +- extra_conf.yml +- work_around.yml diff --git a/tests/format/include/conditional-conflicts-options-included/work_around.yml b/tests/format/include/conditional-conflicts-options-included/work_around.yml new file mode 100644 index 000000000..a527fe124 --- /dev/null +++ b/tests/format/include/conditional-conflicts-options-included/work_around.yml @@ -0,0 +1,5 @@ +variables: + enable-work-around: "false" + (?): + - build_arch == "i586": + enable-work-around: "true" diff --git a/tests/format/include/conditional-conflicts-project/element.bst b/tests/format/include/conditional-conflicts-project/element.bst new file mode 100644 index 000000000..4d7f70266 --- /dev/null +++ b/tests/format/include/conditional-conflicts-project/element.bst @@ -0,0 +1 @@ +kind: manual diff --git a/tests/format/include/conditional-conflicts-project/extra_conf.yml b/tests/format/include/conditional-conflicts-project/extra_conf.yml new file mode 100644 index 000000000..dd58c9855 --- /dev/null +++ b/tests/format/include/conditional-conflicts-project/extra_conf.yml @@ -0,0 +1,6 @@ +variables: + (?): + - build_arch == "i586": + size: "4" + - build_arch == "x86_64": + size: "8" diff --git a/tests/format/include/conditional-conflicts-project/project.conf b/tests/format/include/conditional-conflicts-project/project.conf new file mode 100644 index 000000000..82722e2c6 --- /dev/null +++ b/tests/format/include/conditional-conflicts-project/project.conf @@ -0,0 +1,14 @@ +name: test + +options: + build_arch: + type: arch + description: Architecture + variable: build_arch + values: + - i586 + - x86_64 + +(@): + - extra_conf.yml + - work_around.yml diff --git a/tests/format/include/conditional-conflicts-project/work_around.yml b/tests/format/include/conditional-conflicts-project/work_around.yml new file mode 100644 index 000000000..a527fe124 --- /dev/null +++ b/tests/format/include/conditional-conflicts-project/work_around.yml @@ -0,0 +1,5 @@ +variables: + enable-work-around: "false" + (?): + - build_arch == "i586": + enable-work-around: "true" diff --git a/tests/format/include/conditional-conflicts-toplevel-precedence/element.bst b/tests/format/include/conditional-conflicts-toplevel-precedence/element.bst new file mode 100644 index 000000000..4d7f70266 --- /dev/null +++ b/tests/format/include/conditional-conflicts-toplevel-precedence/element.bst @@ -0,0 +1 @@ +kind: manual diff --git a/tests/format/include/conditional-conflicts-toplevel-precedence/extra_conf.yml b/tests/format/include/conditional-conflicts-toplevel-precedence/extra_conf.yml new file mode 100644 index 000000000..dd58c9855 --- /dev/null +++ b/tests/format/include/conditional-conflicts-toplevel-precedence/extra_conf.yml @@ -0,0 +1,6 @@ +variables: + (?): + - build_arch == "i586": + size: "4" + - build_arch == "x86_64": + size: "8" diff --git a/tests/format/include/conditional-conflicts-toplevel-precedence/project.conf b/tests/format/include/conditional-conflicts-toplevel-precedence/project.conf new file mode 100644 index 000000000..1a970acde --- /dev/null +++ b/tests/format/include/conditional-conflicts-toplevel-precedence/project.conf @@ -0,0 +1,22 @@ +name: test + +options: + build_arch: + type: arch + description: Architecture + variable: build_arch + values: + - i586 + - x86_64 + +# The work_around.yml sets this to false in it's conditional +# and we set it to true, testing here that the including +# fragment still takes precedence over any included fragments. +variables: + (?): + - build_arch == "i586": + enable-work-around: "true" + +(@): + - extra_conf.yml + - work_around.yml diff --git a/tests/format/include/conditional-conflicts-toplevel-precedence/work_around.yml b/tests/format/include/conditional-conflicts-toplevel-precedence/work_around.yml new file mode 100644 index 000000000..e9991aaba --- /dev/null +++ b/tests/format/include/conditional-conflicts-toplevel-precedence/work_around.yml @@ -0,0 +1,5 @@ +variables: + enable-work-around: "true" + (?): + - build_arch == "i586": + enable-work-around: "false" -- cgit v1.2.1