diff options
| author | Benjamin Schubert <contact@benschubert.me> | 2019-07-02 10:58:15 +0000 |
|---|---|---|
| committer | Benjamin Schubert <contact@benschubert.me> | 2019-07-02 10:58:15 +0000 |
| commit | bcbff34e7138021b0887dc1865f9f16252e18b48 (patch) | |
| tree | 80acc8392dae66861e7e148f1778c2aa9b20d373 | |
| parent | d0e92427acc75b23b8183d807b6fbdf71f3ec26a (diff) | |
| parent | 0c4b6217fb2541667456cb96123363c18304b410 (diff) | |
| download | buildstream-bcbff34e7138021b0887dc1865f9f16252e18b48.tar.gz | |
Merge branch 'bschubert/node-api-nosanitize' into 'bschubert/new-node-api'
Remove the need for 'node_sanitize'
See merge request BuildStream/buildstream!1438
80 files changed, 375 insertions, 440 deletions
diff --git a/doc/bst2html.py b/doc/bst2html.py index 9d178bc18..71d497eda 100755 --- a/doc/bst2html.py +++ b/doc/bst2html.py @@ -197,7 +197,7 @@ def workdir(source_cache=None): 'sourcedir': source_cache, 'logdir': os.path.join(tempdir, 'logs'), } - _yaml.dump(config, bst_config_file) + _yaml.roundtrip_dump(config, bst_config_file) yield (tempdir, bst_config_file, source_cache) diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py index ec80e6417..8f7ae9a1d 100644 --- a/src/buildstream/_artifact.py +++ b/src/buildstream/_artifact.py @@ -149,7 +149,7 @@ class Artifact(): # Store public data with tempfile.NamedTemporaryFile(dir=self._tmpdir) as tmp: - _yaml.dump(publicdata, tmp.name) + _yaml.roundtrip_dump(publicdata, tmp.name) public_data_digest = self._cas.add_object(path=tmp.name, link_directly=True) artifact.public_data.CopyFrom(public_data_digest) size += public_data_digest.size_bytes diff --git a/src/buildstream/_cachekey.py b/src/buildstream/_cachekey.py index e56b582fa..89d47671e 100644 --- a/src/buildstream/_cachekey.py +++ b/src/buildstream/_cachekey.py @@ -22,7 +22,6 @@ import hashlib import ujson -from . import _yaml # Internal record of the size of a cache key _CACHEKEY_SIZE = len(hashlib.sha256().hexdigest()) @@ -63,6 +62,5 @@ def is_key(key): # (str): An sha256 hex digest of the given value # def generate_key(value): - ordered = _yaml.node_sanitize(value) - ustring = ujson.dumps(ordered, sort_keys=True, escape_forward_slashes=False).encode('utf-8') + ustring = ujson.dumps(value, sort_keys=True, escape_forward_slashes=False).encode('utf-8') return hashlib.sha256(ustring).hexdigest() diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py index cc31e982a..35ed95e11 100644 --- a/src/buildstream/_context.py +++ b/src/buildstream/_context.py @@ -441,7 +441,7 @@ class Context(): if self._cache_key is None: # Anything that alters the build goes into the unique key - self._cache_key = _cachekey.generate_key(_yaml.new_empty_node()) + self._cache_key = _cachekey.generate_key({}) return self._cache_key diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py index dce63e14f..e2ab3c5a5 100644 --- a/src/buildstream/_frontend/widget.py +++ b/src/buildstream/_frontend/widget.py @@ -28,7 +28,6 @@ import click from .profile import Profile from .. import Element, Consistency, Scope -from .. import _yaml from .. import __version__ as bst_version from .._exceptions import ImplError from .._message import MessageType @@ -393,28 +392,27 @@ class LogLine(Widget): # Element configuration if "%{config" in format_: - config = _yaml.node_sanitize(element._Element__config) line = p.fmt_subst( line, 'config', - yaml.round_trip_dump(config, default_flow_style=False, allow_unicode=True)) + yaml.round_trip_dump(element._Element__config, default_flow_style=False, allow_unicode=True)) # Variables if "%{vars" in format_: - variables = _yaml.node_sanitize(element._Element__variables.flat) + variables = element._Element__variables.flat line = p.fmt_subst( line, 'vars', yaml.round_trip_dump(variables, default_flow_style=False, allow_unicode=True)) # Environment if "%{env" in format_: - environment = _yaml.node_sanitize(element._Element__environment) + environment = element._Element__environment line = p.fmt_subst( line, 'env', yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True)) # Public if "%{public" in format_: - environment = _yaml.node_sanitize(element._Element__public) + environment = element._Element__public line = p.fmt_subst( line, 'public', yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True)) diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index af15bdc45..f3efd8ba9 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -230,7 +230,7 @@ class Project(): # Anything that alters the build goes into the unique key # (currently nothing here) - self._cache_key = _cachekey.generate_key(_yaml.new_empty_node()) + self._cache_key = _cachekey.generate_key({}) return self._cache_key diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 195be55ba..8a2e1d1eb 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -913,7 +913,7 @@ class Stream(): } workspaces.append(workspace_detail) - _yaml.dump({ + _yaml.roundtrip_dump({ 'workspaces': workspaces }) diff --git a/src/buildstream/_workspaces.py b/src/buildstream/_workspaces.py index 95c1817ce..d3a28b879 100644 --- a/src/buildstream/_workspaces.py +++ b/src/buildstream/_workspaces.py @@ -114,7 +114,8 @@ class WorkspaceProject(): def load(cls, directory): workspace_file = os.path.join(directory, WORKSPACE_PROJECT_FILE) if os.path.exists(workspace_file): - data_dict = _yaml.node_sanitize(_yaml.roundtrip_load(workspace_file), dict_type=dict) + data_dict = _yaml.roundtrip_load(workspace_file) + return cls.from_dict(directory, data_dict) else: return None @@ -125,7 +126,7 @@ class WorkspaceProject(): # def write(self): os.makedirs(self._directory, exist_ok=True) - _yaml.dump(self.to_dict(), self.get_filename()) + _yaml.roundtrip_dump(self.to_dict(), self.get_filename()) # get_filename() # @@ -530,7 +531,7 @@ class Workspaces(): } } os.makedirs(self._bst_directory, exist_ok=True) - _yaml.dump(config, self._get_filename()) + _yaml.roundtrip_dump(config, self._get_filename()) # _load_config() # @@ -627,13 +628,15 @@ class Workspaces(): # (Workspace): A newly instantiated Workspace # def _load_workspace(self, node): + running_files = node.get_mapping('running_files', default=None) + if running_files: + running_files = running_files.strip_node_info() + dictionary = { 'prepared': node.get_bool('prepared', default=False), 'path': node.get_str('path'), 'last_successful': node.get_str('last_successful', default=None), - 'running_files': _yaml.node_sanitize( - node.get_mapping('running_files', default=None), - dict_type=dict), + 'running_files': running_files, } return Workspace.from_dict(self._toplevel_project, dictionary) diff --git a/src/buildstream/_yaml.pxd b/src/buildstream/_yaml.pxd index 48bed1dea..45d2d351a 100644 --- a/src/buildstream/_yaml.pxd +++ b/src/buildstream/_yaml.pxd @@ -28,6 +28,7 @@ cdef class Node: cdef public int column cpdef Node copy(self) + cpdef object strip_node_info(self) cdef bint _shares_position_with(self, Node target) cdef bint _walk_find(self, Node target, list path) except * diff --git a/src/buildstream/_yaml.pyx b/src/buildstream/_yaml.pyx index 46f642578..99e3160b2 100644 --- a/src/buildstream/_yaml.pyx +++ b/src/buildstream/_yaml.pyx @@ -21,14 +21,15 @@ # James Ennis <james.ennis@codethink.co.uk> # Benjamin Schubert <bschubert@bloomberg.net> +import datetime import sys import string from contextlib import ExitStack from collections import OrderedDict -from collections.abc import Mapping, Sequence -from copy import deepcopy +from collections.abc import Mapping from ruamel import yaml + from ._exceptions import LoadError, LoadErrorReason @@ -83,6 +84,12 @@ cdef class Node: cpdef Node copy(self): raise NotImplementedError() + cpdef object strip_node_info(self): + raise NotImplementedError() + + def __json__(self): + raise ValueError("Nodes should not be allowed when jsonify-ing data", self) + cdef class ScalarNode(Node): @@ -132,6 +139,9 @@ cdef class ScalarNode(Node): return None return str(self.value) + cpdef object strip_node_info(self): + return self.value + cdef bint _walk_find(self, Node target, list path) except *: return self._shares_position_with(target) @@ -273,6 +283,12 @@ cdef class MappingNode(Node): cpdef object values(self): return self.value.values() + cpdef object strip_node_info(self): + cdef str key + cdef Node value + + return {key: value.strip_node_info() for key, value in self.value.items()} + def __delitem__(self, str key): del self.value[key] @@ -365,6 +381,10 @@ cdef class SequenceNode(Node): cpdef list as_str_list(self): return [node.as_str() for node in self.value] + cpdef object strip_node_info(self): + cdef Node value + return [value.strip_node_info() for value in self.value] + cdef bint _walk_find(self, Node target, list path) except *: cdef int i cdef Node v @@ -816,21 +836,6 @@ cpdef Node load_data(str data, int file_index=_SYNTHETIC_FILE_INDEX, str file_na return contents -# dump() -# -# Write a YAML node structure out to disk. -# -# This will always call `node_sanitize` on its input, so if you wanted -# to output something close to what you read in, consider using the -# `roundtrip_load` and `roundtrip_dump` function pair instead. -# -# Args: -# contents (any): Content to write out -# filename (str): The (optional) file name to write out to -def dump(object contents, str filename=None): - roundtrip_dump(node_sanitize(contents), file=filename) - - # node_get_provenance() # # Gets the provenance for a node @@ -1210,60 +1215,6 @@ def composite_and_move(MappingNode target, MappingNode source): del target.value[key] -# Types we can short-circuit in node_sanitize for speed. -__SANITIZE_SHORT_CIRCUIT_TYPES = (int, float, str, bool) - - -# node_sanitize() -# -# Returns an alphabetically ordered recursive copy -# of the source node with internal provenance information stripped. -# -# Only dicts are ordered, list elements are left in order. -# -cpdef object node_sanitize(object node, object dict_type=OrderedDict): - node_type = type(node) - - # If we have an unwrappable node, unwrap it - # FIXME: we should only ever have Nodes here - if node_type in [MappingNode, SequenceNode]: - node = node.value - node_type = type(node) - - if node_type is ScalarNode: - return node.value - - # Short-circuit None which occurs ca. twice per element - if node is None: - return node - - # Next short-circuit integers, floats, strings, booleans, and tuples - if node_type in __SANITIZE_SHORT_CIRCUIT_TYPES: - return node - - # Now short-circuit lists. - elif node_type is list: - return [node_sanitize(elt, dict_type=dict_type) for elt in node] - - # Finally dict, and other Mappings need special handling - elif node_type is dict: - result = dict_type() - - key_list = [key for key, _ in node.items()] - for key in sorted(key_list): - result[key] = node_sanitize(node[key], dict_type=dict_type) - - return result - - # Sometimes we're handed tuples and we can't be sure what they contain - # so we have to sanitize into them - elif node_type is tuple: - return tuple([node_sanitize(v, dict_type=dict_type) for v in node]) - - # Everything else just gets returned as-is. - return node - - # node_validate() # # Validate the node so as to ensure the user has not specified @@ -1390,6 +1341,34 @@ def assert_symbol_name(ProvenanceInformation provenance, str symbol_name, str pu # Roundtrip code +# Represent Nodes automatically + +def represent_mapping(self, MappingNode mapping): + return self.represent_dict(mapping.value) + +def represent_scalar(self, ScalarNode scalar): + return self.represent_str(scalar.value) + +def represent_sequence(self, SequenceNode sequence): + return self.represent_list(sequence.value) + + +yaml.RoundTripRepresenter.add_representer(MappingNode, represent_mapping) +yaml.RoundTripRepresenter.add_representer(ScalarNode, represent_scalar) +yaml.RoundTripRepresenter.add_representer(SequenceNode, represent_sequence) + +# Represent simple types as strings + +def represent_as_str(self, value): + return self.represent_str(str(value)) + +yaml.RoundTripRepresenter.add_representer(type(None), represent_as_str) +yaml.RoundTripRepresenter.add_representer(int, represent_as_str) +yaml.RoundTripRepresenter.add_representer(float, represent_as_str) +yaml.RoundTripRepresenter.add_representer(bool, represent_as_str) +yaml.RoundTripRepresenter.add_representer(datetime.datetime, represent_as_str) +yaml.RoundTripRepresenter.add_representer(datetime.date, represent_as_str) + # Always represent things consistently: yaml.RoundTripRepresenter.add_representer(OrderedDict, @@ -1520,33 +1499,6 @@ def roundtrip_load_data(contents, *, filename=None): # file (any): The file to write to # def roundtrip_dump(contents, file=None): - assert type(contents) is not Node - - def stringify_dict(thing): - for k, v in thing.items(): - if type(v) is str: - pass - elif isinstance(v, Mapping): - stringify_dict(v) - elif isinstance(v, Sequence): - stringify_list(v) - else: - thing[k] = str(v) - - def stringify_list(thing): - for i, v in enumerate(thing): - if type(v) is str: - pass - elif isinstance(v, Mapping): - stringify_dict(v) - elif isinstance(v, Sequence): - stringify_list(v) - else: - thing[i] = str(v) - - contents = deepcopy(contents) - stringify_dict(contents) - with ExitStack() as stack: if type(file) is str: from . import utils diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 1a5439fda..c2e4b9105 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -883,7 +883,7 @@ class Element(Plugin): (dict): A dictionary of string key/values suitable for passing to :func:`Sandbox.run() <buildstream.sandbox.Sandbox.run>` """ - return _yaml.node_sanitize(self.__environment) + return self.__environment def get_variable(self, varname): """Fetch the value of a variable resolved for this element. @@ -2166,7 +2166,7 @@ class Element(Plugin): 'environment': cache_env, 'sources': [s._get_unique_key(workspace is None) for s in self.__sources], 'workspace': '' if workspace is None else workspace.get_key(self._get_project()), - 'public': self.__public, + 'public': self.__public.strip_node_info(), 'cache': 'CASCache' } diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py index 38dfe3b09..a544331de 100644 --- a/src/buildstream/sandbox/_sandboxremote.py +++ b/src/buildstream/sandbox/_sandboxremote.py @@ -175,7 +175,8 @@ class SandboxRemote(Sandbox): if tls_key in config: config[tls_key] = resolve_path(config.get_str(tls_key)) - return RemoteExecutionSpec(*[_yaml.node_sanitize(conf) for conf in service_configs]) + # TODO: we should probably not be stripping node info and rather load files the safe way + return RemoteExecutionSpec(*[conf.strip_node_info() for conf in service_configs]) def run_remote_command(self, channel, action_digest): # Sends an execution request to the remote execution server. diff --git a/src/buildstream/source.py b/src/buildstream/source.py index c87d77020..c09e318d2 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -893,8 +893,9 @@ class Source(Plugin): # # Step 2 - Set the ref in memory, and determine changed state # - clean = _yaml.node_sanitize(node, dict_type=dict) - to_modify = _yaml.node_sanitize(node, dict_type=dict) + # TODO: we are working on dictionaries here, would be nicer to just work on the nodes themselves + clean = node.strip_node_info() + to_modify = node.strip_node_info() current_ref = self.get_ref() # pylint: disable=assignment-from-no-return diff --git a/src/buildstream/testing/_sourcetests/build_checkout.py b/src/buildstream/testing/_sourcetests/build_checkout.py index 3619d2b7e..e2842e0e0 100644 --- a/src/buildstream/testing/_sourcetests/build_checkout.py +++ b/src/buildstream/testing/_sourcetests/build_checkout.py @@ -63,9 +63,8 @@ def test_fetch_build_checkout(cli, tmpdir, datafiles, strict, kind): repo.source_config(ref=ref) ] } - _yaml.dump(element, - os.path.join(element_path, - element_name)) + _yaml.roundtrip_dump(element, + os.path.join(element_path, element_name)) assert cli.get_element_state(project, element_name) == 'fetch needed' result = cli.run(project=project, args=strict_args(['build', element_name], strict)) diff --git a/src/buildstream/testing/_sourcetests/fetch.py b/src/buildstream/testing/_sourcetests/fetch.py index aaf92a14d..d9b0876c6 100644 --- a/src/buildstream/testing/_sourcetests/fetch.py +++ b/src/buildstream/testing/_sourcetests/fetch.py @@ -53,9 +53,8 @@ def test_fetch(cli, tmpdir, datafiles, kind): repo.source_config(ref=ref) ] } - _yaml.dump(element, - os.path.join(element_path, - element_name)) + _yaml.roundtrip_dump(element, + os.path.join(element_path, element_name)) # Assert that a fetch is needed assert cli.get_element_state(project, element_name) == 'fetch needed' @@ -89,7 +88,7 @@ def test_fetch_cross_junction(cli, tmpdir, datafiles, ref_storage, kind): repo.source_config(ref=(ref if ref_storage == 'inline' else None)) ] } - _yaml.dump(element, import_etc_path) + _yaml.roundtrip_dump(element, import_etc_path) configure_project(project, { 'ref-storage': ref_storage diff --git a/src/buildstream/testing/_sourcetests/mirror.py b/src/buildstream/testing/_sourcetests/mirror.py index afbadbef7..f532049dd 100644 --- a/src/buildstream/testing/_sourcetests/mirror.py +++ b/src/buildstream/testing/_sourcetests/mirror.py @@ -66,7 +66,7 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind): full_mirror = mirror_repo.source_config()['url'] mirror_map, _ = os.path.split(full_mirror) os.makedirs(element_dir) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) project = { 'name': 'test', @@ -84,7 +84,7 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind): ] } project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) # No obvious ways of checking that the mirror has been fetched # But at least we can be sure it succeeds @@ -124,7 +124,7 @@ def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind): full_mirror = mirror_repo.source_config()['url'] mirror_map, _ = os.path.split(full_mirror) os.makedirs(element_dir) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) project = { 'name': 'test', @@ -142,7 +142,7 @@ def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind): ] } project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) result.assert_success() @@ -179,14 +179,14 @@ def test_mirror_from_includes(cli, tmpdir, datafiles, kind): full_mirror = mirror_repo.source_config()['url'] mirror_map, _ = os.path.split(full_mirror) os.makedirs(element_dir) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) config_project_dir = str(tmpdir.join('config')) os.makedirs(config_project_dir, exist_ok=True) config_project = { 'name': 'config' } - _yaml.dump(config_project, os.path.join(config_project_dir, 'project.conf')) + _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, 'project.conf')) extra_mirrors = { 'mirrors': [ { @@ -197,7 +197,7 @@ def test_mirror_from_includes(cli, tmpdir, datafiles, kind): } ] } - _yaml.dump(extra_mirrors, os.path.join(config_project_dir, 'mirrors.yml')) + _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, 'mirrors.yml')) generate_junction(str(tmpdir.join('config_repo')), config_project_dir, os.path.join(element_dir, 'config.bst')) @@ -213,7 +213,7 @@ def test_mirror_from_includes(cli, tmpdir, datafiles, kind): ] } project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) # Now make the upstream unavailable. os.rename(upstream_repo.repo, '{}.bak'.format(upstream_repo.repo)) @@ -252,14 +252,14 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind): full_mirror = mirror_repo.source_config()['url'] mirror_map, _ = os.path.split(full_mirror) os.makedirs(element_dir) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) config_project_dir = str(tmpdir.join('config')) os.makedirs(config_project_dir, exist_ok=True) config_project = { 'name': 'config' } - _yaml.dump(config_project, os.path.join(config_project_dir, 'project.conf')) + _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, 'project.conf')) extra_mirrors = { 'mirrors': [ { @@ -270,7 +270,7 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind): } ] } - _yaml.dump(extra_mirrors, os.path.join(config_project_dir, 'mirrors.yml')) + _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, 'mirrors.yml')) generate_junction(str(tmpdir.join('config_repo')), config_project_dir, os.path.join(element_dir, 'config.bst')) @@ -286,7 +286,7 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind): ] } project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) # Now make the upstream unavailable. os.rename(upstream_repo.repo, '{}.bak'.format(upstream_repo.repo)) @@ -332,7 +332,7 @@ def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind): full_mirror = mirror_repo.source_config()['url'] mirror_map, _ = os.path.split(full_mirror) os.makedirs(element_dir) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) project = { 'name': 'test', @@ -350,7 +350,7 @@ def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind): ] } project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) result = cli.run(project=project_dir, args=['source', 'track', element_name]) result.assert_success() @@ -397,7 +397,7 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind): full_mirror = mirror_repo.source_config()['url'] mirror_map, _ = os.path.split(full_mirror) os.makedirs(element_dir) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) project = { 'name': 'test', @@ -415,7 +415,7 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind): ] } project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) result = cli.run(project=project_dir, args=['source', 'track', element_name]) result.assert_success() diff --git a/src/buildstream/testing/_sourcetests/source_determinism.py b/src/buildstream/testing/_sourcetests/source_determinism.py index 3a5c264d9..fc8ad9893 100644 --- a/src/buildstream/testing/_sourcetests/source_determinism.py +++ b/src/buildstream/testing/_sourcetests/source_determinism.py @@ -91,7 +91,7 @@ def test_deterministic_source_umask(cli, tmpdir, datafiles, kind): ] } } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) def get_value_for_umask(umask): checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(umask)) diff --git a/src/buildstream/testing/_sourcetests/track.py b/src/buildstream/testing/_sourcetests/track.py index 58679e9f1..01a39951f 100644 --- a/src/buildstream/testing/_sourcetests/track.py +++ b/src/buildstream/testing/_sourcetests/track.py @@ -43,7 +43,7 @@ def generate_element(repo, element_path, dep_name=None): if dep_name: element['depends'] = [dep_name] - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) @pytest.mark.datafiles(DATA_DIR) @@ -290,8 +290,8 @@ def test_track_include(cli, tmpdir, datafiles, ref_storage, kind): ] } - _yaml.dump(element, os.path.join(element_path, element_name)) - _yaml.dump(sources, os.path.join(element_path, 'sources.yml')) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(sources, os.path.join(element_path, 'sources.yml')) # Assert that a fetch is needed assert cli.get_element_state(project, element_name) == 'no reference' @@ -363,8 +363,8 @@ def test_track_include_junction(cli, tmpdir, datafiles, ref_storage, kind): ] } - _yaml.dump(element, os.path.join(element_path, element_name)) - _yaml.dump(sources, os.path.join(sub_element_path, 'sources.yml')) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(sources, os.path.join(sub_element_path, 'sources.yml')) generate_junction(str(tmpdir.join('junction_repo')), subproject_path, junction_path, store_ref=True) diff --git a/src/buildstream/testing/_sourcetests/track_cross_junction.py b/src/buildstream/testing/_sourcetests/track_cross_junction.py index ece3e0b8f..31443bdf9 100644 --- a/src/buildstream/testing/_sourcetests/track_cross_junction.py +++ b/src/buildstream/testing/_sourcetests/track_cross_junction.py @@ -42,7 +42,7 @@ def generate_element(repo, element_path, dep_name=None): if dep_name: element['depends'] = [dep_name] - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) def generate_import_element(tmpdir, kind, project, name): @@ -75,7 +75,7 @@ def generate_project(tmpdir, name, config=None): 'element-path': 'elements' } project_conf.update(config) - _yaml.dump(project_conf, os.path.join(subproject_path, 'project.conf')) + _yaml.roundtrip_dump(project_conf, os.path.join(subproject_path, 'project.conf')) return project_name, subproject_path @@ -87,7 +87,7 @@ def generate_simple_stack(project, name, dependencies): 'kind': 'stack', 'depends': dependencies } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) return element_name diff --git a/src/buildstream/testing/_sourcetests/workspace.py b/src/buildstream/testing/_sourcetests/workspace.py index 5218f8f1e..5ceab5108 100644 --- a/src/buildstream/testing/_sourcetests/workspace.py +++ b/src/buildstream/testing/_sourcetests/workspace.py @@ -73,9 +73,8 @@ class WorkspaceCreator(): } if element_attrs: element = {**element, **element_attrs} - _yaml.dump(element, - os.path.join(element_path, - element_name)) + _yaml.roundtrip_dump(element, + os.path.join(element_path, element_name)) return element_name, element_path, workspace_dir def create_workspace_elements(self, kinds, track, suffixs=None, workspace_dir_usr=None, diff --git a/src/buildstream/testing/_utils/__init__.py b/src/buildstream/testing/_utils/__init__.py index b419d72b7..575226e22 100644 --- a/src/buildstream/testing/_utils/__init__.py +++ b/src/buildstream/testing/_utils/__init__.py @@ -7,4 +7,4 @@ from .junction import generate_junction def configure_project(path, config): config['name'] = 'test' config['element-path'] = 'elements' - _yaml.dump(config, os.path.join(path, 'project.conf')) + _yaml.roundtrip_dump(config, os.path.join(path, 'project.conf')) diff --git a/src/buildstream/testing/_utils/junction.py b/src/buildstream/testing/_utils/junction.py index 2bf53ac7c..98d23b0a2 100644 --- a/src/buildstream/testing/_utils/junction.py +++ b/src/buildstream/testing/_utils/junction.py @@ -34,7 +34,7 @@ def generate_junction(tmpdir, subproject_path, junction_path, *, store_ref=True) repo.source_config(ref=source_ref) ] } - _yaml.dump(element, junction_path) + _yaml.roundtrip_dump(element, junction_path) return ref diff --git a/src/buildstream/testing/runcli.py b/src/buildstream/testing/runcli.py index a77deeb9e..16593fa8e 100644 --- a/src/buildstream/testing/runcli.py +++ b/src/buildstream/testing/runcli.py @@ -576,7 +576,7 @@ class CliIntegration(Cli): _yaml.composite_dict(base_config, project_config) - _yaml.dump(base_config, project_filename) + _yaml.roundtrip_dump(base_config, project_filename) else: @@ -887,6 +887,6 @@ def configured(directory, config=None): # Dump it and yield the filename for test scripts to feed it # to buildstream as an artument filename = os.path.join(directory, "buildstream.conf") - _yaml.dump(config, filename) + _yaml.roundtrip_dump(config, filename) yield filename diff --git a/tests/artifactcache/cache_size.py b/tests/artifactcache/cache_size.py index 3a2956c5c..fb34b5fad 100644 --- a/tests/artifactcache/cache_size.py +++ b/tests/artifactcache/cache_size.py @@ -22,7 +22,7 @@ def create_project(project_dir): project_conf = { "name": "test" } - _yaml.dump(project_conf, project_file) + _yaml.roundtrip_dump(project_conf, project_file) element_name = "test.bst" create_element_size(element_name, project_dir, ".", [], 1024) @@ -77,7 +77,7 @@ def test_quota_over_1024T(cli, tmpdir): }) project = tmpdir.join("main") os.makedirs(str(project)) - _yaml.dump({'name': 'main'}, str(project.join("project.conf"))) + _yaml.roundtrip_dump({'name': 'main'}, str(project.join("project.conf"))) volume_space_patch = mock.patch( "buildstream._cas.CASQuota._get_cache_volume_size", diff --git a/tests/artifactcache/config.py b/tests/artifactcache/config.py index 928a6c524..db51d196c 100644 --- a/tests/artifactcache/config.py +++ b/tests/artifactcache/config.py @@ -101,11 +101,11 @@ def test_artifact_cache_precedence(tmpdir, override_caches, project_caches, user project_config['name'] = 'test' user_config_file = str(tmpdir.join('buildstream.conf')) - _yaml.dump(user_config, filename=user_config_file) + _yaml.roundtrip_dump(user_config, file=user_config_file) project_dir = tmpdir.mkdir('project') project_config_file = str(project_dir.join('project.conf')) - _yaml.dump(project_config, filename=project_config_file) + _yaml.roundtrip_dump(project_config, file=project_config_file) context = Context() context.load(config=user_config_file) @@ -141,7 +141,7 @@ def test_missing_certs(cli, datafiles, config_key, config_value): } } project_conf_file = os.path.join(project, 'project.conf') - _yaml.dump(project_conf, project_conf_file) + _yaml.roundtrip_dump(project_conf, project_conf_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # diff --git a/tests/artifactcache/junctions.py b/tests/artifactcache/junctions.py index f099ee566..6170816c3 100644 --- a/tests/artifactcache/junctions.py +++ b/tests/artifactcache/junctions.py @@ -24,7 +24,7 @@ def project_set_artifacts(project, url): 'url': url, 'push': True } - _yaml.dump(project_config, filename=project_conf_file) + _yaml.roundtrip_dump(project_config.strip_node_info(), file=project_conf_file) @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py index 0b57a6c9a..d5da39a99 100644 --- a/tests/artifactcache/pull.py +++ b/tests/artifactcache/pull.py @@ -73,7 +73,7 @@ def test_pull(cli, tmpdir, datafiles): } # Write down the user configuration file - _yaml.dump(user_config, filename=user_config_file) + _yaml.roundtrip_dump(user_config, file=user_config_file) # Ensure CLI calls will use it cli.configure(user_config) @@ -182,7 +182,7 @@ def test_pull_tree(cli, tmpdir, datafiles): } # Write down the user configuration file - _yaml.dump(user_config, filename=user_config_file) + _yaml.roundtrip_dump(user_config, file=user_config_file) # Ensure CLI calls will use it cli.configure(user_config) diff --git a/tests/artifactcache/push.py b/tests/artifactcache/push.py index 862393f35..fd9758229 100644 --- a/tests/artifactcache/push.py +++ b/tests/artifactcache/push.py @@ -66,7 +66,7 @@ def test_push(cli, tmpdir, datafiles): } # Write down the user configuration file - _yaml.dump(user_config, filename=user_config_file) + _yaml.roundtrip_dump(user_config, file=user_config_file) # Fake minimal context context = Context() @@ -164,7 +164,7 @@ def test_push_message(tmpdir, datafiles): } # Write down the user configuration file - _yaml.dump(user_config, filename=user_config_file) + _yaml.roundtrip_dump(user_config, file=user_config_file) queue = multiprocessing.Queue() # Use subprocess to avoid creation of gRPC threads in main BuildStream process diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py index bbececb8c..acfe97947 100644 --- a/tests/cachekey/cachekey.py +++ b/tests/cachekey/cachekey.py @@ -198,11 +198,11 @@ def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings, project_dir = tmpdir.mkdir(project_name) project_config_file = str(project_dir.join('project.conf')) - _yaml.dump(config, filename=project_config_file) + _yaml.roundtrip_dump(config, file=project_config_file) elem_dir = project_dir.mkdir('elements') element_file = str(elem_dir.join('stack.bst')) - _yaml.dump({'kind': 'stack'}, filename=element_file) + _yaml.roundtrip_dump({'kind': 'stack'}, file=element_file) result = cli.run(project=str(project_dir), args=[ 'show', diff --git a/tests/elements/filter.py b/tests/elements/filter.py index f9c80afff..db20529bc 100644 --- a/tests/elements/filter.py +++ b/tests/elements/filter.py @@ -202,7 +202,7 @@ def test_filter_track(datafiles, cli, tmpdir): "element-path": "elements", } project_file = os.path.join(str(tmpdir), "project.conf") - _yaml.dump(project_config, project_file) + _yaml.roundtrip_dump(project_config, project_file) input_config = { "kind": "import", @@ -210,7 +210,7 @@ def test_filter_track(datafiles, cli, tmpdir): } input_file = os.path.join(elements_dir, input_name) - _yaml.dump(input_config, input_file) + _yaml.roundtrip_dump(input_config, input_file) filter1_config = { "kind": "filter", @@ -219,7 +219,7 @@ def test_filter_track(datafiles, cli, tmpdir): ] } filter1_file = os.path.join(elements_dir, "filter1.bst") - _yaml.dump(filter1_config, filter1_file) + _yaml.roundtrip_dump(filter1_config, filter1_file) filter2_config = { "kind": "filter", @@ -228,7 +228,7 @@ def test_filter_track(datafiles, cli, tmpdir): ] } filter2_file = os.path.join(elements_dir, "filter2.bst") - _yaml.dump(filter2_config, filter2_file) + _yaml.roundtrip_dump(filter2_config, filter2_file) # Assert that a fetch is needed assert cli.get_element_state(project, input_name) == 'no reference' @@ -257,7 +257,7 @@ def test_filter_track_excepted(datafiles, cli, tmpdir): "element-path": "elements", } project_file = os.path.join(str(tmpdir), "project.conf") - _yaml.dump(project_config, project_file) + _yaml.roundtrip_dump(project_config, project_file) input_config = { "kind": "import", @@ -265,7 +265,7 @@ def test_filter_track_excepted(datafiles, cli, tmpdir): } input_file = os.path.join(elements_dir, input_name) - _yaml.dump(input_config, input_file) + _yaml.roundtrip_dump(input_config, input_file) filter1_config = { "kind": "filter", @@ -274,7 +274,7 @@ def test_filter_track_excepted(datafiles, cli, tmpdir): ] } filter1_file = os.path.join(elements_dir, "filter1.bst") - _yaml.dump(filter1_config, filter1_file) + _yaml.roundtrip_dump(filter1_config, filter1_file) filter2_config = { "kind": "filter", @@ -283,7 +283,7 @@ def test_filter_track_excepted(datafiles, cli, tmpdir): ] } filter2_file = os.path.join(elements_dir, "filter2.bst") - _yaml.dump(filter2_config, filter2_file) + _yaml.roundtrip_dump(filter2_config, filter2_file) # Assert that a fetch is needed assert cli.get_element_state(project, input_name) == 'no reference' @@ -311,7 +311,7 @@ def test_filter_track_multi_to_one(datafiles, cli, tmpdir): "element-path": "elements", } project_file = os.path.join(str(tmpdir), "project.conf") - _yaml.dump(project_config, project_file) + _yaml.roundtrip_dump(project_config, project_file) input_config = { "kind": "import", @@ -319,7 +319,7 @@ def test_filter_track_multi_to_one(datafiles, cli, tmpdir): } input_file = os.path.join(elements_dir, input_name) - _yaml.dump(input_config, input_file) + _yaml.roundtrip_dump(input_config, input_file) filter1_config = { "kind": "filter", @@ -328,7 +328,7 @@ def test_filter_track_multi_to_one(datafiles, cli, tmpdir): ] } filter1_file = os.path.join(elements_dir, "filter1.bst") - _yaml.dump(filter1_config, filter1_file) + _yaml.roundtrip_dump(filter1_config, filter1_file) filter2_config = { "kind": "filter", @@ -337,7 +337,7 @@ def test_filter_track_multi_to_one(datafiles, cli, tmpdir): ] } filter2_file = os.path.join(elements_dir, "filter2.bst") - _yaml.dump(filter2_config, filter2_file) + _yaml.roundtrip_dump(filter2_config, filter2_file) # Assert that a fetch is needed assert cli.get_element_state(project, input_name) == 'no reference' @@ -367,7 +367,7 @@ def test_filter_track_multi(datafiles, cli, tmpdir): "element-path": "elements", } project_file = os.path.join(str(tmpdir), "project.conf") - _yaml.dump(project_config, project_file) + _yaml.roundtrip_dump(project_config, project_file) input_config = { "kind": "import", @@ -375,11 +375,11 @@ def test_filter_track_multi(datafiles, cli, tmpdir): } input_file = os.path.join(elements_dir, input_name) - _yaml.dump(input_config, input_file) + _yaml.roundtrip_dump(input_config, input_file) input2_config = dict(input_config) input2_file = os.path.join(elements_dir, input2_name) - _yaml.dump(input2_config, input2_file) + _yaml.roundtrip_dump(input2_config, input2_file) filter1_config = { "kind": "filter", @@ -388,7 +388,7 @@ def test_filter_track_multi(datafiles, cli, tmpdir): ] } filter1_file = os.path.join(elements_dir, "filter1.bst") - _yaml.dump(filter1_config, filter1_file) + _yaml.roundtrip_dump(filter1_config, filter1_file) filter2_config = { "kind": "filter", @@ -397,7 +397,7 @@ def test_filter_track_multi(datafiles, cli, tmpdir): ] } filter2_file = os.path.join(elements_dir, "filter2.bst") - _yaml.dump(filter2_config, filter2_file) + _yaml.roundtrip_dump(filter2_config, filter2_file) # Assert that a fetch is needed states = cli.get_element_states(project, [input_name, input2_name]) @@ -437,7 +437,7 @@ def test_filter_track_multi_exclude(datafiles, cli, tmpdir): "element-path": "elements", } project_file = os.path.join(str(tmpdir), "project.conf") - _yaml.dump(project_config, project_file) + _yaml.roundtrip_dump(project_config, project_file) input_config = { "kind": "import", @@ -445,11 +445,11 @@ def test_filter_track_multi_exclude(datafiles, cli, tmpdir): } input_file = os.path.join(elements_dir, input_name) - _yaml.dump(input_config, input_file) + _yaml.roundtrip_dump(input_config, input_file) input2_config = dict(input_config) input2_file = os.path.join(elements_dir, input2_name) - _yaml.dump(input2_config, input2_file) + _yaml.roundtrip_dump(input2_config, input2_file) filter1_config = { "kind": "filter", @@ -458,7 +458,7 @@ def test_filter_track_multi_exclude(datafiles, cli, tmpdir): ] } filter1_file = os.path.join(elements_dir, "filter1.bst") - _yaml.dump(filter1_config, filter1_file) + _yaml.roundtrip_dump(filter1_config, filter1_file) filter2_config = { "kind": "filter", @@ -467,7 +467,7 @@ def test_filter_track_multi_exclude(datafiles, cli, tmpdir): ] } filter2_file = os.path.join(elements_dir, "filter2.bst") - _yaml.dump(filter2_config, filter2_file) + _yaml.roundtrip_dump(filter2_config, filter2_file) # Assert that a fetch is needed states = cli.get_element_states(project, [input_name, input2_name]) diff --git a/tests/elements/filter/basic/element_plugins/dynamic.py b/tests/elements/filter/basic/element_plugins/dynamic.py index eb462ceb1..1ec7d4dd3 100644 --- a/tests/elements/filter/basic/element_plugins/dynamic.py +++ b/tests/elements/filter/basic/element_plugins/dynamic.py @@ -11,7 +11,7 @@ class DynamicElement(Element): pass def get_unique_key(self): - return {'split-rules': self.split_rules} + return {'split-rules': self.split_rules.strip_node_info()} def configure_sandbox(self, sandbox): pass diff --git a/tests/format/include.py b/tests/format/include.py index e2e16c34a..8902aa3eb 100644 --- a/tests/format/include.py +++ b/tests/format/include.py @@ -35,7 +35,7 @@ def test_include_missing_file(cli, tmpdir): tmpdir.join('project.conf').write('{"name": "test"}') element = tmpdir.join('include_missing_file.bst') - # Normally we would use dicts and _yaml.dump to write such things, but here + # Normally we would use dicts and _yaml.roundtrip_dump to write such things, but here # we want to be sure of a stable line and column number. element.write(textwrap.dedent(""" kind: manual @@ -55,7 +55,7 @@ def test_include_dir(cli, tmpdir): tmpdir.mkdir('subdir') element = tmpdir.join('include_dir.bst') - # Normally we would use dicts and _yaml.dump to write such things, but here + # Normally we would use dicts and _yaml.roundtrip_dump to write such things, but here # we want to be sure of a stable line and column number. element.write(textwrap.dedent(""" kind: manual @@ -126,7 +126,7 @@ def test_junction_element_partial_project_project(cli, tmpdir, datafiles): repo.source_config(ref=ref) ] } - _yaml.dump(element, junction_path) + _yaml.roundtrip_dump(element, junction_path) result = cli.run(project=project, args=[ 'show', @@ -159,7 +159,7 @@ def test_junction_element_not_partial_project_file(cli, tmpdir, datafiles): repo.source_config(ref=ref) ] } - _yaml.dump(element, junction_path) + _yaml.roundtrip_dump(element, junction_path) result = cli.run(project=project, args=[ 'show', diff --git a/tests/format/include_composition.py b/tests/format/include_composition.py index b922c7a0e..f764b16a6 100644 --- a/tests/format/include_composition.py +++ b/tests/format/include_composition.py @@ -6,8 +6,7 @@ from buildstream import _yaml def make_includes(basedir): - _yaml.dump({'name': 'test'}, - os.path.join(basedir, 'project.conf')) + _yaml.roundtrip_dump({'name': 'test'}, os.path.join(basedir, 'project.conf')) context = Context() project = Project(basedir, context) loader = project.loader @@ -17,14 +16,12 @@ def make_includes(basedir): def test_main_has_priority(tmpdir): includes = make_includes(str(tmpdir)) - _yaml.dump({'(@)': ['a.yml'], - 'test': ['main']}, - str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({'(@)': ['a.yml'], 'test': ['main']}, + str(tmpdir.join('main.yml'))) main = _yaml.load(str(tmpdir.join('main.yml'))) - _yaml.dump({'test': ['a']}, - str(tmpdir.join('a.yml'))) + _yaml.roundtrip_dump({'test': ['a']}, str(tmpdir.join('a.yml'))) includes.process(main) @@ -34,13 +31,12 @@ def test_main_has_priority(tmpdir): def test_include_cannot_append(tmpdir): includes = make_includes(str(tmpdir)) - _yaml.dump({'(@)': ['a.yml'], - 'test': ['main']}, - str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({'(@)': ['a.yml'], 'test': ['main']}, + str(tmpdir.join('main.yml'))) main = _yaml.load(str(tmpdir.join('main.yml'))) - _yaml.dump({'test': {'(>)': ['a']}}, - str(tmpdir.join('a.yml'))) + _yaml.roundtrip_dump({'test': {'(>)': ['a']}}, + str(tmpdir.join('a.yml'))) includes.process(main) @@ -50,13 +46,11 @@ def test_include_cannot_append(tmpdir): def test_main_can_append(tmpdir): includes = make_includes(str(tmpdir)) - _yaml.dump({'(@)': ['a.yml'], - 'test': {'(>)': ['main']}}, - str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({'(@)': ['a.yml'], 'test': {'(>)': ['main']}}, + str(tmpdir.join('main.yml'))) main = _yaml.load(str(tmpdir.join('main.yml'))) - _yaml.dump({'test': ['a']}, - str(tmpdir.join('a.yml'))) + _yaml.roundtrip_dump({'test': ['a']}, str(tmpdir.join('a.yml'))) includes.process(main) @@ -66,14 +60,14 @@ def test_main_can_append(tmpdir): def test_sibling_cannot_append_backward(tmpdir): includes = make_includes(str(tmpdir)) - _yaml.dump({'(@)': ['a.yml', 'b.yml']}, - str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({'(@)': ['a.yml', 'b.yml']}, + str(tmpdir.join('main.yml'))) main = _yaml.load(str(tmpdir.join('main.yml'))) - _yaml.dump({'test': {'(>)': ['a']}}, - str(tmpdir.join('a.yml'))) - _yaml.dump({'test': ['b']}, - str(tmpdir.join('b.yml'))) + _yaml.roundtrip_dump({'test': {'(>)': ['a']}}, + str(tmpdir.join('a.yml'))) + _yaml.roundtrip_dump({'test': ['b']}, + str(tmpdir.join('b.yml'))) includes.process(main) @@ -83,14 +77,14 @@ def test_sibling_cannot_append_backward(tmpdir): def test_sibling_can_append_forward(tmpdir): includes = make_includes(str(tmpdir)) - _yaml.dump({'(@)': ['a.yml', 'b.yml']}, - str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({'(@)': ['a.yml', 'b.yml']}, + str(tmpdir.join('main.yml'))) main = _yaml.load(str(tmpdir.join('main.yml'))) - _yaml.dump({'test': ['a']}, - str(tmpdir.join('a.yml'))) - _yaml.dump({'test': {'(>)': ['b']}}, - str(tmpdir.join('b.yml'))) + _yaml.roundtrip_dump({'test': ['a']}, + str(tmpdir.join('a.yml'))) + _yaml.roundtrip_dump({'test': {'(>)': ['b']}}, + str(tmpdir.join('b.yml'))) includes.process(main) @@ -100,14 +94,14 @@ def test_sibling_can_append_forward(tmpdir): def test_lastest_sibling_has_priority(tmpdir): includes = make_includes(str(tmpdir)) - _yaml.dump({'(@)': ['a.yml', 'b.yml']}, - str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({'(@)': ['a.yml', 'b.yml']}, + str(tmpdir.join('main.yml'))) main = _yaml.load(str(tmpdir.join('main.yml'))) - _yaml.dump({'test': ['a']}, - str(tmpdir.join('a.yml'))) - _yaml.dump({'test': ['b']}, - str(tmpdir.join('b.yml'))) + _yaml.roundtrip_dump({'test': ['a']}, + str(tmpdir.join('a.yml'))) + _yaml.roundtrip_dump({'test': ['b']}, + str(tmpdir.join('b.yml'))) includes.process(main) @@ -117,13 +111,11 @@ def test_lastest_sibling_has_priority(tmpdir): def test_main_keeps_keys(tmpdir): includes = make_includes(str(tmpdir)) - _yaml.dump({'(@)': ['a.yml'], - 'something': 'else'}, - str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({'(@)': ['a.yml'], 'something': 'else'}, + str(tmpdir.join('main.yml'))) main = _yaml.load(str(tmpdir.join('main.yml'))) - _yaml.dump({'test': ['a']}, - str(tmpdir.join('a.yml'))) + _yaml.roundtrip_dump({'test': ['a']}, str(tmpdir.join('a.yml'))) includes.process(main) @@ -134,20 +126,19 @@ def test_main_keeps_keys(tmpdir): def test_overwrite_directive_on_later_composite(tmpdir): includes = make_includes(str(tmpdir)) - _yaml.dump({'(@)': ['a.yml', 'b.yml'], - 'test': {'(=)': ['Overwritten']}}, - str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({'(@)': ['a.yml', 'b.yml'], 'test': {'(=)': ['Overwritten']}}, + str(tmpdir.join('main.yml'))) main = _yaml.load(str(tmpdir.join('main.yml'))) # a.yml - _yaml.dump({'test': ['some useless', 'list', 'to be overwritten'], - 'foo': 'should not be present'}, - str(tmpdir.join('a.yml'))) + _yaml.roundtrip_dump({'test': ['some useless', 'list', 'to be overwritten'], + 'foo': 'should not be present'}, + str(tmpdir.join('a.yml'))) # b.yaml isn't going to have a 'test' node to overwrite - _yaml.dump({'foo': 'should be present'}, - str(tmpdir.join('b.yml'))) + _yaml.roundtrip_dump({'foo': 'should be present'}, + str(tmpdir.join('b.yml'))) includes.process(main) diff --git a/tests/format/junctions.py b/tests/format/junctions.py index 8842bc617..b810c55a3 100644 --- a/tests/format/junctions.py +++ b/tests/format/junctions.py @@ -331,7 +331,7 @@ def test_git_show(cli, tmpdir, datafiles): repo.source_config(ref=ref) ] } - _yaml.dump(element, os.path.join(project, 'base.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'base.bst')) # Check that bst show succeeds with implicit subproject fetching and the # pipeline includes the subproject element @@ -356,7 +356,7 @@ def test_git_build(cli, tmpdir, datafiles): repo.source_config(ref=ref) ] } - _yaml.dump(element, os.path.join(project, 'base.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'base.bst')) # Build (with implicit fetch of subproject), checkout result = cli.run(project=project, args=['build', 'target.bst']) @@ -389,7 +389,7 @@ def test_git_missing_project_conf(cli, tmpdir, datafiles): repo.source_config(ref=ref) ] } - _yaml.dump(element, str(project / 'base.bst')) + _yaml.roundtrip_dump(element, str(project / 'base.bst')) result = cli.run(project=project, args=['build', 'app.bst']) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION) @@ -423,7 +423,7 @@ def test_build_git_cross_junction_names(cli, tmpdir, datafiles): repo.source_config(ref=ref) ] } - _yaml.dump(element, os.path.join(project, 'base.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'base.bst')) print(element) print(cli.get_pipeline(project, ['base.bst'])) diff --git a/tests/format/project.py b/tests/format/project.py index 697fe7e3d..2e0a729dc 100644 --- a/tests/format/project.py +++ b/tests/format/project.py @@ -200,7 +200,7 @@ def test_plugin_no_load_ref(cli, datafiles, ref_storage): } ] } - _yaml.dump(config, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(config, os.path.join(project, 'project.conf')) result = cli.run(project=project, silent=True, args=['show', 'noloadref.bst']) diff --git a/tests/format/variables.py b/tests/format/variables.py index ed5407b83..93814279a 100644 --- a/tests/format/variables.py +++ b/tests/format/variables.py @@ -110,7 +110,7 @@ def test_use_of_protected_var_project_conf(cli, datafiles, protected_var): protected_var: 'some-value' } } - _yaml.dump(conf, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(conf, os.path.join(project, 'project.conf')) element = { 'kind': 'import', @@ -121,7 +121,7 @@ def test_use_of_protected_var_project_conf(cli, datafiles, protected_var): } ], } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) result = cli.run(project=project, args=['build', 'target.bst']) result.assert_main_error(ErrorDomain.LOAD, @@ -142,7 +142,7 @@ def test_use_of_protected_var_element_overrides(cli, datafiles, protected_var): } } } - _yaml.dump(conf, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(conf, os.path.join(project, 'project.conf')) element = { 'kind': 'manual', @@ -153,7 +153,7 @@ def test_use_of_protected_var_element_overrides(cli, datafiles, protected_var): } ], } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) result = cli.run(project=project, args=['build', 'target.bst']) result.assert_main_error(ErrorDomain.LOAD, @@ -176,7 +176,7 @@ def test_use_of_protected_var_in_element(cli, datafiles, protected_var): protected_var: 'some-value' } } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) result = cli.run(project=project, args=['build', 'target.bst']) result.assert_main_error(ErrorDomain.LOAD, diff --git a/tests/frontend/__init__.py b/tests/frontend/__init__.py index 8cf7625a9..f1c8c41b8 100644 --- a/tests/frontend/__init__.py +++ b/tests/frontend/__init__.py @@ -7,4 +7,4 @@ from buildstream import _yaml def configure_project(path, config): config['name'] = 'test' config['element-path'] = 'elements' - _yaml.dump(config, os.path.join(path, 'project.conf')) + _yaml.roundtrip_dump(config, os.path.join(path, 'project.conf')) diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py index f9cd1cba4..07ebe7e6a 100644 --- a/tests/frontend/buildcheckout.py +++ b/tests/frontend/buildcheckout.py @@ -133,7 +133,7 @@ def test_build_invalid_filename_chars(datafiles, cli): element = { 'kind': 'stack', } - _yaml.dump(element, os.path.join(project, 'elements', element_name)) + _yaml.roundtrip_dump(element, os.path.join(project, 'elements', element_name)) result = cli.run(project=project, args=strict_args(['build', element_name], 'non-strict')) result.assert_main_error(ErrorDomain.LOAD, "bad-characters-in-name") @@ -151,7 +151,7 @@ def test_build_invalid_filename_chars_dep(datafiles, cli): element = { 'kind': 'stack', } - _yaml.dump(element, os.path.join(project, 'elements', element_name)) + _yaml.roundtrip_dump(element, os.path.join(project, 'elements', element_name)) result = cli.run(project=project, args=strict_args(['build', 'invalid-chars-in-dep.bst'], 'non-strict')) result.assert_main_error(ErrorDomain.LOAD, "bad-characters-in-name") @@ -452,7 +452,7 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): } ] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Now try to track it, this will bail with the appropriate error # informing the user to track the junction first @@ -492,7 +492,7 @@ def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage): } ] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Dump a project.refs if we're using project.refs storage # @@ -508,7 +508,7 @@ def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage): } } } - _yaml.dump(project_refs, os.path.join(project, 'junction.refs')) + _yaml.roundtrip_dump(project_refs, os.path.join(project, 'junction.refs')) # Now try to build it, this should automatically result in fetching # the junction itself at load time. @@ -541,7 +541,7 @@ def test_build_checkout_junction(cli, tmpdir, datafiles): } ] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should automatically result in fetching # the junction itself at load time. @@ -588,7 +588,7 @@ def test_build_checkout_junction_default_targets(cli, tmpdir, datafiles): } ] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should automatically result in fetching # the junction itself at load time. @@ -635,7 +635,7 @@ def test_build_checkout_workspaced_junction(cli, tmpdir, datafiles): } ] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Now open a workspace on the junction # @@ -712,7 +712,7 @@ def test_build_junction_short_notation(cli, tmpdir, datafiles): 'kind': 'stack', 'depends': ['junction.bst:import-etc.bst'] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should automatically result in fetching # the junction itself at load time. @@ -753,7 +753,7 @@ def test_build_junction_short_notation_filename(cli, tmpdir, datafiles): 'kind': 'stack', 'depends': [{'filename': 'junction.bst:import-etc.bst'}] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should automatically result in fetching # the junction itself at load time. @@ -796,7 +796,7 @@ def test_build_junction_short_notation_with_junction(cli, tmpdir, datafiles): 'junction': 'junction.bst', }] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should fail as filenames should not contain # `:` when junction is explicity specified @@ -820,7 +820,7 @@ def test_build_junction_transitive_short_notation_with_junction(cli, tmpdir, dat 'kind': 'stack', 'depends': ['junction.bst:import-etc.bst:foo.bst'] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should fail as recursive lookups for # cross-junction elements is not allowed. diff --git a/tests/frontend/buildtrack.py b/tests/frontend/buildtrack.py index d42b6d1ba..12b90b0d7 100644 --- a/tests/frontend/buildtrack.py +++ b/tests/frontend/buildtrack.py @@ -31,7 +31,7 @@ def create_element(repo, name, path, dependencies, ref=None): ], 'depends': dependencies } - _yaml.dump(element, os.path.join(path, name)) + _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(os.path.join(DATA_DIR)) @@ -120,7 +120,7 @@ def test_build_track(cli, datafiles, tmpdir, ref_storage, strict, 'test': initial_project_refs } } - _yaml.dump(project_refs, os.path.join(project, 'project.refs')) + _yaml.roundtrip_dump(project_refs, os.path.join(project, 'project.refs')) args = ['build'] args += itertools.chain.from_iterable(zip(itertools.repeat('--track'), track_targets)) @@ -219,7 +219,7 @@ def test_build_track_all(cli, tmpdir, datafiles, strict, ref_storage): } ] } - _yaml.dump(element, os.path.join(element_path, 'composed.bst')) + _yaml.roundtrip_dump(element, os.path.join(element_path, 'composed.bst')) # Track the junction itself first. result = cli.run(project=project, args=['source', 'track', 'junction.bst']) diff --git a/tests/frontend/configurable_warnings.py b/tests/frontend/configurable_warnings.py index c63a0b673..7936b2f89 100644 --- a/tests/frontend/configurable_warnings.py +++ b/tests/frontend/configurable_warnings.py @@ -40,7 +40,7 @@ def build_project(datafiles, fatal_warnings): project = get_project(fatal_warnings) - _yaml.dump(project, os.path.join(project_path, "project.conf")) + _yaml.roundtrip_dump(project, os.path.join(project_path, "project.conf")) return project_path diff --git a/tests/frontend/cross_junction_workspace.py b/tests/frontend/cross_junction_workspace.py index 38aafa558..ca21e7548 100644 --- a/tests/frontend/cross_junction_workspace.py +++ b/tests/frontend/cross_junction_workspace.py @@ -13,8 +13,8 @@ def prepare_junction_project(cli, tmpdir): os.makedirs(str(main_project)) os.makedirs(str(sub_project)) - _yaml.dump({'name': 'main'}, str(main_project.join("project.conf"))) - _yaml.dump({'name': 'sub'}, str(sub_project.join("project.conf"))) + _yaml.roundtrip_dump({'name': 'main'}, str(main_project.join("project.conf"))) + _yaml.roundtrip_dump({'name': 'sub'}, str(sub_project.join("project.conf"))) import_dir = tmpdir.join("import") os.makedirs(str(import_dir)) @@ -26,18 +26,18 @@ def prepare_junction_project(cli, tmpdir): import_repo = create_repo("git", str(import_repo_dir)) import_ref = import_repo.create(str(import_dir)) - _yaml.dump({'kind': 'import', - 'sources': [import_repo.source_config(ref=import_ref)]}, - str(sub_project.join("data.bst"))) + _yaml.roundtrip_dump({'kind': 'import', + 'sources': [import_repo.source_config(ref=import_ref)]}, + str(sub_project.join("data.bst"))) sub_repo_dir = tmpdir.join("sub_repo") os.makedirs(str(sub_repo_dir)) sub_repo = create_repo("git", str(sub_repo_dir)) sub_ref = sub_repo.create(str(sub_project)) - _yaml.dump({'kind': 'junction', - 'sources': [sub_repo.source_config(ref=sub_ref)]}, - str(main_project.join("sub.bst"))) + _yaml.roundtrip_dump({'kind': 'junction', + 'sources': [sub_repo.source_config(ref=sub_ref)]}, + str(main_project.join("sub.bst"))) args = ['source', 'fetch', 'sub.bst'] result = cli.run(project=str(main_project), args=args) diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py index d7e0ecafb..ba79dcfd1 100644 --- a/tests/frontend/fetch.py +++ b/tests/frontend/fetch.py @@ -37,9 +37,8 @@ def test_fetch_default_targets(cli, tmpdir, datafiles): repo.source_config(ref=ref) ] } - _yaml.dump(element, - os.path.join(element_path, - element_name)) + _yaml.roundtrip_dump(element, + os.path.join(element_path, element_name)) # Assert that a fetch is needed assert cli.get_element_state(project, element_name) == 'fetch needed' @@ -113,7 +112,7 @@ def test_unfetched_junction(cli, tmpdir, datafiles, strict, ref_storage): } ] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Dump a project.refs if we're using project.refs storage # @@ -129,7 +128,7 @@ def test_unfetched_junction(cli, tmpdir, datafiles, strict, ref_storage): } } } - _yaml.dump(project_refs, os.path.join(project, 'junction.refs')) + _yaml.roundtrip_dump(project_refs, os.path.join(project, 'junction.refs')) # Now try to fetch it, this should automatically result in fetching # the junction itself. @@ -163,7 +162,7 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): } ] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Now try to fetch it, this will bail with the appropriate error # informing the user to track the junction first diff --git a/tests/frontend/logging.py b/tests/frontend/logging.py index a8f894976..6a17bf771 100644 --- a/tests/frontend/logging.py +++ b/tests/frontend/logging.py @@ -39,9 +39,8 @@ def test_default_logging(cli, tmpdir, datafiles): repo.source_config(ref=ref) ] } - _yaml.dump(element, - os.path.join(element_path, - element_name)) + _yaml.roundtrip_dump(element, + os.path.join(element_path, element_name)) # Now try to fetch it result = cli.run(project=project, args=['source', 'fetch', element_name]) @@ -76,9 +75,8 @@ def test_custom_logging(cli, tmpdir, datafiles): repo.source_config(ref=ref) ] } - _yaml.dump(element, - os.path.join(element_path, - element_name)) + _yaml.roundtrip_dump(element, + os.path.join(element_path, element_name)) # Now try to fetch it result = cli.run(project=project, args=['source', 'fetch', element_name]) @@ -104,7 +102,7 @@ def test_failed_build_listing(cli, datafiles): ] } } - _yaml.dump(element, os.path.join(project, element_path)) + _yaml.roundtrip_dump(element, os.path.join(project, element_path)) element_names.append(element_name) result = cli.run(project=project, args=['--on-error=continue', 'build', *element_names]) result.assert_main_error(ErrorDomain.STREAM, None) diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py index 09beb38b8..855155785 100644 --- a/tests/frontend/mirror.py +++ b/tests/frontend/mirror.py @@ -116,7 +116,7 @@ def test_mirror_fetch_ref_storage(cli, tmpdir, datafiles, ref_storage, mirror): full_mirror = mirror_repo.source_config()['url'] mirror_map, _ = os.path.split(full_mirror) os.makedirs(element_dir) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) if ref_storage == 'project.refs': # Manually set project.refs to avoid caching the repo prematurely @@ -128,7 +128,7 @@ def test_mirror_fetch_ref_storage(cli, tmpdir, datafiles, ref_storage, mirror): } }} project_refs_path = os.path.join(project_dir, 'project.refs') - _yaml.dump(project_refs, project_refs_path) + _yaml.roundtrip_dump(project_refs, project_refs_path) project = { 'name': 'test', @@ -151,7 +151,7 @@ def test_mirror_fetch_ref_storage(cli, tmpdir, datafiles, ref_storage, mirror): project['mirrors'] = mirror_data project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) result.assert_success() @@ -167,11 +167,11 @@ def test_mirror_fetch_multi(cli, tmpdir): element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) project_file = os.path.join(project_dir, 'project.conf') project = generate_project() - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) result.assert_success() @@ -191,11 +191,11 @@ def test_mirror_fetch_default_cmdline(cli, tmpdir): element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) project_file = os.path.join(project_dir, 'project.conf') project = generate_project() - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) result = cli.run(project=project_dir, args=['--default-mirror', 'arrakis', 'source', 'fetch', element_name]) result.assert_success() @@ -222,11 +222,11 @@ def test_mirror_fetch_default_userconfig(cli, tmpdir): element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) project_file = os.path.join(project_dir, 'project.conf') project = generate_project() - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) userconfig = { 'projects': { @@ -262,11 +262,11 @@ def test_mirror_fetch_default_cmdline_overrides_config(cli, tmpdir): element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) project_file = os.path.join(project_dir, 'project.conf') project = generate_project() - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) userconfig = { 'projects': { @@ -346,7 +346,7 @@ def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles): aliased_repo = alias + ':' + repo_name element['sources'][0]['submodules']['defined']['url'] = aliased_repo - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) full_mirror = main_mirror.source_config()['url'] mirror_map, _ = os.path.split(full_mirror) @@ -366,7 +366,7 @@ def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles): ] } project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) result.assert_success() @@ -433,7 +433,7 @@ def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles): element_name = 'test.bst' element_path = os.path.join(element_dir, element_name) os.makedirs(element_dir) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) project = { 'name': 'test', @@ -451,7 +451,7 @@ def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles): ] } project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) # Now make the upstream unavailable. os.rename(upstream_bin_repo.repo, '{}.bak'.format(upstream_bin_repo.repo)) @@ -529,7 +529,7 @@ def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles): element_name = 'test.bst' element_path = os.path.join(element_dir, element_name) os.makedirs(element_dir) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) project = { 'name': 'test', @@ -547,7 +547,7 @@ def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles): ] } project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) + _yaml.roundtrip_dump(project, project_file) # Now make the upstream unavailable. os.rename(upstream_main_repo.repo, '{}.bak'.format(upstream_main_repo.repo)) diff --git a/tests/frontend/order.py b/tests/frontend/order.py index 7f354c88e..c62377419 100644 --- a/tests/frontend/order.py +++ b/tests/frontend/order.py @@ -37,7 +37,7 @@ def create_element(project, name, dependencies): ], 'depends': dependencies } - _yaml.dump(element, os.path.join(element_path, name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, name)) return repo diff --git a/tests/frontend/overlaps.py b/tests/frontend/overlaps.py index 6f6ab26a3..eb2cd4a86 100644 --- a/tests/frontend/overlaps.py +++ b/tests/frontend/overlaps.py @@ -25,7 +25,7 @@ def gen_project(project_dir, fail_on_overlap, use_fatal_warnings=True, project_n else: template["fail-on-overlap"] = fail_on_overlap projectfile = os.path.join(project_dir, "project.conf") - _yaml.dump(template, projectfile) + _yaml.roundtrip_dump(template, projectfile) @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/frontend/remote-caches.py b/tests/frontend/remote-caches.py index 8a5ef9c7b..6ee57df23 100644 --- a/tests/frontend/remote-caches.py +++ b/tests/frontend/remote-caches.py @@ -57,7 +57,7 @@ def test_source_artifact_caches(cli, tmpdir, datafiles): }, 'cachedir': cachedir } - _yaml.dump(user_config, filename=user_config_file) + _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) create_element_size('repo.bst', project_dir, element_path, [], 10000) diff --git a/tests/frontend/show.py b/tests/frontend/show.py index 4192ba1e5..2bb04f465 100644 --- a/tests/frontend/show.py +++ b/tests/frontend/show.py @@ -252,7 +252,7 @@ def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage, element_name, w } ] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Dump a project.refs if we're using project.refs storage # @@ -268,7 +268,7 @@ def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage, element_name, w } } } - _yaml.dump(project_refs, os.path.join(project, 'junction.refs')) + _yaml.roundtrip_dump(project_refs, os.path.join(project, 'junction.refs')) # Open a workspace if we're testing workspaced behavior if workspaced: @@ -310,7 +310,7 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, workspaced): } ] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Open a workspace if we're testing workspaced behavior if workspaced: @@ -366,7 +366,7 @@ def test_fetched_junction(cli, tmpdir, datafiles, element_name, workspaced): } ] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project, silent=True, args=[ 'source', 'fetch', 'junction.bst']) @@ -421,7 +421,7 @@ def test_exceed_max_recursion_depth(cli, tmpdir, dependency_depth): } if i == 0: del element['depends'] - _yaml.dump(element, os.path.join(element_path, "element{}.bst".format(str(i)))) + _yaml.roundtrip_dump(element, os.path.join(element_path, "element{}.bst".format(str(i)))) source = os.path.join(sourcefiles_path, "source{}".format(str(i))) open(source, 'x').close() diff --git a/tests/frontend/source_checkout.py b/tests/frontend/source_checkout.py index 38041c45d..1831ee863 100644 --- a/tests/frontend/source_checkout.py +++ b/tests/frontend/source_checkout.py @@ -154,7 +154,7 @@ def test_source_checkout_fetch(datafiles, cli): element = generate_remote_import_element( os.path.join(project, 'files', 'dev-files', 'usr', 'include', 'pony.h'), 'pony.h') - _yaml.dump(element, target_path) + _yaml.roundtrip_dump(element, target_path) # Testing implicit fetching requires that we do not have the sources # cached already diff --git a/tests/frontend/track.py b/tests/frontend/track.py index 1163cc687..b8ab0d4eb 100644 --- a/tests/frontend/track.py +++ b/tests/frontend/track.py @@ -27,7 +27,7 @@ def generate_element(repo, element_path, dep_name=None): if dep_name: element['depends'] = [dep_name] - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) @pytest.mark.datafiles(DATA_DIR) @@ -156,7 +156,7 @@ def test_track_cross_junction(cli, tmpdir, datafiles, cross_junction, ref_storag 'name': 'test', 'ref-storage': ref_storage } - _yaml.dump(project_conf, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(project_conf, os.path.join(project, 'project.conf')) # # FIXME: This can be simplified when we have support @@ -267,7 +267,7 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): } ] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Now try to track it, this will bail with the appropriate error # informing the user to track the junction first @@ -307,7 +307,7 @@ def test_junction_element(cli, tmpdir, datafiles, ref_storage): } ] } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # First demonstrate that showing the pipeline yields an error result = cli.run(project=project, args=['show', 'junction-dep.bst']) diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py index 6b71fbc08..9f8398d14 100644 --- a/tests/frontend/workspace.py +++ b/tests/frontend/workspace.py @@ -92,9 +92,8 @@ class WorkspaceCreator(): } if element_attrs: element = {**element, **element_attrs} - _yaml.dump(element, - os.path.join(element_path, - element_name)) + _yaml.roundtrip_dump(element, + os.path.join(element_path, element_name)) return element_name, element_path, workspace_dir def create_workspace_elements(self, kinds, track, suffixs=None, workspace_dir_usr=None, @@ -683,9 +682,8 @@ def test_buildable_no_ref(cli, tmpdir, datafiles): repo.source_config() ] } - _yaml.dump(element, - os.path.join(element_path, - element_name)) + _yaml.roundtrip_dump(element, + os.path.join(element_path, element_name)) # Assert that this target is not buildable when no workspace is associated. assert cli.get_element_state(project, element_name) == 'no reference' @@ -804,7 +802,7 @@ def test_list_unsupported_workspace(cli, datafiles, workspace_cfg): os.makedirs(os.path.join(project, '.bst')) workspace_config_path = os.path.join(project, '.bst', 'workspaces.yml') - _yaml.dump(workspace_cfg, workspace_config_path) + _yaml.roundtrip_dump(workspace_cfg, workspace_config_path) result = cli.run(project=project, args=['workspace', 'list']) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @@ -909,20 +907,20 @@ def test_list_unsupported_workspace(cli, datafiles, workspace_cfg): def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expected): def parse_dict_as_yaml(node): tempfile = os.path.join(str(tmpdir), 'yaml_dump') - _yaml.dump(node, tempfile) - return _yaml.node_sanitize(_yaml.load(tempfile)) + _yaml.roundtrip_dump(node, tempfile) + return _yaml.load(tempfile).strip_node_info() project = str(datafiles) os.makedirs(os.path.join(project, '.bst')) workspace_config_path = os.path.join(project, '.bst', 'workspaces.yml') - _yaml.dump(workspace_cfg, workspace_config_path) + _yaml.roundtrip_dump(workspace_cfg, workspace_config_path) # Check that we can still read workspace config that is in old format result = cli.run(project=project, args=['workspace', 'list']) result.assert_success() - loaded_config = _yaml.node_sanitize(_yaml.load(workspace_config_path)) + loaded_config = _yaml.load(workspace_config_path).strip_node_info() # Check that workspace config remains the same if no modifications # to workspaces were made @@ -947,9 +945,8 @@ def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expecte repo.source_config(ref=ref) ] } - _yaml.dump(element, - os.path.join(element_path, - element_name)) + _yaml.roundtrip_dump(element, + os.path.join(element_path, element_name)) # Make a change to the workspaces file result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) @@ -958,7 +955,7 @@ def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expecte result.assert_success() # Check that workspace config is converted correctly if necessary - loaded_config = _yaml.node_sanitize(_yaml.load(workspace_config_path)) + loaded_config = _yaml.load(workspace_config_path).strip_node_info() assert loaded_config == parse_dict_as_yaml(expected) @@ -994,9 +991,8 @@ def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict): } ] } - _yaml.dump(element, - os.path.join(element_path, - back_dep_element_name)) + _yaml.roundtrip_dump(element, + os.path.join(element_path, back_dep_element_name)) # Modify workspace shutil.rmtree(os.path.join(workspace, 'usr', 'bin')) @@ -1133,7 +1129,7 @@ def test_external_track(cli, datafiles, tmpdir_factory, guess_element): # element has been tracked element_contents = _yaml.load(element_file) del element_contents.get_sequence('sources').mapping_at(0)['ref'] - _yaml.dump(element_contents, element_file) + _yaml.roundtrip_dump(element_contents, element_file) result = cli.run(project=project, args=['-C', workspace, 'source', 'track', *arg_elm]) result.assert_success() @@ -1247,7 +1243,7 @@ def test_multisource_workspace(cli, datafiles, tmpdir): }] } element_path = os.path.join(project, 'elements', element_name) - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) workspace_dir = os.path.join(str(tmpdir), 'multisource') res = cli.run(project=project, diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py index a2273a06d..64fccadad 100644 --- a/tests/integration/cachedfail.py +++ b/tests/integration/cachedfail.py @@ -45,7 +45,7 @@ def test_build_checkout_cached_fail(cli, datafiles): ], }, } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Try to build it, this should result in a failure that contains the content result = cli.run(project=project, args=['build', 'element.bst']) @@ -87,7 +87,7 @@ def test_build_depend_on_cached_fail(cli, datafiles): ], }, } - _yaml.dump(dep, dep_path) + _yaml.roundtrip_dump(dep, dep_path) target = { 'kind': 'script', 'depends': [ @@ -106,7 +106,7 @@ def test_build_depend_on_cached_fail(cli, datafiles): ], }, } - _yaml.dump(target, target_path) + _yaml.roundtrip_dump(target, target_path) # Try to build it, this should result in caching a failure to build dep result = cli.run(project=project, args=['build', 'dep.bst']) @@ -150,7 +150,7 @@ def test_push_cached_fail(cli, tmpdir, datafiles, on_error): ], }, } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as share: cli.configure({ @@ -188,7 +188,7 @@ def test_host_tools_errors_are_not_cached(cli, datafiles): ], }, } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Build without access to host tools, this will fail result1 = cli.run(project=project, args=['build', 'element.bst'], env={'PATH': ''}) diff --git a/tests/integration/compose.py b/tests/integration/compose.py index 2b37942fa..3562ed94b 100644 --- a/tests/integration/compose.py +++ b/tests/integration/compose.py @@ -36,7 +36,7 @@ def create_compose_element(name, path, config=None): 'config': config } os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True) - _yaml.dump(element, os.path.join(path, name)) + _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/integration/import.py b/tests/integration/import.py index 5371ec61a..bac92cadf 100644 --- a/tests/integration/import.py +++ b/tests/integration/import.py @@ -32,7 +32,7 @@ def create_import_element(name, path, source, target, source_path): } } os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True) - _yaml.dump(element, os.path.join(path, name)) + _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/integration/manual.py b/tests/integration/manual.py index b3124a852..2ac7f74d0 100644 --- a/tests/integration/manual.py +++ b/tests/integration/manual.py @@ -31,7 +31,7 @@ def create_manual_element(name, path, config, variables, environment): 'environment': environment } os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True) - _yaml.dump(element, os.path.join(path, name)) + _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/integration/messages.py b/tests/integration/messages.py index edfb435ae..42725fc5b 100644 --- a/tests/integration/messages.py +++ b/tests/integration/messages.py @@ -59,7 +59,7 @@ def test_disable_message_lines(cli, datafiles): } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # First we check that we get the "Silly message" result = cli.run(project=project, args=["build", element_name]) @@ -94,7 +94,7 @@ def test_disable_error_lines(cli, datafiles): } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # First we check that we get the syntax error result = cli.run(project=project, args=["--error-lines", "0", diff --git a/tests/integration/pip_element.py b/tests/integration/pip_element.py index 91dcaa39e..da0badcb3 100644 --- a/tests/integration/pip_element.py +++ b/tests/integration/pip_element.py @@ -47,7 +47,7 @@ def test_pip_build(cli, datafiles): }] } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) result = cli.run(project=project, args=['build', element_name]) assert result.exit_code == 0 @@ -119,7 +119,7 @@ def test_pip_element_should_install_pip_deps(cli, datafiles, setup_pypi_repo): } ] } - _yaml.dump(element, os.path.join(elements_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(elements_path, element_name)) result = cli.run(project=project, args=['source', 'track', element_name]) assert result.exit_code == 0 diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py index 632b5ae24..c221910a6 100644 --- a/tests/integration/pip_source.py +++ b/tests/integration/pip_source.py @@ -58,7 +58,7 @@ def test_pip_source_import_packages(cli, datafiles, setup_pypi_repo): ] } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) result = cli.run(project=project, args=['source', 'track', element_name]) assert result.exit_code == 0 @@ -116,7 +116,7 @@ def test_pip_source_import_requirements_files(cli, datafiles, setup_pypi_repo): ] } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) result = cli.run(project=project, args=['source', 'track', element_name]) assert result.exit_code == 0 @@ -182,7 +182,7 @@ def test_pip_source_build(cli, datafiles, setup_pypi_repo): } } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) result = cli.run(project=project, args=['source', 'track', element_name]) assert result.exit_code == 0 diff --git a/tests/integration/script.py b/tests/integration/script.py index 1025709f4..fc57e8744 100644 --- a/tests/integration/script.py +++ b/tests/integration/script.py @@ -35,7 +35,7 @@ def create_script_element(name, path, config=None, variables=None): 'variables': variables } os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True) - _yaml.dump(element, os.path.join(path, name)) + _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/integration/shell.py b/tests/integration/shell.py index f7de3e462..a1f38d879 100644 --- a/tests/integration/shell.py +++ b/tests/integration/shell.py @@ -163,7 +163,7 @@ def test_no_shell(cli, datafiles): } } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) result = execute_shell(cli, project, ['/bin/echo', 'Pegasissies!'], element=element_name) assert result.exit_code == 0 diff --git a/tests/integration/source-determinism.py b/tests/integration/source-determinism.py index 4590d4102..70c4b79de 100644 --- a/tests/integration/source-determinism.py +++ b/tests/integration/source-determinism.py @@ -60,7 +60,7 @@ def test_deterministic_source_local(cli, tmpdir, datafiles): ] } } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) def get_value_for_mask(mask): checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(mask)) diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py index fff9518a3..78379912c 100644 --- a/tests/integration/workspace.py +++ b/tests/integration/workspace.py @@ -74,7 +74,7 @@ def test_workspace_updated_dependency(cli, datafiles): } } os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)), exist_ok=True) - _yaml.dump(dependency, os.path.join(element_path, dep_name)) + _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # First open the workspace res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) @@ -90,7 +90,7 @@ def test_workspace_updated_dependency(cli, datafiles): 'mkdir -p %{install-root}/etc/test/', 'echo "Hello china!" > %{install-root}/etc/test/hello.txt' ] - _yaml.dump(dependency, os.path.join(element_path, dep_name)) + _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # `Make` would look at timestamps and normally not realize that # our dependency's header files changed. BuildStream must @@ -129,7 +129,7 @@ def test_workspace_update_dependency_failed(cli, datafiles): } } os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)), exist_ok=True) - _yaml.dump(dependency, os.path.join(element_path, dep_name)) + _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # First open the workspace res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) @@ -146,7 +146,7 @@ def test_workspace_update_dependency_failed(cli, datafiles): 'echo "Hello china!" > %{install-root}/etc/test/hello.txt', 'echo "Hello brazil!" > %{install-root}/etc/test/brazil.txt' ] - _yaml.dump(dependency, os.path.join(element_path, dep_name)) + _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # And our build fails! with open(os.path.join(workspace, 'Makefile'), 'a') as f: @@ -161,7 +161,7 @@ def test_workspace_update_dependency_failed(cli, datafiles): 'echo "Hello world!" > %{install-root}/etc/test/hello.txt', 'echo "Hello spain!" > %{install-root}/etc/test/brazil.txt' ] - _yaml.dump(dependency, os.path.join(element_path, dep_name)) + _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # And fix the source with open(os.path.join(workspace, 'Makefile'), 'r') as f: @@ -204,7 +204,7 @@ def test_updated_dependency_nested(cli, datafiles): } } os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)), exist_ok=True) - _yaml.dump(dependency, os.path.join(element_path, dep_name)) + _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # First open the workspace res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) @@ -221,7 +221,7 @@ def test_updated_dependency_nested(cli, datafiles): 'echo "Hello world!" > %{install-root}/etc/test/hello.txt', 'echo "Hello test!" > %{install-root}/etc/test/tests/tests.txt' ] - _yaml.dump(dependency, os.path.join(element_path, dep_name)) + _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) res = cli.run(project=project, args=['build', element_name]) assert res.exit_code == 0 @@ -257,7 +257,7 @@ def test_incremental_configure_commands_run_only_once(cli, datafiles): ] } } - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # We open a workspace on the above element res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) diff --git a/tests/sandboxes/missing_dependencies.py b/tests/sandboxes/missing_dependencies.py index ee346010e..42be4c9c2 100644 --- a/tests/sandboxes/missing_dependencies.py +++ b/tests/sandboxes/missing_dependencies.py @@ -39,7 +39,7 @@ def test_missing_brwap_has_nice_error_message(cli, datafiles): ], }, } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Build without access to host tools, this should fail with a nice error result = cli.run( @@ -79,7 +79,7 @@ def test_old_brwap_has_nice_error_message(cli, datafiles, tmp_path): ], }, } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Build without access to host tools, this should fail with a nice error result = cli.run( diff --git a/tests/sandboxes/remote-exec-config.py b/tests/sandboxes/remote-exec-config.py index 90418d6fc..a6aeeb7ab 100644 --- a/tests/sandboxes/remote-exec-config.py +++ b/tests/sandboxes/remote-exec-config.py @@ -38,7 +38,7 @@ def test_old_and_new_configs(cli, datafiles): } } project_conf_file = os.path.join(project, 'project.conf') - _yaml.dump(project_conf, project_conf_file) + _yaml.roundtrip_dump(project_conf, project_conf_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # @@ -72,7 +72,7 @@ def test_missing_certs(cli, datafiles, config_key, config_value): } } project_conf_file = os.path.join(project, 'project.conf') - _yaml.dump(project_conf, project_conf_file) + _yaml.roundtrip_dump(project_conf, project_conf_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # @@ -93,7 +93,7 @@ def test_empty_config(cli, datafiles): } } project_conf_file = os.path.join(project, 'project.conf') - _yaml.dump(project_conf, project_conf_file) + _yaml.roundtrip_dump(project_conf, project_conf_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # diff --git a/tests/sourcecache/cache.py b/tests/sourcecache/cache.py index 20faaa64e..9672fabd4 100644 --- a/tests/sourcecache/cache.py +++ b/tests/sourcecache/cache.py @@ -96,7 +96,7 @@ def test_source_cache_key(cli, datafiles): } ] } - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) res = cli.run(project=project_dir, args=["source", "track", element_name]) res.assert_success() diff --git a/tests/sourcecache/config.py b/tests/sourcecache/config.py index 7c33adbe0..2ab11e9f9 100644 --- a/tests/sourcecache/config.py +++ b/tests/sourcecache/config.py @@ -53,7 +53,7 @@ def test_missing_certs(cli, datafiles, config_key, config_value): } } project_conf_file = os.path.join(project, 'project.conf') - _yaml.dump(project_conf, project_conf_file) + _yaml.roundtrip_dump(project_conf, project_conf_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py index cd8a3e989..3f1bce777 100644 --- a/tests/sourcecache/fetch.py +++ b/tests/sourcecache/fetch.py @@ -56,7 +56,7 @@ def test_source_fetch(cli, tmpdir, datafiles): }, 'cachedir': cache_dir, } - _yaml.dump(user_config, filename=user_config_file) + _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) repo = create_repo('git', str(tmpdir)) @@ -67,7 +67,7 @@ def test_source_fetch(cli, tmpdir, datafiles): 'kind': 'import', 'sources': [repo.source_config(ref=ref)] } - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) context = Context() context.load(config=user_config_file) @@ -131,7 +131,7 @@ def test_fetch_fallback(cli, tmpdir, datafiles): }, 'cachedir': cache_dir, } - _yaml.dump(user_config, filename=user_config_file) + _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) repo = create_repo('git', str(tmpdir)) @@ -142,7 +142,7 @@ def test_fetch_fallback(cli, tmpdir, datafiles): 'kind': 'import', 'sources': [repo.source_config(ref=ref)] } - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) context = Context() context.load(config=user_config_file) @@ -188,7 +188,7 @@ def test_pull_fail(cli, tmpdir, datafiles): }, 'cachedir': cache_dir, } - _yaml.dump(user_config, filename=user_config_file) + _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) repo = create_repo('git', str(tmpdir)) @@ -199,7 +199,7 @@ def test_pull_fail(cli, tmpdir, datafiles): 'kind': 'import', 'sources': [repo.source_config(ref=ref)] } - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # get the source object context = Context() diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py index 4c652d21e..466ee1b48 100644 --- a/tests/sourcecache/push.py +++ b/tests/sourcecache/push.py @@ -56,7 +56,7 @@ def test_source_push(cli, tmpdir, datafiles): }, 'cachedir': cache_dir, } - _yaml.dump(user_config, filename=user_config_file) + _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) repo = create_repo('git', str(tmpdir)) @@ -67,7 +67,7 @@ def test_source_push(cli, tmpdir, datafiles): 'kind': 'import', 'sources': [repo.source_config(ref=ref)] } - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # get the source object context = Context() @@ -116,7 +116,7 @@ def test_push_pull(cli, datafiles, tmpdir): }, 'cachedir': cache_dir, } - _yaml.dump(user_config, filename=user_config_file) + _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) # create repo to pull from @@ -128,7 +128,7 @@ def test_push_pull(cli, datafiles, tmpdir): 'kind': 'import', 'sources': [repo.source_config(ref=ref)] } - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) res = cli.run(project=project_dir, args=['build', 'push.bst']) res.assert_success() @@ -162,7 +162,7 @@ def test_push_fail(cli, tmpdir, datafiles): }, 'cachedir': cache_dir, } - _yaml.dump(user_config, filename=user_config_file) + _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) # create repo to pull from @@ -174,7 +174,7 @@ def test_push_fail(cli, tmpdir, datafiles): 'kind': 'import', 'sources': [repo.source_config(ref=ref)] } - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # build and check that it fails to set up the remote res = cli.run(project=project_dir, args=['build', 'push.bst']) @@ -212,7 +212,7 @@ def test_source_push_build_fail(cli, tmpdir, datafiles): 'kind': 'always_fail', 'sources': [repo.source_config(ref=ref)] } - _yaml.dump(element, os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) res = cli.run(project=project_dir, args=['build', 'always-fail.bst']) res.assert_main_error(ErrorDomain.STREAM, None) diff --git a/tests/sources/bzr.py b/tests/sources/bzr.py index 4a66d89b3..c6e78f8c1 100644 --- a/tests/sources/bzr.py +++ b/tests/sources/bzr.py @@ -32,7 +32,7 @@ def test_fetch_checkout(cli, tmpdir, datafiles): repo.source_config(ref=ref) ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) diff --git a/tests/sources/deb.py b/tests/sources/deb.py index bdde20aaa..e536e522a 100644 --- a/tests/sources/deb.py +++ b/tests/sources/deb.py @@ -22,7 +22,7 @@ deb_name = "a_deb.deb" def generate_project(project_dir, tmpdir): project_file = os.path.join(project_dir, "project.conf") - _yaml.dump({ + _yaml.roundtrip_dump({ 'name': 'foo', 'aliases': { 'tmpdir': "file:///" + str(tmpdir) diff --git a/tests/sources/git.py b/tests/sources/git.py index 7ff28114c..c280b5a5a 100644 --- a/tests/sources/git.py +++ b/tests/sources/git.py @@ -58,7 +58,7 @@ def test_fetch_bad_ref(cli, tmpdir, datafiles): repo.source_config(ref='5') ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Assert that fetch raises an error here result = cli.run(project=project, args=[ @@ -92,7 +92,7 @@ def test_submodule_fetch_checkout(cli, tmpdir, datafiles): repo.source_config(ref=ref) ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) @@ -131,7 +131,7 @@ def test_submodule_fetch_source_enable_explicit(cli, tmpdir, datafiles): repo.source_config_extra(ref=ref, checkout_submodules=True) ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) @@ -170,7 +170,7 @@ def test_submodule_fetch_source_disable(cli, tmpdir, datafiles): repo.source_config_extra(ref=ref, checkout_submodules=False) ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) @@ -209,7 +209,7 @@ def test_submodule_fetch_submodule_does_override(cli, tmpdir, datafiles): repo.source_config_extra(ref=ref, checkout_submodules=False) ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) @@ -253,7 +253,7 @@ def test_submodule_fetch_submodule_individual_checkout(cli, tmpdir, datafiles): repo.source_config_extra(ref=ref, checkout_submodules=True) ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) @@ -298,7 +298,7 @@ def test_submodule_fetch_submodule_individual_checkout_explicit(cli, tmpdir, dat repo.source_config_extra(ref=ref, checkout_submodules=True) ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) @@ -338,7 +338,7 @@ def test_submodule_fetch_project_override(cli, tmpdir, datafiles): repo.source_config(ref=ref) ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) @@ -369,7 +369,7 @@ def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles): repo.source_config(ref=ref) ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Now add a .gitmodules file with an inconsistent submodule, # we are calling this inconsistent because the file was created @@ -409,7 +409,7 @@ def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles): ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Track will encounter an inconsistent submodule without any ref result = cli.run(project=project, args=['show', 'target.bst']) @@ -429,7 +429,7 @@ def test_ref_not_in_track(cli, tmpdir, datafiles, fail): "name": "foo", "fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK] } - _yaml.dump(project_template, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf')) # Create the repo from 'repofiles', create a branch without latest commit repo = create_repo('git', str(tmpdir)) @@ -447,7 +447,7 @@ def test_ref_not_in_track(cli, tmpdir, datafiles, fail): gitsource ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) result = cli.run(project=project, args=['build', 'target.bst']) @@ -472,7 +472,7 @@ def test_unlisted_submodule(cli, tmpdir, datafiles, fail): "name": "foo", "fatal-warnings": ['git:unlisted-submodule'] } - _yaml.dump(project_template, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf')) # Create the submodule first from the 'subrepofiles' subdir subrepo = create_repo('git', str(tmpdir), 'subrepo') @@ -501,7 +501,7 @@ def test_unlisted_submodule(cli, tmpdir, datafiles, fail): gitsource ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # We will not see the warning or error before the first fetch, because # we don't have the repository yet and so we have no knowledge of @@ -545,7 +545,7 @@ def test_track_unlisted_submodule(cli, tmpdir, datafiles, fail): "name": "foo", "fatal-warnings": ['git:unlisted-submodule'] } - _yaml.dump(project_template, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf')) # Create the submodule first from the 'subrepofiles' subdir subrepo = create_repo('git', str(tmpdir), 'subrepo') @@ -571,7 +571,7 @@ def test_track_unlisted_submodule(cli, tmpdir, datafiles, fail): gitsource ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Fetch the repo, we will not see the warning because we # are still pointing to a ref which predates the submodules @@ -608,7 +608,7 @@ def test_invalid_submodule(cli, tmpdir, datafiles, fail): "name": "foo", "fatal-warnings": ['git:invalid-submodule'] } - _yaml.dump(project_template, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf')) # Create the repo from 'repofiles' subdir repo = create_repo('git', str(tmpdir)) @@ -635,7 +635,7 @@ def test_invalid_submodule(cli, tmpdir, datafiles, fail): gitsource ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # We will not see the warning or error before the first fetch, because # we don't have the repository yet and so we have no knowledge of @@ -680,7 +680,7 @@ def test_track_invalid_submodule(cli, tmpdir, datafiles, fail): "name": "foo", "fatal-warnings": ['git:invalid-submodule'] } - _yaml.dump(project_template, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf')) # Create the submodule first from the 'subrepofiles' subdir subrepo = create_repo('git', str(tmpdir), 'subrepo') @@ -707,7 +707,7 @@ def test_track_invalid_submodule(cli, tmpdir, datafiles, fail): gitsource ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Fetch the repo, we will not see the warning because we # are still pointing to a ref which predates the submodules @@ -753,7 +753,7 @@ def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit): } element['sources'][0]['ref-format'] = ref_format element_path = os.path.join(project, 'target.bst') - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) # Track it result = cli.run(project=project, args=['source', 'track', 'target.bst']) @@ -786,7 +786,7 @@ def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type): project_config = _yaml.load(os.path.join(project, 'project.conf')) project_config['ref-storage'] = ref_storage - _yaml.dump(project_config, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf')) repofiles = os.path.join(str(tmpdir), 'repofiles') os.makedirs(repofiles, exist_ok=True) @@ -838,7 +838,7 @@ def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type): ], } element_path = os.path.join(project, 'target.bst') - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) if ref_storage == 'inline': result = cli.run(project=project, args=['source', 'track', 'target.bst']) @@ -900,7 +900,7 @@ def test_git_describe_head_is_tagged(cli, tmpdir, datafiles, ref_storage, tag_ty project_config = _yaml.load(os.path.join(project, 'project.conf')) project_config['ref-storage'] = ref_storage - _yaml.dump(project_config, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf')) repofiles = os.path.join(str(tmpdir), 'repofiles') os.makedirs(repofiles, exist_ok=True) @@ -951,7 +951,7 @@ def test_git_describe_head_is_tagged(cli, tmpdir, datafiles, ref_storage, tag_ty ], } element_path = os.path.join(project, 'target.bst') - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) if ref_storage == 'inline': result = cli.run(project=project, args=['source', 'track', 'target.bst']) @@ -1015,7 +1015,7 @@ def test_git_describe_relevant_history(cli, tmpdir, datafiles): project_config = _yaml.load(os.path.join(project, 'project.conf')) project_config['ref-storage'] = 'project.refs' - _yaml.dump(project_config, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf')) repofiles = os.path.join(str(tmpdir), 'repofiles') os.makedirs(repofiles, exist_ok=True) @@ -1064,7 +1064,7 @@ def test_git_describe_relevant_history(cli, tmpdir, datafiles): ], } element_path = os.path.join(project, 'target.bst') - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project, args=['source', 'track', 'target.bst', '--deps', 'all']) result.assert_success() @@ -1095,7 +1095,7 @@ def test_default_do_not_track_tags(cli, tmpdir, datafiles): project_config = _yaml.load(os.path.join(project, 'project.conf')) project_config['ref-storage'] = 'inline' - _yaml.dump(project_config, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf')) repofiles = os.path.join(str(tmpdir), 'repofiles') os.makedirs(repofiles, exist_ok=True) @@ -1119,7 +1119,7 @@ def test_default_do_not_track_tags(cli, tmpdir, datafiles): ], } element_path = os.path.join(project, 'target.bst') - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project, args=['source', 'track', 'target.bst']) result.assert_success() @@ -1162,7 +1162,7 @@ def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles): } } ] - _yaml.dump(project_config, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf')) repo.add_annotated_tag('tag', 'tag') @@ -1184,7 +1184,7 @@ def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles): ], } element_path = os.path.join(project, 'target.bst') - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project, args=['build', 'target.bst']) result.assert_success() @@ -1208,7 +1208,7 @@ def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles): repodir, reponame = os.path.split(repo.repo) - _yaml.dump(project_config, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf')) config = repo.source_config(ref=new_ref) del config['track'] @@ -1220,7 +1220,7 @@ def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles): config ], } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project, args=['build', 'target.bst']) result.assert_success() diff --git a/tests/sources/local.py b/tests/sources/local.py index fb5d36081..4b72a4343 100644 --- a/tests/sources/local.py +++ b/tests/sources/local.py @@ -217,4 +217,4 @@ def test_deterministic_source_umask(cli, tmpdir, datafiles): ] } } - _yaml.dump(element, element_path) + _yaml.roundtrip_dump(element, element_path) diff --git a/tests/sources/no_fetch_cached.py b/tests/sources/no_fetch_cached.py index fcbb42398..81032881c 100644 --- a/tests/sources/no_fetch_cached.py +++ b/tests/sources/no_fetch_cached.py @@ -39,7 +39,7 @@ def test_no_fetch_cached(cli, tmpdir, datafiles): } ] } - _yaml.dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) # Test fetch of target with a cached and a non-cached source result = cli.run(project=project, args=[ diff --git a/tests/sources/pip.py b/tests/sources/pip.py index cf6ea5498..7f91ba701 100644 --- a/tests/sources/pip.py +++ b/tests/sources/pip.py @@ -17,7 +17,7 @@ DATA_DIR = os.path.join( def generate_project(project_dir): project_file = os.path.join(project_dir, "project.conf") - _yaml.dump({'name': 'foo'}, project_file) + _yaml.roundtrip_dump({'name': 'foo'}, project_file) # Test that without ref, consistency is set appropriately. diff --git a/tests/sources/previous_source_access.py b/tests/sources/previous_source_access.py index 916cd5a6f..750b94381 100644 --- a/tests/sources/previous_source_access.py +++ b/tests/sources/previous_source_access.py @@ -26,7 +26,7 @@ def test_custom_transform_source(cli, datafiles): project_config = _yaml.load(project_config_path) aliases = project_config.get_mapping("aliases") aliases["project_dir"] = "file://{}".format(project) - _yaml.dump(project_config, project_config_path) + _yaml.roundtrip_dump(project_config, project_config_path) # Ensure we can track result = cli.run(project=project, args=[ diff --git a/tests/sources/remote.py b/tests/sources/remote.py index 8b57151b2..5b818b960 100644 --- a/tests/sources/remote.py +++ b/tests/sources/remote.py @@ -18,7 +18,7 @@ DATA_DIR = os.path.join( def generate_project(project_dir, tmpdir): project_file = os.path.join(project_dir, "project.conf") - _yaml.dump({ + _yaml.roundtrip_dump({ 'name': 'foo', 'aliases': { 'tmpdir': "file:///" + str(tmpdir) @@ -28,7 +28,7 @@ def generate_project(project_dir, tmpdir): def generate_project_file_server(server, project_dir): project_file = os.path.join(project_dir, "project.conf") - _yaml.dump({ + _yaml.roundtrip_dump({ 'name': 'foo', 'aliases': { 'tmpdir': server.base_url() diff --git a/tests/sources/tar.py b/tests/sources/tar.py index a6c1a4d9f..06a3cfc0e 100644 --- a/tests/sources/tar.py +++ b/tests/sources/tar.py @@ -47,7 +47,7 @@ def _assemble_tar_lz(workingdir, srcdir, dstfile): def generate_project(project_dir, tmpdir): project_file = os.path.join(project_dir, "project.conf") - _yaml.dump({ + _yaml.roundtrip_dump({ 'name': 'foo', 'aliases': { 'tmpdir': "file:///" + str(tmpdir) @@ -57,7 +57,7 @@ def generate_project(project_dir, tmpdir): def generate_project_file_server(base_url, project_dir): project_file = os.path.join(project_dir, "project.conf") - _yaml.dump({ + _yaml.roundtrip_dump({ 'name': 'foo', 'aliases': { 'tmpdir': base_url diff --git a/tests/sources/zip.py b/tests/sources/zip.py index cb7f440f1..3fd43b4bb 100644 --- a/tests/sources/zip.py +++ b/tests/sources/zip.py @@ -32,7 +32,7 @@ def _assemble_zip(workingdir, dstfile): def generate_project(project_dir, tmpdir): project_file = os.path.join(project_dir, "project.conf") - _yaml.dump({ + _yaml.roundtrip_dump({ 'name': 'foo', 'aliases': { 'tmpdir': "file:///" + str(tmpdir) @@ -42,7 +42,7 @@ def generate_project(project_dir, tmpdir): def generate_project_file_server(server, project_dir): project_file = os.path.join(project_dir, "project.conf") - _yaml.dump({ + _yaml.roundtrip_dump({ 'name': 'foo', 'aliases': { 'tmpdir': server.base_url() diff --git a/tests/testutils/element_generators.py b/tests/testutils/element_generators.py index 38bafc6b6..0fbca7f3e 100644 --- a/tests/testutils/element_generators.py +++ b/tests/testutils/element_generators.py @@ -59,7 +59,7 @@ def create_element_size(name, project_dir, elements_path, dependencies, size): }, 'depends': dependencies } - _yaml.dump(element, os.path.join(project_dir, elements_path, name)) + _yaml.roundtrip_dump(element, os.path.join(project_dir, elements_path, name)) # Return the repo, so that it can later be used to add commits return repo diff --git a/tests/testutils/junction.py b/tests/testutils/junction.py index 8132e291c..e867695c4 100644 --- a/tests/testutils/junction.py +++ b/tests/testutils/junction.py @@ -30,6 +30,6 @@ def generate_junction(tmpdir, subproject_path, junction_path, *, store_ref=True) repo.source_config(ref=source_ref) ] } - _yaml.dump(element, junction_path) + _yaml.roundtrip_dump(element, junction_path) return ref |