diff options
author | Anthon van der Neut <anthon@mnt.org> | 2017-07-13 23:32:42 +0200 |
---|---|---|
committer | Anthon van der Neut <anthon@mnt.org> | 2017-07-13 23:32:42 +0200 |
commit | eaba7badcb3ce04a9db6e4fe8504950cef832f7d (patch) | |
tree | b6fefde9db338bf03ae0569ad9c4d84bd96aa1ac | |
parent | 2ea4de43fe2e0160e27b605f2c65f33c5a2083d7 (diff) | |
download | ruamel.yaml-eaba7badcb3ce04a9db6e4fe8504950cef832f7d.tar.gz |
added register_class/yaml_object0.15.19
-rw-r--r-- | .hgignore | 1 | ||||
-rw-r--r-- | CHANGES | 6 | ||||
-rw-r--r-- | README.rst | 6 | ||||
-rw-r--r-- | __init__.py | 4 | ||||
-rw-r--r-- | _doc/api.rst | 76 | ||||
-rw-r--r-- | _doc/api.ryd | 269 | ||||
-rw-r--r-- | _doc/basicuse.rst | 45 | ||||
-rw-r--r-- | _doc/basicuse.ryd | 78 | ||||
-rw-r--r-- | _doc/dumpcls.rst | 97 | ||||
-rw-r--r-- | _doc/dumpcls.ryd | 106 | ||||
-rw-r--r-- | _doc/index.rst | 1 | ||||
-rw-r--r-- | _test/roundtrip.py | 12 | ||||
-rw-r--r-- | _test/test_class_register.py | 137 | ||||
-rw-r--r-- | _test/test_literal.py | 13 | ||||
-rw-r--r-- | constructor.py | 11 | ||||
-rw-r--r-- | main.py | 56 | ||||
-rw-r--r-- | scanner.py | 51 | ||||
-rw-r--r-- | setup.py | 8 | ||||
-rw-r--r-- | tox.ini | 4 |
19 files changed, 890 insertions, 91 deletions
@@ -5,3 +5,4 @@ README.pdf venv TODO.rst try_* +_doc/*.pdf @@ -1,3 +1,9 @@ +[0, 15, 19]: 2017-07-13 + - added object constructor for rt, decorator ``yaml_object`` to replace YAMLObject. + - fix for problem using load_all with Path() instance + - fix for load_all in combination with zero indent block style literal + (``pure=True`` only!) + [0, 15, 18]: 2017-07-04 - missing ``pure`` attribute on ``YAML`` useful for implementing `!include` tag constructor for `including YAML files in a YAML file @@ -32,6 +32,12 @@ ChangeLog .. should insert NEXT: at the beginning of line for next key +0.15.19 (2017-07-13): + - added object constructor for rt, decorator ``yaml_object`` to replace YAMLObject. + - fix for problem using load_all with Path() instance + - fix for load_all in combination with zero indent block style literal + (``pure=True`` only!) + 0.15.18 (2017-07-04): - missing ``pure`` attribute on ``YAML`` useful for implementing `!include` tag constructor for `including YAML files in a YAML file diff --git a/__init__.py b/__init__.py index 30fe00a..925a5c2 100644 --- a/__init__.py +++ b/__init__.py @@ -7,8 +7,8 @@ if False: # MYPY _package_data = dict( full_package_name='ruamel.yaml', - version_info=(0, 15, 19, 'dev'), - __version__='0.15.19.dev', + version_info=(0, 15, 19), + __version__='0.15.19', author='Anthon van der Neut', author_email='a.van.der.neut@ruamel.eu', description='ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order', # NOQA diff --git a/_doc/api.rst b/_doc/api.rst index f6ff28c..7f34c93 100644 --- a/_doc/api.rst +++ b/_doc/api.rst @@ -1,5 +1,3 @@ - - Departure from previous API --------------------------- @@ -35,23 +33,24 @@ of ``YAML`` before calling ``load()`` or ``dump()`` Before 0.15.0:: - from pathlib import Path - from ruamel import yaml - - data = yaml.safe_load("abc: 1") - out = path('/tmp/out.yaml') - with out.open('w') as fp: - yaml.safe_dump(data, fp, default_flow_style=False) + from pathlib import Path + from ruamel import yaml + + data = yaml.safe_load("abc: 1") + out = Path('/tmp/out.yaml') + with out.open('w') as fp: + yaml.safe_dump(data, fp, default_flow_style=False) after:: - from ruamel.yaml import YAML - - yaml = YAML(typ='safe') - yaml.default_flow_style = False - data = yaml.load("abc: 1") - out = path('/tmp/out.yaml') - yaml.dump(data, out) + from pathlib import Path + from ruamel.yaml import YAML + + yaml = YAML(typ='safe') + yaml.default_flow_style = False + data = yaml.load("abc: 1") + out = Path('/tmp/out.yaml') + yaml.dump(data, out) If you previously used an keyword argument ``explicit_start=True`` you now do ``yaml.explicit_start = True`` before calling ``dump()``. The @@ -63,6 +62,12 @@ is possible by setting the attributes ``.Parser``, ``.Constructor``, ``.Emitter``, etc., to the class of the type to create for that stage (typically a subclass of an existing class implementing that). +The default loader (`rt`) is a direct derivative of the safe loader, without the +methods to construct arbitrary Python objects that make the ``unsafe`` loader +unsafe, but with the changes needed for round-trip preservation of comments, +etc.. For trusted Python classes a constructor can of course be added to the round-trip +or safe-loader, but this has to be done explicitly (``add_constructor``). + All data is dumped (not just for round-trip-mode) with ``.allow_unicode = True`` @@ -76,7 +81,6 @@ all functionality of the old interface will be available via Loading ------- - Duplicate keys ++++++++++++++ @@ -127,55 +131,53 @@ for reading resp. writing. Loading and dumping using the ``SafeLoader``:: - if yaml.version_info < (0, 15): + if ruamel.yaml.version_info < (0, 15): data = yaml.safe_load(istream) yaml.safe_dump(data, ostream) else: - yml = yaml.YAML(typ='safe', pure=True) # 'safe' load and dump + yml = ruamel.yaml.YAML(typ='safe', pure=True) # 'safe' load and dump data = yml.load(istream) - yml.dump(ostream) - + yml.dump(data, ostream) Loading with the ``CSafeLoader``, dumping with ``RoundTripLoader``. You need two ``YAML`` instances, but each of them -can be re-used :: +can be re-used:: - if yaml.version_info < (0, 15): + if ruamel.yaml.version_info < (0, 15): data = yaml.load(istream, Loader=yaml.CSafeLoader) yaml.round_trip_dump(data, ostream, width=1000, explicit_start=True) else: - yml = yaml.YAML(typ='safe') + yml = ruamel.yaml.YAML(typ='safe') data = yml.load(istream) - ymlo = yaml.YAML() # or yaml.YAML(typ='rt') + ymlo = ruamel.yaml.YAML() # or yaml.YAML(typ='rt') ymlo.width = 1000 ymlo.explicit_start = True - ymlo.dump(ostream) - + ymlo.dump(data, ostream) Loading and dumping from ``pathlib.Path`` instances using the round-trip-loader:: # in myyaml.py - if yaml.version_info < (0, 15): + if ruamel.yaml.version_info < (0, 15): class MyYAML(yaml.YAML): def __init__(self): yaml.YAML.__init__(self) self.preserve_quotes = True self.indent = 4 self.block_seq_indent = 2 - + # in your code try: from myyaml import MyYAML - except ImportError: - if yaml.version_info >= (0, 15): + except (ModuleNotFoundError, ImportError): + if ruamel.yaml.version_info >= (0, 15): raise - + # some pathlib.Path from pathlib import Path inf = Path('/tmp/in.yaml') outf = Path('/tmp/out.yaml') - - if yaml.version_info < (0, 15): + + if ruamel.yaml.version_info < (0, 15): with inf.open() as ifp: data = yaml.round_trip_load(ifp, preserve_quotes=True) with outf.open('w') as ofp: @@ -184,10 +186,7 @@ round-trip-loader:: yml = MyYAML() # no need for with statement when using pathlib.Path instances data = yml.load(inf) - yml.dump(outf) - - - + yml.dump(data, outf) Reason for API change --------------------- @@ -242,3 +241,4 @@ information via that instance. Representers, etc., are added to a reusable instance and different YAML instances can co-exists. This change eases development and helps prevent regressions. + diff --git a/_doc/api.ryd b/_doc/api.ryd new file mode 100644 index 0000000..24f06e6 --- /dev/null +++ b/_doc/api.ryd @@ -0,0 +1,269 @@ +--- +version: 0.1 +output: rst +fix_inline_single_backquotes: true +pdf: true +--- !python-pre | +import sys +from io import StringIO +import ruamel.yaml +from ruamel.yaml import YAML +yaml=YAML() +ostream = s = StringIO() +istream = stream = doc = "a: 1" +data = dict(a=1) +from pathlib import Path # or: from ruamel.std.pathlib import Path +--- | + +Departure from previous API +--------------------------- + +With version 0.15.0 ``ruamel.yaml`` starts to depart from the previous (PyYAML) way +of loading and dumping. During a transition period the original +``load()`` and ``dump()`` in its various formats will still be supported, +but this is not guaranteed to be so with the transition to 1.0. + +At the latest with 1.0, but possible earlier transition error and +warning messages will be issued, so any packages depending on +ruamel.yaml should pin the version with which they are testing. + + +Up to 0.15.0, the loaders (``load()``, ``safe_load()``, +``round_trip_load()``, ``load_all``, etc.) took, apart from the input +stream, a ``version`` argument to allow downgrading to YAML 1.1, +sometimes needed for +documents without directive. When round-tripping, there was an option to +preserve quotes. + +Up to 0.15.0, the dumpers (``dump()``, ``safe_dump``, +``round_trip_dump()``, ``dump_all()``, etc.) had a plethora of +arguments, some inhereted from ``PyYAML``, some added in +``ruamel.yaml``. The only required argument is the ``data`` to be +dumped. If the stream argument is not provided to the dumper, then a +string representation is build up in memory and returned to the +caller. + +Starting with 0.15.0 ``load()`` and ``dump()`` are methods on a +``YAML`` instance and only take the stream, +resp. the data and stram argument. All other parameters are set on the instance +of ``YAML`` before calling ``load()`` or ``dump()`` + +Before 0.15.0:: +--- !python | +from pathlib import Path +from ruamel import yaml + +data = yaml.safe_load("abc: 1") +out = Path('/tmp/out.yaml') +with out.open('w') as fp: + yaml.safe_dump(data, fp, default_flow_style=False) +--- | +after:: +--- !python | +from pathlib import Path +from ruamel.yaml import YAML + +yaml = YAML(typ='safe') +yaml.default_flow_style = False +data = yaml.load("abc: 1") +out = Path('/tmp/out.yaml') +yaml.dump(data, out) + +--- | +If you previously used an keyword argument ``explicit_start=True`` you +now do ``yaml.explicit_start = True`` before calling ``dump()``. The +``Loader`` and ``Dumper`` keyword arguments are not supported that +way. You can provide the ``typ`` keyword to ``rt`` (default), +``safe``, ``unsafe`` or ``base`` (for round-trip load/dump, safe_load/dump, +load/dump resp. using the BaseLoader / BaseDumper. More fine-control +is possible by setting the attributes ``.Parser``, ``.Constructor``, +``.Emitter``, etc., to the class of the type to create for that stage +(typically a subclass of an existing class implementing that). + +The default loader (`rt`) is a direct derivative of the safe loader, without the +methods to construct arbitrary Python objects that make the ``unsafe`` loader +unsafe, but with the changes needed for round-trip preservation of comments, +etc.. For trusted Python classes a constructor can of course be added to the round-trip +or safe-loader, but this has to be done explicitly (``add_constructor``). + +All data is dumped (not just for round-trip-mode) with ``.allow_unicode += True`` + +You can of course have multiple YAML instances active at the same +time, with different load and/or dump behaviour. + +Initially only the typical operations are supported, but in principle +all functionality of the old interface will be available via +``YAML`` instances (if you are using something that isn't let me know). + +Loading +------- + +Duplicate keys +++++++++++++++ + +In JSON mapping keys should be unique, in YAML they must be unique. +PyYAML never enforced this although the YAML 1.1 specification already +required this. + +In the new API (starting 0.15.1) duplicate keys in mappings are no longer allowed by +default. To allow duplicate keys in mappings:: + +--- !python | +yaml = ruamel.yaml.YAML() +yaml.allow_duplicate_keys = True +yaml.load(stream) +--- | +In the old API this is a warning starting with 0.15.2 and an error in +0.16.0. + +Dumping +------- + +Controls +++++++++ + +On your ``YAML()`` instance you can set attributes e.g with:: + + yaml = YAML(typ='safe', pure=True) + yaml.allow_unicode = False + +available attributes include: + +``unicode_supplementary`` + Defaults to ``True`` if Python's Unicode size is larger than 2 bytes. Set to ``False`` to + enforce output of the form ``\U0001f601`` (ignored if ``allow_unicode`` is ``False``) + +Transparent usage of new and old API +------------------------------------ + +If you have multiple packages depending on ``ruamel.yaml``, or install +your utility together with other packages not under your control, then +fixing your ``install_requires`` might not be so easy. + +Depending on your usage you might be able to "version" your usage to +be compatible with both the old and the new. The following are some +examples all assuming ``from ruamel import yaml`` somewhere at the top +of your file and some ``istream`` and ``ostream`` apropriately opened +for reading resp. writing. + + +Loading and dumping using the ``SafeLoader``:: + +--- !python | +if ruamel.yaml.version_info < (0, 15): + data = yaml.safe_load(istream) + yaml.safe_dump(data, ostream) +else: + yml = ruamel.yaml.YAML(typ='safe', pure=True) # 'safe' load and dump + data = yml.load(istream) + yml.dump(data, ostream) +--- | + +Loading with the ``CSafeLoader``, dumping with +``RoundTripLoader``. You need two ``YAML`` instances, but each of them +can be re-used:: + +--- !python | +if ruamel.yaml.version_info < (0, 15): + data = yaml.load(istream, Loader=yaml.CSafeLoader) + yaml.round_trip_dump(data, ostream, width=1000, explicit_start=True) +else: + yml = ruamel.yaml.YAML(typ='safe') + data = yml.load(istream) + ymlo = ruamel.yaml.YAML() # or yaml.YAML(typ='rt') + ymlo.width = 1000 + ymlo.explicit_start = True + ymlo.dump(data, ostream) +--- | + + +Loading and dumping from ``pathlib.Path`` instances using the +round-trip-loader:: + +--- !code | +# in myyaml.py +if ruamel.yaml.version_info < (0, 15): + class MyYAML(yaml.YAML): + def __init__(self): + yaml.YAML.__init__(self) + self.preserve_quotes = True + self.indent = 4 + self.block_seq_indent = 2 +# in your code +try: + from myyaml import MyYAML +except (ModuleNotFoundError, ImportError): + if ruamel.yaml.version_info >= (0, 15): + raise + +# some pathlib.Path +from pathlib import Path +inf = Path('/tmp/in.yaml') +outf = Path('/tmp/out.yaml') + +if ruamel.yaml.version_info < (0, 15): + with inf.open() as ifp: + data = yaml.round_trip_load(ifp, preserve_quotes=True) + with outf.open('w') as ofp: + yaml.round_trip_dump(data, ofp, indent=4, block_seq_indent=2) +else: + yml = MyYAML() + # no need for with statement when using pathlib.Path instances + data = yml.load(inf) + yml.dump(data, outf) +--- | + +Reason for API change +--------------------- + +``ruamel.yaml`` inherited the way of doing things from ``PyYAML``. In +particular when calling the function ``load()`` or ``dump()`` a +temporary instances of ``Loader()`` resp. ``Dumper()`` were +created that were discarded on termination of the function. + +This way of doing things leads to several problems: + +- it is virtually impossible to return information to the caller apart from the + constructed data structure. E.g. if you would get a YAML document + version number from a directive, there is no way to let the caller + know apart from handing back special data structures. The same + problem exists when trying to do on the fly + analysis of a document for indentation width. + +- these instances were composites of the various load/dump steps and + if you wanted to enhance one of the steps, you needed e.g. subclass + the emitter and make a new composite (dumper) as well, providing all + of the parameters (i.e. copy paste) + + Alternatives, like making a class that returned a ``Dumper`` when + called and sets attributes before doing so, is cumbersome for + day-to-day use. + +- many routines (like ``add_representer()``) have a direct global + impact on all of the following calls to ``dump()`` and those are + difficult if not impossible to turn back. This forces the need to + subclass ``Loaders`` and ``Dumpers``, a long time problem in PyYAML + as some attributes were not ``deep_copied`` although a bug-report + (and fix) had been available a long time. + +- If you want to set an attribute, e.g. to control whether literal + block style scalars are allowed to have trailing spaces on a line + instead of being dumped as double quoted scalars, you have to change + the ``dump()`` family of routines, all of the ``Dumpers()`` as well + as the actual functionality change in ``emitter.Emitter()``. The + functionality change takes changing 4 (four!) lines in one file, and being able + to enable that another 50+ line changes (non-contiguous) in 3 more files resulting + in diff that is far over 200 lines long. + +- replacing libyaml with something that doesn't both support `0o52` + and `052` for the integer ``42`` (instead of ``52`` as per YAML 1.2) + is difficult + + +With ``ruamel.yaml>=0.15.0`` the various steps "know" about the +``YAML`` instance and can pick up setting, as well as report back +information via that instance. Representers, etc., are added to a +reusable instance and different YAML instances can co-exists. + +This change eases development and helps prevent regressions. diff --git a/_doc/basicuse.rst b/_doc/basicuse.rst index 56a28d6..f99d609 100644 --- a/_doc/basicuse.rst +++ b/_doc/basicuse.rst @@ -9,11 +9,10 @@ the process of being fleshed out*. Please pin your dependency to You load a YAML document using:: - from ruamel.yaml import YAML - - yaml=YAML(typ='safe') # default if not specfied is round-trip - - yaml.load(doc) + from ruamel.yaml import YAML + + yaml=YAML(typ='safe') # default if not specfied is round-trip + yaml.load(doc) in this ``doc`` can be a file pointer (i.e. an object that has the `.read()` method, a string or a ``pathlib.Path()``. `typ='safe'` @@ -24,11 +23,11 @@ when possible/available) Dumping works in the same way:: - from ruamel.yaml import YAML - - yaml=YAML() - yaml.default_flow_style = False - yaml.dump({a: [1, 2]}, s) + from ruamel.yaml import YAML + + yaml=YAML() + yaml.default_flow_style = False + yaml.dump({'a': [1, 2]}, s) in this ``s`` can be a file pointer (i.e. an object that has the `.write()` method, or a ``pathlib.Path()``. If you want to display @@ -37,26 +36,24 @@ your output, just stream to `sys.stdout`. If you need to transform a string representation of the output provide a function that takes a string as input and returns one:: - def tr(s): - return s.replace('\n', '<\n') # such output is not valid YAML! - - yaml.dump(data, sys.stdout, transform=tr) - + def tr(s): + return s.replace('\n', '<\n') # such output is not valid YAML! + + yaml.dump(data, sys.stdout, transform=tr) More examples ------------- - Using the C based SafeLoader (at this time is inherited from libyaml/PyYAML and e.g. loads ``0o52`` as well as ``052`` load as integer ``42``):: - from ruamel.yaml import YAML - - yaml=YAML(typ="safe") - yaml.load("""a:\n b: 2\n c: 3\n""") + from ruamel.yaml import YAML + + yaml=YAML(typ="safe") + yaml.load("""a:\n b: 2\n c: 3\n""") Using the Python based SafeLoader (YAML 1.2 support, ``052`` loads as ``52``):: - from ruamel.yaml import YAML - - yaml=YAML(typ="safe", pure=True) - yaml.load("""a:\n b: 2\n c: 3\n""") + from ruamel.yaml import YAML + + yaml=YAML(typ="safe", pure=True) + yaml.load("""a:\n b: 2\n c: 3\n""") diff --git a/_doc/basicuse.ryd b/_doc/basicuse.ryd new file mode 100644 index 0000000..7d3ae96 --- /dev/null +++ b/_doc/basicuse.ryd @@ -0,0 +1,78 @@ +--- +version: 0.1 +output: rst +fix_inline_single_backquotes: true +pdf: true +--- !python-pre | +import sys +from io import StringIO +from ruamel.yaml import YAML +yaml=YAML() +s = StringIO() +doc = "a: 1" +data = dict(a=1) +--- | +Basic Usage +=========== + +*This is the new (0.15+) interface for ``ruamel.yaml``, it is still in +the process of being fleshed out*. Please pin your dependency to +``ruamel.yaml<0.15`` for production software. + +------ + +You load a YAML document using:: +--- !python | +from ruamel.yaml import YAML + +yaml=YAML(typ='safe') # default if not specfied is round-trip +yaml.load(doc) + +--- | +in this ``doc`` can be a file pointer (i.e. an object that has the +`.read()` method, a string or a ``pathlib.Path()``. `typ='safe'` +accomplishes the same as what ``safe_load()`` did before: loading of a +document without resolving unknow tags. Provide `pure=True` to +enforce using the pure Python implementation (faster C libraries will be used +when possible/available) + +Dumping works in the same way:: +--- !python | +from ruamel.yaml import YAML + +yaml=YAML() +yaml.default_flow_style = False +yaml.dump({'a': [1, 2]}, s) +--- | +in this ``s`` can be a file pointer (i.e. an object that has the +`.write()` method, or a ``pathlib.Path()``. If you want to display +your output, just stream to `sys.stdout`. + +If you need to transform a string representation of the output provide +a function that takes a string as input and returns one:: + +--- !python | +def tr(s): + return s.replace('\n', '<\n') # such output is not valid YAML! + +yaml.dump(data, sys.stdout, transform=tr) + +--- | +More examples +------------- +Using the C based SafeLoader (at this time is inherited from +libyaml/PyYAML and e.g. loads ``0o52`` as well as ``052`` load as integer ``42``):: + +--- !python | + from ruamel.yaml import YAML + + yaml=YAML(typ="safe") + yaml.load("""a:\n b: 2\n c: 3\n""") + +--- | +Using the Python based SafeLoader (YAML 1.2 support, ``052`` loads as ``52``):: +--- !python | + from ruamel.yaml import YAML + + yaml=YAML(typ="safe", pure=True) + yaml.load("""a:\n b: 2\n c: 3\n""")
\ No newline at end of file diff --git a/_doc/dumpcls.rst b/_doc/dumpcls.rst new file mode 100644 index 0000000..0195689 --- /dev/null +++ b/_doc/dumpcls.rst @@ -0,0 +1,97 @@ +Dumping Python classes +====================== + +Only ``yaml = YAML(typ='unsafe')`` loads and dumps Python objects out-of-the-box. And +since it loads **any** Python object, this can be unsafe. + +If you have instances of some class(es) that you want to dump or load, it is +easy to allow the YAML instance to that explicitly. You can either registering the +class with the ``YAML`` instance or decorate the class. + +Registering is done with ``YAML.register_class()``:: + + import sys + import ruamel.yaml + + + class User(object): + def __init__(self, name, age): + self.name = name + self.age = age + + + yaml = ruamel.yaml.YAML() + yaml.register_class(User) + yaml.dump([User('Anthon', 18)], sys.stdout) + +which gives as output:: + + - !User + name: Anthon + age: 18 + +The tag ``!User`` originates from the name of the class. + +You can specify a different tag by adding the attribute `yaml_tag`, and explicitly specify dump and/or load *classmethods* which have to be called ``from_yaml`` resp. ``from_yaml``:: + + import sys + import ruamel.yaml + + + class User: + yaml_tag = u'!user' + + def __init__(self, name, age): + self.name = name + self.age = age + + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_scalar(cls.yaml_tag, + u'{.name}-{.age}'.format(node, node)) + + @classmethod + def from_yaml(cls, constructor, node): + return cls(*node.value.split('-')) + + + yaml = ruamel.yaml.YAML() + yaml.register_class(User) + yaml.dump([User('Anthon', 18)], sys.stdout) + +which gives as output:: + + - !user Anthon-18 + +When using the decorator, which takes the ``YAML()`` instance as a parameter, +the ``yaml = YAML()`` line needs to be moved up in the file:: + + import sys + from ruamel.yaml import YAML, yaml_object + + yaml = YAML() + + + @yaml_object(yaml) + class User: + yaml_tag = u'!user' + + def __init__(self, name, age): + self.name = name + self.age = age + + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_scalar(cls.yaml_tag, + u'{.name}-{.age}'.format(node, node)) + + @classmethod + def from_yaml(cls, constructor, node): + return cls(*node.value.split('-')) + + + yaml.dump([User('Anthon', 18)], sys.stdout) + +The ``yaml_tag``, ``from_yaml`` and ``to_yaml`` work in the same way as when using +``.register_class()``. + diff --git a/_doc/dumpcls.ryd b/_doc/dumpcls.ryd new file mode 100644 index 0000000..489d1b7 --- /dev/null +++ b/_doc/dumpcls.ryd @@ -0,0 +1,106 @@ +--- +version: 0.1 +output: rst +fix_inline_single_backquotes: true +pdf: true +# code_directory: ../_example +--- | + +Dumping Python classes +====================== + +Only ``yaml = YAML(typ='unsafe')`` loads and dumps Python objects out-of-the-box. And +since it loads **any** Python object, this can be unsafe. + +If you have instances of some class(es) that you want to dump or load, it is +easy to allow the YAML instance to that explicitly. You can either registering the +class with the ``YAML`` instance or decorate the class. + +Registering is done with ``YAML.register_class()``:: + +--- !python | + +import sys +import ruamel.yaml + + +class User(object): + def __init__(self, name, age): + self.name = name + self.age = age + + +yaml = ruamel.yaml.YAML() +yaml.register_class(User) +yaml.dump([User('Anthon', 18)], sys.stdout) +--- !stdout | +which gives as output:: + +--- | +The tag ``!User`` originates from the name of the class. + +You can specify a different tag by adding the attribute `yaml_tag`, and explicitly specify dump and/or load *classmethods* which have to be called ``from_yaml`` resp. ``from_yaml``:: + +--- !python | +import sys +import ruamel.yaml + + +class User: + yaml_tag = u'!user' + + def __init__(self, name, age): + self.name = name + self.age = age + + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_scalar(cls.yaml_tag, + u'{.name}-{.age}'.format(node, node)) + + @classmethod + def from_yaml(cls, constructor, node): + return cls(*node.value.split('-')) + + +yaml = ruamel.yaml.YAML() +yaml.register_class(User) +yaml.dump([User('Anthon', 18)], sys.stdout) +--- !stdout | +which gives as output:: + +--- | + +When using the decorator, which takes the ``YAML()`` instance as a parameter, +the ``yaml = YAML()`` line needs to be moved up in the file:: + +--- !python | +import sys +from ruamel.yaml import YAML, yaml_object + +yaml = YAML() + + +@yaml_object(yaml) +class User: + yaml_tag = u'!user' + + def __init__(self, name, age): + self.name = name + self.age = age + + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_scalar(cls.yaml_tag, + u'{.name}-{.age}'.format(node, node)) + + @classmethod + def from_yaml(cls, constructor, node): + return cls(*node.value.split('-')) + + +yaml.dump([User('Anthon', 18)], sys.stdout) + +--- | +The ``yaml_tag``, ``from_yaml`` and ``to_yaml`` work in the same way as when using +``.register_class()``. diff --git a/_doc/index.rst b/_doc/index.rst index 28e3cdc..c9f9659 100644 --- a/_doc/index.rst +++ b/_doc/index.rst @@ -14,6 +14,7 @@ Contents: overview install basicuse + dumpcls detail example api diff --git a/_test/roundtrip.py b/_test/roundtrip.py index f226e38..53bc667 100644 --- a/_test/roundtrip.py +++ b/_test/roundtrip.py @@ -102,16 +102,28 @@ class YAML(ruamel.yaml.YAML): stream = textwrap.dedent(stream) return ruamel.yaml.YAML.load(self, stream) + def load_all(self, stream): + if isinstance(stream, str): + if stream and stream[0] == '\n': + stream = stream[1:] + stream = textwrap.dedent(stream) + for d in ruamel.yaml.YAML.load_all(self, stream): + yield d + def dump(self, data, **kw): assert ('stream' in kw) ^ ('compare' in kw) if 'stream' in kw: return ruamel.yaml.YAML.dump(data, **kw) lkw = kw.copy() expected = textwrap.dedent(lkw.pop('compare')) + unordered_lines = lkw.pop('unordered_lines', False) if expected and expected[0] == '\n': expected = expected[1:] lkw['stream'] = st = StringIO() if self.encoding is None else BytesIO() ruamel.yaml.YAML.dump(self, data, **lkw) res = st.getvalue() print(res) + if unordered_lines: + res = sorted(res.splitlines()) + expected = sorted(expected.splitlines()) assert res == expected diff --git a/_test/test_class_register.py b/_test/test_class_register.py new file mode 100644 index 0000000..cb3d6f0 --- /dev/null +++ b/_test/test_class_register.py @@ -0,0 +1,137 @@ +# coding: utf-8 + +""" +testing of YAML.register_class and @yaml_object +""" + +from roundtrip import YAML +from ruamel.yaml import yaml_object + + +class User0(object): + def __init__(self, name, age): + self.name = name + self.age = age + + +class User1(object): + yaml_tag = u'!user' + + def __init__(self, name, age): + self.name = name + self.age = age + + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_scalar(cls.yaml_tag, + u'{.name}-{.age}'.format(node, node)) + + @classmethod + def from_yaml(cls, constructor, node): + return cls(*node.value.split('-')) + + +class TestRegisterClass(object): + def test_register_0_rt(self): + yaml = YAML() + yaml.register_class(User0) + ys = ''' + - !User0 + name: Anthon + age: 18 + ''' + d = yaml.load(ys) + yaml.dump(d, compare=ys, unordered_lines=True) + + def test_register_0_safe(self): + # default_flow_style = None + yaml = YAML(typ="safe") + yaml.register_class(User0) + ys = ''' + - !User0 {age: 18, name: Anthon} + ''' + d = yaml.load(ys) + yaml.dump(d, compare=ys) + + def test_register_0_unsafe(self): + # default_flow_style = None + yaml = YAML(typ="unsafe") + yaml.register_class(User0) + ys = ''' + - !User0 {age: 18, name: Anthon} + ''' + d = yaml.load(ys) + yaml.dump(d, compare=ys) + + def test_register_1_rt(self): + yaml = YAML() + yaml.register_class(User1) + ys = ''' + - !user Anthon-18 + ''' + d = yaml.load(ys) + yaml.dump(d, compare=ys) + + def test_register_1_safe(self): + yaml = YAML(typ="safe") + yaml.register_class(User1) + ys = ''' + [!user Anthon-18] + ''' + d = yaml.load(ys) + yaml.dump(d, compare=ys) + + def test_register_1_unsafe(self): + yaml = YAML(typ="unsafe") + yaml.register_class(User1) + ys = ''' + [!user Anthon-18] + ''' + d = yaml.load(ys) + yaml.dump(d, compare=ys) + + +yml = YAML() + + +@yaml_object(yml) +class User2(object): + def __init__(self, name, age): + self.name = name + self.age = age + + +@yaml_object(yml) +class User3(object): + yaml_tag = u'!USER' + + def __init__(self, name, age): + self.name = name + self.age = age + + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_scalar(cls.yaml_tag, + u'{.name}-{.age}'.format(node, node)) + + @classmethod + def from_yaml(cls, constructor, node): + return cls(*node.value.split('-')) + + +class TestDecorator(object): + def test_decorator_implicit(self): + ys = ''' + - !User2 + name: Anthon + age: 18 + ''' + d = yml.load(ys) + yml.dump(d, compare=ys, unordered_lines=True) + + def test_decorator_explicit(self): + ys = ''' + - !USER Anthon-18 + ''' + d = yml.load(ys) + yml.dump(d, compare=ys) diff --git a/_test/test_literal.py b/_test/test_literal.py index 1bb8367..0499a16 100644 --- a/_test/test_literal.py +++ b/_test/test_literal.py @@ -154,6 +154,19 @@ class TestNoIndent: print(d) assert d == s + '\n' + def test_top_literal_multi_doc(self): + yaml = YAML(typ='safe', pure=True) + s1 = 'abc' + s2 = 'klm' + for idx, d1 in enumerate(yaml.load_all(""" + --- |- + {} + --- | + {} + """.format(s1, s2))): + print('d1:', d1) + assert ['abc', 'klm\n'][idx] == d1 + class Test_RoundTripLiteral: def test_rt_top_literal_scalar_no_indent(self): diff --git a/constructor.py b/constructor.py index fd1926e..c957cd0 100644 --- a/constructor.py +++ b/constructor.py @@ -1278,6 +1278,17 @@ class RoundTripConstructor(SafeConstructor): yield data self.construct_mapping(node, data) + def construct_yaml_object(self, node, cls): + # type: (Any, Any) -> Any + data = cls.__new__(cls) + yield data + if hasattr(data, '__setstate__'): + state = SafeConstructor.construct_mapping(self, node, deep=True) + data.__setstate__(state) + else: + state = SafeConstructor.construct_mapping(self, node) + data.__dict__.update(state) + def construct_yaml_omap(self, node): # type: (Any) -> Any # Note: we do now check for duplicate keys @@ -269,7 +269,9 @@ class YAML(object): if not hasattr(stream, 'read') and hasattr(stream, 'open'): # pathlib.Path() instance with stream.open('r') as fp: # type: ignore - yield self.load_all(fp, _kw=enforce) + for d in self.load_all(fp, _kw=enforce): + yield d + raise StopIteration() # if skip is None: # skip = [] # elif isinstance(skip, int): @@ -452,9 +454,61 @@ class YAML(object): res = [x.replace(gpbd, '')[1:-3] for x in glob.glob(bd + '/*/__plug_in__.py')] return res + def register_class(self, cls): + """ + register a class for dumping loading + - if it has attribute yaml_tag use that to register, else use class name + - if it has methods to_yaml/from_yaml use those to dump/load else dump attributes + as mapping + """ + tag = getattr(cls, 'yaml_tag', '!' + cls.__name__) + try: + self.representer.add_representer(cls, cls.to_yaml) + except AttributeError: + def t_y(representer, data): + return representer.represent_yaml_object( + tag, data, cls, flow_style=representer.default_flow_style) + + self.representer.add_representer(cls, t_y) + try: + self.constructor.add_constructor(tag, cls.from_yaml) + except AttributeError: + def f_y(constructor, node): + return constructor.construct_yaml_object(node, cls) + + self.constructor.add_constructor(tag, f_y) + + +def yaml_object(yml): + """ decorator for classes that needs to dump/load objects + The tag for such objects is taken from the class attribute yaml_tag (or the + class name in lowercase in case unavailable) + If methods to_yaml and/or from_yaml are available, these are called for dumping resp. + loading, default routines (dumping a mapping of the attributes) used otherwise. + """ + def yo_deco(cls): + tag = getattr(cls, 'yaml_tag', '!' + cls.__name__) + try: + yml.representer.add_representer(cls, cls.to_yaml) + except AttributeError: + def t_y(representer, data): + return representer.represent_yaml_object( + tag, data, cls, flow_style=representer.default_flow_style) + + yml.representer.add_representer(cls, t_y) + try: + yml.constructor.add_constructor(tag, cls.from_yaml) + except AttributeError: + def f_y(constructor, node): + return constructor.construct_yaml_object(node, cls) + + yml.constructor.add_constructor(tag, f_y) + return cls + return yo_deco ######################################################################################## + def scan(stream, Loader=Loader): # type: (StreamTextType, Any) -> Any """ @@ -41,6 +41,10 @@ if False: # MYPY __all__ = ['Scanner', 'RoundTripScanner', 'ScannerError'] +_THE_END = u'\0\r\n\x85\u2028\u2029' +_THE_END_SPACE_TAB = u'\0 \t\r\n\x85\u2028\u2029' + + class ScannerError(MarkedYAMLError): pass @@ -726,7 +730,7 @@ class Scanner(object): # DOCUMENT-START: ^ '---' (' '|'\n') if self.reader.column == 0: if self.reader.prefix(3) == u'---' \ - and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + and self.reader.peek(3) in _THE_END_SPACE_TAB: return True return None @@ -735,14 +739,14 @@ class Scanner(object): # DOCUMENT-END: ^ '...' (' '|'\n') if self.reader.column == 0: if self.reader.prefix(3) == u'...' \ - and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + and self.reader.peek(3) in _THE_END_SPACE_TAB: return True return None def check_block_entry(self): # type: () -> Any # BLOCK-ENTRY: '-' (' '|'\n') - return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029' + return self.reader.peek(1) in _THE_END_SPACE_TAB def check_key(self): # type: () -> Any @@ -750,7 +754,7 @@ class Scanner(object): if bool(self.flow_level): return True # KEY(block context): '?' (' '|'\n') - return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029' + return self.reader.peek(1) in _THE_END_SPACE_TAB def check_value(self): # type: () -> Any @@ -758,7 +762,7 @@ class Scanner(object): if bool(self.flow_level): return True # VALUE(block context): ':' (' '|'\n') - return self.reader.peek(1) in u'\0 \t\r\n\x85\u2028\u2029' + return self.reader.peek(1) in _THE_END_SPACE_TAB def check_plain(self): # type: () -> Any @@ -776,7 +780,7 @@ class Scanner(object): # independent. ch = self.reader.peek() return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' or \ - (self.reader.peek(1) not in u'\0 \t\r\n\x85\u2028\u2029' and + (self.reader.peek(1) not in _THE_END_SPACE_TAB and (ch == u'-' or (not self.flow_level and ch in u'?:'))) # Scanners. @@ -809,7 +813,7 @@ class Scanner(object): while self.reader.peek() == u' ': self.reader.forward() if self.reader.peek() == u'#': - while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029': + while self.reader.peek() not in _THE_END: self.reader.forward() if self.scan_line_break(): if not self.flow_level: @@ -833,7 +837,7 @@ class Scanner(object): end_mark = self.reader.get_mark() else: end_mark = self.reader.get_mark() - while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029': + while self.reader.peek() not in _THE_END: self.reader.forward() self.scan_directive_ignored_line(start_mark) return DirectiveToken(name, value, start_mark, end_mark) @@ -939,10 +943,10 @@ class Scanner(object): while self.reader.peek() == u' ': self.reader.forward() if self.reader.peek() == u'#': - while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029': + while self.reader.peek() not in _THE_END: self.reader.forward() ch = self.reader.peek() - if ch not in u'\0\r\n\x85\u2028\u2029': + if ch not in _THE_END: raise ScannerError( "while scanning a directive", start_mark, "expected a comment or a line break, but found %r" @@ -1006,7 +1010,7 @@ class Scanner(object): "expected '>', but found %r" % utf8(self.reader.peek()), self.reader.get_mark()) self.reader.forward() - elif ch in u'\0 \t\r\n\x85\u2028\u2029': + elif ch in _THE_END_SPACE_TAB: handle = None suffix = u'!' self.reader.forward() @@ -1075,12 +1079,17 @@ class Scanner(object): chunks.extend(breaks) leading_non_space = self.reader.peek() not in u' \t' length = 0 - while self.reader.peek(length) not in u'\0\r\n\x85\u2028\u2029': + while self.reader.peek(length) not in _THE_END: length += 1 chunks.append(self.reader.prefix(length)) self.reader.forward(length) line_break = self.scan_line_break() breaks, end_mark = self.scan_block_scalar_breaks(indent) + if style in '|>' and min_indent == 0: + # at the beginning of a line, if in block style see if + # end of document/start_new_document + if self.check_document_start() or self.check_document_end(): + break if self.reader.column == indent and self.reader.peek() != u'\0': # Unfortunately, folding rules are ambiguous. @@ -1187,10 +1196,10 @@ class Scanner(object): while self.reader.peek() == u' ': self.reader.forward() if self.reader.peek() == u'#': - while self.reader.peek() not in u'\0\r\n\x85\u2028\u2029': + while self.reader.peek() not in _THE_END: self.reader.forward() ch = self.reader.peek() - if ch not in u'\0\r\n\x85\u2028\u2029': + if ch not in _THE_END: raise ScannerError( "while scanning a block scalar", start_mark, "expected a comment or a line break, but found %r" @@ -1364,7 +1373,7 @@ class Scanner(object): # separators. prefix = self.reader.prefix(3) if (prefix == u'---' or prefix == u'...') \ - and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + and self.reader.peek(3) in _THE_END_SPACE_TAB: raise ScannerError("while scanning a quoted scalar", start_mark, "found unexpected document separator", @@ -1399,11 +1408,11 @@ class Scanner(object): while True: ch = self.reader.peek(length) if (ch == u':' and - self.reader.peek(length + 1) not in u'\0 \t\r\n\x85\u2028\u2029'): + self.reader.peek(length + 1) not in _THE_END_SPACE_TAB): pass - elif (ch in u'\0 \t\r\n\x85\u2028\u2029' or + elif (ch in _THE_END_SPACE_TAB or (not self.flow_level and ch == u':' and - self.reader.peek(length + 1) in u'\0 \t\r\n\x85\u2028\u2029') or + self.reader.peek(length + 1) in _THE_END_SPACE_TAB) or (self.flow_level and ch in u',:?[]{}')): break length += 1 @@ -1453,7 +1462,7 @@ class Scanner(object): self.allow_simple_key = True prefix = self.reader.prefix(3) if (prefix == u'---' or prefix == u'...') \ - and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + and self.reader.peek(3) in _THE_END_SPACE_TAB: return breaks = [] while self.reader.peek() in u' \r\n\x85\u2028\u2029': @@ -1463,7 +1472,7 @@ class Scanner(object): breaks.append(self.scan_line_break()) prefix = self.reader.prefix(3) if (prefix == u'---' or prefix == u'...') \ - and self.reader.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + and self.reader.peek(3) in _THE_END_SPACE_TAB: return if line_break != u'\n': chunks.append(line_break) @@ -1700,7 +1709,7 @@ class RoundTripScanner(Scanner): start_mark = self.reader.get_mark() comment = ch self.reader.forward() - while ch not in u'\0\r\n\x85\u2028\u2029': + while ch not in _THE_END: ch = self.reader.peek() if ch == u'\0': # don't gobble the end-of-stream character break @@ -169,6 +169,7 @@ def _package_data(fn): raise NotImplementedError return data + # make sure you can run "python ../some/dir/setup.py install" pkg_data = _package_data(__file__.replace('setup.py', '__init__.py')) @@ -241,7 +242,7 @@ class MySdist(_sdist): # because of unicode_literals # self.formats = fmt if fmt else [b'bztar'] if sys.version_info < (3, ) else ['bztar'] dist_base = os.environ.get('PYDISTBASE') - fpn = getattr(getattr(self, 'nsp', self), 'full_package_name', None) + fpn = getattr(getattr(self, 'nsp', self), 'full_package_name', None) if fpn and dist_base: print('setting distdir {}/{}'.format(dist_base, fpn)) self.dist_dir = os.path.join(dist_base, fpn) @@ -256,7 +257,7 @@ try: def initialize_options(self): _bdist_wheel.initialize_options(self) dist_base = os.environ.get('PYDISTBASE') - fpn = getattr(getattr(self, 'nsp', self), 'full_package_name', None) + fpn = getattr(getattr(self, 'nsp', self), 'full_package_name', None) if fpn and dist_base: print('setting distdir {}/{}'.format(dist_base, fpn)) self.dist_dir = os.path.join(dist_base, fpn) @@ -356,7 +357,7 @@ class NameSpacePackager(object): sys.exit(1) # If you only support an extension module on Linux, Windows thinks it # is pure. That way you would get pure python .whl files that take - # precedence for downloading on Linux over source with compilable C + # precedence for downloading on Linux over source with compilable C code if self._pkg_data.get('universal'): Distribution.is_pure = lambda *args: True else: @@ -917,4 +918,5 @@ def main(): imz.delete_from_zip_file(nsp.full_package_name + '.*.pth') break + main() @@ -6,12 +6,12 @@ commands = /bin/bash -c 'pytest _test/test_*.py' deps = pytest - flake8==2.5.5 + flake8==3.3.0 ruamel.std.pathlib [testenv:pep8] commands = - flake8 --exclude jabsy,jinja2,base,cmd,convert{posargs} + flake8 --exclude .tox,jabsy,jinja2,base,cmd,convert{posargs} [flake8] show-source = True |