diff options
| -rw-r--r-- | README.rst | 191 | ||||
| -rw-r--r-- | __init__.py | 2 | ||||
| -rw-r--r-- | _test/test_comment_manipulation.py | 134 | ||||
| -rw-r--r-- | comments.py | 32 | ||||
| -rw-r--r-- | error.py | 20 | ||||
| -rw-r--r-- | main.py | 14 | ||||
| -rw-r--r-- | tox.ini | 2 |
7 files changed, 276 insertions, 119 deletions
@@ -16,103 +16,116 @@ ruamel.yaml ChangeLog ========= -:: +.. should insert 0.13.0 (2016-11-20): for next key - 0.12.18 (2016-11-16): - - another fix for numpy (re-reported by Nathanial Burdic) - 0.12.17 (2016-11-15): - - only the RoundTripLoader included the Resolver that supports YAML 1.2 - now all loaders do (reported by mixmastamyk) +NEXT: + - if load() or load_all() is called with only a single argument (stream or string) + a UnsafeLoaderWarning will be issued once. If appropriate you can surpress this + warning by filtering it. Explicitly supplying the ``Loader=ruamel.yaml.Loader`` + argument, will also prevent it from being issued. You should however consider + using ``safe_load()``, ``safe_load_all()`` if your YAML input does not use tags. + - allow adding comments before and after keys (based on + `StackOveflow Q&A <http://stackoverflow.com/a/40705671/1307905>`_ by + `msinn <http://stackoverflow.com/users/7185467/msinn>`_) - 0.12.16 (2016-11-13): - - allow dot char (and many others) in anchor name - Fix issue 72 (reported by Shalon Wood) - - Slightly smarter behaviour dumping strings when no style is - specified. Single string scalars that start with single quotes - or have newlines now are dumped double quoted: "'abc\nklm'" instead of - '''abc +0.12.18 (2016-11-16): + - another fix for numpy (re-reported by Nathanial Burdic) + +0.12.17 (2016-11-15): + - only the RoundTripLoader included the Resolver that supports YAML 1.2 + now all loaders do (reported by mixmastamyk) + +0.12.16 (2016-11-13): + - allow dot char (and many others) in anchor name + Fix issue 72 (reported by Shalon Wood) + - Slightly smarter behaviour dumping strings when no style is + specified. Single string scalars that start with single quotes + or have newlines now are dumped double quoted: "'abc\nklm'" instead of:: + + '''abc klm''' - 0.12.14 (2016-09-21): - - preserve round-trip sequences that are mapping keys - (prompted by stackoverflow question 39595807 from Nowox) - - 0.12.13 (2016-09-15): - - Fix for issue #60 representation of CommentedMap with merge - keys incorrect (reported by Tal Liron) - - 0.12.11 (2016-09-06): - - Fix issue 58 endless loop in scanning tokens (reported by - Christopher Lambert) - - 0.12.10 (2016-09-05): - - Make previous fix depend on unicode char width (32 bit unicode support - is a problem on MacOS reported by David Tagatac) - - 0.12.8 (2016-09-05): - - To be ignored Unicode characters were not properly regex matched - (no specific tests, PR by Haraguroicha Hsu) - - 0.12.7 (2016-09-03): - - fixing issue 54 empty lines with spaces (reported by Alex Harvey) - - 0.12.6 (2016-09-03): - - fixing issue 46 empty lines between top-level keys were gobbled (but - not between sequence elements, nor between keys in netsted mappings - (reported by Alex Harvey) - - 0.12.5 (2016-08-20): - - fixing issue 45 preserving datetime formatting (submitted by altuin) - Several formatting parameters are preserved with some normalisation: - - preserve 'T', 't' is replaced by 'T', multiple spaces between date - and time reduced to one. - - optional space before timezone is removed - - still using microseconds, but now rounded (.1234567 -> .123457) - - Z/-5/+01:00 preserved - - 0.12.4 (2016-08-19): - - Fix for issue 44: missing preserve_quotes keyword argument (reported - by M. Crusoe) - - 0.12.3 (2016-08-17): - - correct 'in' operation for merged CommentedMaps in round-trip mode - (implementation inspired by J.Ngo, but original not working for merges) - - iteration over round-trip loaded mappings, that contain merges. Also - keys(), items(), values() (Py3/Py2) and iterkeys(), iteritems(), - itervalues(), viewkeys(), viewitems(), viewvalues() (Py2) - - reuse of anchor name now generates warning, not an error. Round-tripping such - anchors works correctly. This inherited PyYAML issue was brought to attention - by G. Coddut (and was long standing https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=515634) - suppressing the warning: - import warnings - from ruamel.yaml.error import ReusedAnchorWarning - warnings.simplefilter("ignore", ReusedAnchorWarning) - - 0.12.2 (2016-08-16): - - minor improvements based on feedback from M. Crusoe - https://bitbucket.org/ruamel/yaml/issues/42/ - - 0.12.0 (2016-08-16): - - drop support for Python 2.6 - - include initial Type information (inspired by M. Crusoe) - - 0.11.15 (2016-08-07): - - Change to prevent FutureWarning in NumPy, as reported by tgehring +0.12.14 (2016-09-21): + - preserve round-trip sequences that are mapping keys + (prompted by stackoverflow question 39595807 from Nowox) + +0.12.13 (2016-09-15): + - Fix for issue #60 representation of CommentedMap with merge + keys incorrect (reported by Tal Liron) + +0.12.11 (2016-09-06): + - Fix issue 58 endless loop in scanning tokens (reported by + Christopher Lambert) + +0.12.10 (2016-09-05): + - Make previous fix depend on unicode char width (32 bit unicode support + is a problem on MacOS reported by David Tagatac) + +0.12.8 (2016-09-05): + - To be ignored Unicode characters were not properly regex matched + (no specific tests, PR by Haraguroicha Hsu) + +0.12.7 (2016-09-03): + - fixing issue 54 empty lines with spaces (reported by Alex Harvey) + +0.12.6 (2016-09-03): + - fixing issue 46 empty lines between top-level keys were gobbled (but + not between sequence elements, nor between keys in netsted mappings + (reported by Alex Harvey) + +0.12.5 (2016-08-20): + - fixing issue 45 preserving datetime formatting (submitted by altuin) + Several formatting parameters are preserved with some normalisation: + - preserve 'T', 't' is replaced by 'T', multiple spaces between date + and time reduced to one. + - optional space before timezone is removed + - still using microseconds, but now rounded (.1234567 -> .123457) + - Z/-5/+01:00 preserved + +0.12.4 (2016-08-19): + - Fix for issue 44: missing preserve_quotes keyword argument (reported + by M. Crusoe) + +0.12.3 (2016-08-17): + - correct 'in' operation for merged CommentedMaps in round-trip mode + (implementation inspired by J.Ngo, but original not working for merges) + - iteration over round-trip loaded mappings, that contain merges. Also + keys(), items(), values() (Py3/Py2) and iterkeys(), iteritems(), + itervalues(), viewkeys(), viewitems(), viewvalues() (Py2) + - reuse of anchor name now generates warning, not an error. Round-tripping such + anchors works correctly. This inherited PyYAML issue was brought to attention + by G. Coddut (and was long standing https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=515634) + suppressing the warning:: + + import warnings + from ruamel.yaml.error import ReusedAnchorWarning + warnings.simplefilter("ignore", ReusedAnchorWarning) + +0.12.2 (2016-08-16): + - minor improvements based on feedback from M. Crusoe + https://bitbucket.org/ruamel/yaml/issues/42/ + +0.12.0 (2016-08-16): + - drop support for Python 2.6 + - include initial Type information (inspired by M. Crusoe) + +0.11.15 (2016-08-07): + - Change to prevent FutureWarning in NumPy, as reported by tgehring ("comparison to None will result in an elementwise object comparison in the future") - 0.11.14 (2016-07-06): - - fix preserve_quotes missing on original Loaders (as reported - by Leynos, bitbucket issue 38) +0.11.14 (2016-07-06): + - fix preserve_quotes missing on original Loaders (as reported + by Leynos, bitbucket issue 38) - 0.11.13 (2016-07-06): - - documentation only, automated linux wheels +0.11.13 (2016-07-06): + - documentation only, automated linux wheels - 0.11.12 (2016-07-06): - - added support for roundtrip of single/double quoted scalars using: - ruamel.yaml.round_trip_load(stream, preserve_quotes=True) +0.11.12 (2016-07-06): + - added support for roundtrip of single/double quoted scalars using: + ruamel.yaml.round_trip_load(stream, preserve_quotes=True) - 0.11.0 (2016-02-18): - - RoundTripLoader loads 1.2 by default (no sexagesimals, 012 octals nor - yes/no/on/off booleans +0.11.0 (2016-02-18): + - RoundTripLoader loads 1.2 by default (no sexagesimals, 012 octals nor + yes/no/on/off booleans diff --git a/__init__.py b/__init__.py index bf976aa..adfd7f8 100644 --- a/__init__.py +++ b/__init__.py @@ -9,7 +9,7 @@ from __future__ import absolute_import _package_data = dict( full_package_name="ruamel.yaml", - version_info=(0, 12, 18), + version_info=(0, 13, 0), author="Anthon van der Neut", author_email="a.van.der.neut@ruamel.eu", description="ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order", # NOQA diff --git a/_test/test_comment_manipulation.py b/_test/test_comment_manipulation.py index 306c365..dfb6ff6 100644 --- a/_test/test_comment_manipulation.py +++ b/_test/test_comment_manipulation.py @@ -13,6 +13,11 @@ def load(s): def compare(data, s): assert round_trip_dump(data) == dedent(s) + + +def compare_eol(data, s): + assert round_trip_dump(data).replace('\n', '|\n') == \ + dedent(s).replace('EOL', '').replace('\n', '|\n') # @pytest.mark.xfail @@ -224,10 +229,6 @@ class TestCommentsManipulation: e: 5 # comment 3 """) -# the ugly {comment} in the following is because -# py.test cannot handle comments in strings properly -# https://bitbucket.org/pytest-dev/pytest/issue/752/internalerror-indexerror-list-index-out-of - # @pytest.mark.xfail def test_before_top_map_rt(self): data = load(""" @@ -236,8 +237,8 @@ class TestCommentsManipulation: """) data.yaml_set_start_comment('Hello\nWorld\n') compare(data, """ - {comment} Hello - {comment} World + # Hello + # World a: 1 b: 2 """.format(comment='#')) @@ -251,8 +252,8 @@ class TestCommentsManipulation: """) data.yaml_set_start_comment('Hello\nWorld\n') compare(data, """ - {comment} Hello - {comment} World + # Hello + # World a: 1 # 1 b: 2 """.format(comment='#')) @@ -266,8 +267,8 @@ class TestCommentsManipulation: # print(data.ca) # print(data.ca._items) compare(data, """ - {comment} Hello - {comment} World + # Hello + # World a: 1 b: 2 """.format(comment='#')) @@ -288,16 +289,16 @@ class TestCommentsManipulation: def test_before_top_seq_rt_replace(self): data = load(""" - {comment} this - {comment} that + # this + # that - a - b """.format(comment='#')) data.yaml_set_start_comment('Hello\nWorld\n') print(round_trip_dump(data)) compare(data, """ - {comment} Hello - {comment} World + # Hello + # World - a - b """.format(comment='#')) @@ -310,8 +311,8 @@ class TestCommentsManipulation: data.yaml_set_start_comment('Hello\nWorld\n') print(round_trip_dump(data)) compare(data, """ - {comment} Hello - {comment} World + # Hello + # World - a - b """.format(comment='#')) @@ -328,8 +329,8 @@ class TestCommentsManipulation: compare(data, """ a: 1 b: - {comment} Hello - {comment} World + # Hello + # World c: 2 d: 3 """.format(comment='#')) @@ -345,8 +346,8 @@ class TestCommentsManipulation: compare(data, """ a: 1 b: - {comment} Hello - {comment} World + # Hello + # World c: 2 d: 3 """.format(comment='#')) @@ -364,8 +365,8 @@ class TestCommentsManipulation: compare(data, """ a: 1 b: - {comment} Hello - {comment} World + # Hello + # World c: 2 d: 3 """.format(comment='#')) @@ -382,8 +383,93 @@ class TestCommentsManipulation: compare(data, """ a: 1 b: - {comment} Hello - {comment} World + # Hello + # World - c - d """.format(comment='#')) + + def test_map_set_comment_before_and_after_non_first_key_00(self): + # http://stackoverflow.com/a/40705671/1307905 + data = load(""" + xyz: + a: 1 # comment 1 + b: 2 + + test1: + test2: + test3: 3 + """) + data.yaml_set_comment_before_after_key('test1', 'before test1 (top level)', + after='before test2') + data['test1']['test2'].yaml_set_start_comment('after test2', indent=4) + compare(data, """ + xyz: + a: 1 # comment 1 + b: 2 + + # before test1 (top level) + test1: + # before test2 + test2: + # after test2 + test3: 3 + """) + + def test_map_set_comment_before_and_after_non_first_key_01(self): + data = load(""" + xyz: + a: 1 # comment 1 + b: 2 + + test1: + test2: + test3: 3 + """) + data.yaml_set_comment_before_after_key('test1', 'before test1 (top level)', + after='before test2\n\n') + data['test1']['test2'].yaml_set_start_comment('after test2', indent=4) + # EOL is needed here as dedenting gets rid of spaces (as well as does Emacs + compare_eol(data, """ + xyz: + a: 1 # comment 1 + b: 2 + + # before test1 (top level) + test1: + # before test2 + EOL + test2: + # after test2 + test3: 3 + """) + + def test_map_set_comment_before_and_after_non_first_key_02(self): + data = load(""" + xyz: + a: 1 # comment 1 + b: 2 + + test1: + test2: + test3: 3 + """) + data.yaml_set_comment_before_after_key('test1', 'xyz\n\nbefore test1 (top level)', + after='\nbefore test2', after_indent=4) + data['test1']['test2'].yaml_set_start_comment('after test2', indent=4) + # EOL is needed here as dedenting gets rid of spaces (as well as does Emacs + compare_eol(data, """ + xyz: + a: 1 # comment 1 + b: 2 + + # xyz + + # before test1 (top level) + test1: + EOL + # before test2 + test2: + # after test2 + test3: 3 + """) diff --git a/comments.py b/comments.py index 0091c78..25f0f8a 100644 --- a/comments.py +++ b/comments.py @@ -180,7 +180,7 @@ class CommentedBase(object): def yaml_set_start_comment(self, comment, indent=0): """overwrites any preceding comment lines on an object - expects comment to be without `#` and possible have mutlple lines + expects comment to be without `#` and possible have multiple lines """ from .error import Mark from .tokens import CommentToken @@ -191,6 +191,36 @@ class CommentedBase(object): for com in comment.split('\n'): pre_comments.append(CommentToken('# ' + com + '\n', start_mark, None)) + def yaml_set_comment_before_after_key(self, key, before=None, indent=0, + after=None, after_indent=None): + """ + expects comment (before/after) to be without `#` and possible have multiple lines + """ + from ruamel.yaml.error import Mark + from ruamel.yaml.tokens import CommentToken + + def comment_token(s, mark): + # handle empty lines as having no comment + return CommentToken(('# ' if s else '') + s + '\n', mark, None) + + if after_indent is None: + after_indent = indent + 2 + if before and before[-1] == '\n': + before = before[:-1] # strip final newline if there + if after and after[-1] == '\n': + after = after[:-1] # strip final newline if there + start_mark = Mark(None, None, None, indent, None, None) + c = self.ca.items.setdefault(key, [None, [], None, None]) + if before: + for com in before.split('\n'): + c[1].append(comment_token(com, start_mark)) + if after: + start_mark = Mark(None, None, None, after_indent, None, None) + if c[3] is None: + c[3] = [] + for com in after.split('\n'): + c[3].append(comment_token(com, start_mark)) + @property def fa(self): """format attribute @@ -2,9 +2,12 @@ from __future__ import absolute_import +import warnings + from ruamel.yaml.compat import utf8 -__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError', 'ReusedAnchorWarning'] +__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError', 'ReusedAnchorWarning', + 'UnsafeLoaderWarning'] class Mark(object): @@ -84,3 +87,18 @@ class MarkedYAMLError(YAMLError): class ReusedAnchorWarning(Warning): pass + + +class UnsafeLoaderWarning(Warning): + text = """ +The default 'Loader' for 'load(stream)' without further arguments can be unsafe. +Use 'load(stream, Loader=ruamel.yaml.Loader)' explicitly if that is OK. +Alternatively include the following in your code: + + import warnings + warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning) + +In most other cases you should consider using 'safe_load(stream)'""" + pass + +warnings.simplefilter('once', UnsafeLoaderWarning) @@ -70,12 +70,17 @@ def compose_all(stream, Loader=Loader): loader.dispose() -def load(stream, Loader=Loader, version=None, preserve_quotes=None): +def load(stream, Loader=None, version=None, preserve_quotes=None): # type: (StreamType, Any, VersionType, Any) -> Any """ Parse the first YAML document in a stream and produce the corresponding Python object. """ + if Loader is None: + from ruamel.yaml.loader import Loader as UnsafeLoader + import warnings + warnings.warn(UnsafeLoaderWarning.text, UnsafeLoaderWarning, stacklevel=2) + Loader = UnsafeLoader loader = Loader(stream, version, preserve_quotes=preserve_quotes) try: return loader.get_single_data() @@ -83,11 +88,16 @@ def load(stream, Loader=Loader, version=None, preserve_quotes=None): loader.dispose() -def load_all(stream, Loader=Loader, version=None, preserve_quotes=None): +def load_all(stream, Loader=None, version=None, preserve_quotes=None): """ Parse all YAML documents in a stream and produce corresponding Python objects. """ + if Loader is None: + from ruamel.yaml.loader import Loader as UnsafeLoader + import warnings + warnings.warn(UnsafeLoaderWarning.text, UnsafeLoaderWarning, stacklevel=2) + Loader = UnsafeLoader loader = Loader(stream, version, preserve_quotes=preserve_quotes) try: while loader.check_data(): @@ -1,7 +1,7 @@ [tox] # envlist = pep8,py35,py27,py34,py33,py26,pypy,jython #envlist = py35,py27,py34,py33,py26,pypy,jython -envlist = pep8,py35,py27,py34,py33,pypy +envlist = pep8,py35,py36,py27,py34,pypy [testenv] commands = |