fix #158 folded scalars not roundtripping0.15.61

10 files changed, 95 insertions, 42 deletions

diff --git a/CHANGES b/CHANGES
index e6ef72a..ddadfe8 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,15 @@
+[0, 15, 61]: 2018-08-23
+  - support for round-tripping folded style scalars (initially requested 
+    by `Johnathan Viduchinsky <https://bitbucket.org/johnathanvidu/>`__)
+  - update of C code
+  - speed up of scanning (~30% depending on the input)
+
+[0, 16, 0]: 2018-08-23
+  - initial support for round-tripping folded style scalars (initially requested 
+    by `Johnathan Viduchinsky <https://bitbucket.org/johnathanvidu/>`__)
+  - update of C code
+  - speed up of scanning (~30% depending on the input)
+
 [0, 15, 60]: 2018-08-18
   - cleanup for mypy 
   - spurious print in library (reported by 
diff --git a/README.rst b/README.rst
index 5eed39e..ecd8302 100644
--- a/README.rst
+++ b/README.rst
@@ -4,8 +4,8 @@ ruamel.yaml
 
 ``ruamel.yaml`` is a YAML 1.2 loader/dumper package for Python.
 
-:version:       0.15.60
-:updated:       2018-08-18
+:version:       0.15.61
+:updated:       2018-08-23
 :documentation: http://yaml.readthedocs.io
 :repository:    https://bitbucket.org/ruamel/
 :pypi:          https://pypi.org/project/ruamel.yaml/
@@ -54,6 +54,12 @@ ChangeLog
 
 .. should insert NEXT: at the beginning of line for next key (with empty line)
 
+0.15.61 (2018-08-23):
+  - support for round-tripping folded style scalars (initially requested 
+    by `Johnathan Viduchinsky <https://bitbucket.org/johnathanvidu/>`__)
+  - update of C code
+  - speed up of scanning (~30% depending on the input)
+
 0.15.60 (2018-08-18):
   - again allow single entry map in flow sequence context (reported by 
     `Lee Goolsbee <https://bitbucket.org/lgoolsbee/>`__)
diff --git a/__init__.py b/__init__.py
index 6bf5b91..f65a31d 100644
--- a/__init__.py
+++ b/__init__.py
@@ -7,8 +7,8 @@ if False:  # MYPY
 
 _package_data = dict(
     full_package_name='ruamel.yaml',
-    version_info=(0, 15, 60),
-    __version__='0.15.60',
+    version_info=(0, 15, 61),
+    __version__='0.15.61',
     author='Anthon van der Neut',
     author_email='a.van.der.neut@ruamel.eu',
     description='ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order',  # NOQA
diff --git a/_doc/_static/pypi.svg b/_doc/_static/pypi.svg
index 15dd630..63129dc 100644
--- a/_doc/_static/pypi.svg
+++ b/_doc/_static/pypi.svg
@@ -1 +1 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="86" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="86" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h33v20H0z"/><path fill="#007ec6" d="M33 0h53v20H33z"/><path fill="url(#b)" d="M0 0h86v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"> <text x="175" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="230">pypi</text><text x="175" y="140" transform="scale(.1)" textLength="230">pypi</text><text x="585" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">0.15.60</text><text x="585" y="140" transform="scale(.1)" textLength="430">0.15.60</text></g> </svg>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="86" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="86" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h33v20H0z"/><path fill="#007ec6" d="M33 0h53v20H33z"/><path fill="url(#b)" d="M0 0h86v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"> <text x="175" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="230">pypi</text><text x="175" y="140" transform="scale(.1)" textLength="230">pypi</text><text x="585" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">0.15.61</text><text x="585" y="140" transform="scale(.1)" textLength="430">0.15.61</text></g> </svg>
diff --git a/_test/test_string.py b/_test/test_string.py
index a12bdd6..b9fe1ae 100644
--- a/_test/test_string.py
+++ b/_test/test_string.py
@@ -86,16 +86,6 @@ class TestPreservedScalarString:
             """
         round_trip(inp, intermediate=dict(a='ghi\njkl\n\n'))
 
-    def Xtest_fold_string(self):
-        with pytest.raises(AssertionError) as excinfo:  # NOQA
-            inp = """
-            a: >
-              abc
-              def
-
-            """
-            round_trip(inp, intermediate=dict(a='abc def\n'))
-
     def test_fold_string(self):
         inp = """
         a: >
@@ -106,16 +96,6 @@ class TestPreservedScalarString:
         round_trip(inp)
 
     def test_fold_string_strip(self):
-        with pytest.raises(AssertionError) as excinfo:  # NOQA
-            inp = """
-            a: >-
-              abc
-              def
-
-            """
-            round_trip(inp, intermediate=dict(a='abc def'))
-
-    def test_fold_string_strip(self):
         inp = """
         a: >-
           abc
diff --git a/constructor.py b/constructor.py
index cc2ca8c..a7abaf3 100644
--- a/constructor.py
+++ b/constructor.py
@@ -20,10 +20,11 @@ from ruamel.yaml.compat import (utf8, builtins_module, to_str, PY2, PY3,  # NOQA
                                 MutableSequence, MutableMapping)
 from ruamel.yaml.comments import *                               # NOQA
 from ruamel.yaml.comments import (CommentedMap, CommentedOrderedMap, CommentedSet,
-                                  CommentedKeySeq, CommentedSeq, TaggedScalar, CommentedKeyMap)
-from ruamel.yaml.scalarstring import *                           # NOQA
-from ruamel.yaml.scalarstring import (PreservedScalarString, SingleQuotedScalarString,
-                                      DoubleQuotedScalarString, ScalarString)
+                                  CommentedKeySeq, CommentedSeq, TaggedScalar,
+                                  CommentedKeyMap)
+from ruamel.yaml.scalarstring import (SingleQuotedScalarString, DoubleQuotedScalarString,
+                                      LiteralScalarString, FoldedScalarString,
+                                      ScalarString,)
 from ruamel.yaml.scalarint import ScalarInt, BinaryInt, OctalInt, HexInt, HexCapsInt
 from ruamel.yaml.scalarfloat import ScalarFloat
 from ruamel.yaml.timestamp import TimeStamp
@@ -1030,10 +1031,24 @@ class RoundTripConstructor(SafeConstructor):
             )
 
         if node.style == '|' and isinstance(node.value, text_type):
-            pss = PreservedScalarString(node.value)
+            pss = LiteralScalarString(node.value)
             if node.comment and node.comment[1]:
                 pss.comment = node.comment[1][0]  # type: ignore
             return pss
+        if node.style == '>' and isinstance(node.value, text_type):
+            fold_positions = []
+            idx = None
+            while True:
+                idx = node.value.find('\a', None if idx is None else idx + 1)
+                if idx < 0:
+                    break
+                fold_positions.append(idx - len(fold_positions))
+            pss = FoldedScalarString(node.value.replace('\a', ''))
+            if node.comment and node.comment[1]:
+                pss.comment = node.comment[1][0]  # type: ignore
+            if fold_positions:
+                pss.fold_pos = fold_positions
+            return pss
         elif bool(self._preserve_quotes) and isinstance(node.value, text_type):
             if node.style == "'":
                 return SingleQuotedScalarString(node.value)
diff --git a/emitter.py b/emitter.py
index 0e447d3..5e5e29a 100644
--- a/emitter.py
+++ b/emitter.py
@@ -1365,7 +1365,7 @@ class Emitter(object):
             if end < len(text):
                 ch = text[end]
             if breaks:
-                if ch is None or ch not in u'\n\x85\u2028\u2029':
+                if ch is None or ch not in u'\n\x85\u2028\u2029\a':
                     if (
                         not leading_space
                         and ch is not None
@@ -1394,12 +1394,16 @@ class Emitter(object):
                         self.stream.write(data)
                     start = end
             else:
-                if ch is None or ch in u' \n\x85\u2028\u2029':
+                if ch is None or ch in u' \n\x85\u2028\u2029\a':
                     data = text[start:end]
                     self.column += len(data)
                     if bool(self.encoding):
                         data = data.encode(self.encoding)
                     self.stream.write(data)
+                    if ch == u'\a':
+                        self.write_line_break()
+                        self.write_indent()
+                        end += 2  # \a and the space that is inserted on the fold
                     if ch is None:
                         self.write_line_break()
                     start = end
diff --git a/representer.py b/representer.py
index b9175d6..3088139 100644
--- a/representer.py
+++ b/representer.py
@@ -7,7 +7,8 @@ from ruamel.yaml.error import *  # NOQA
 from ruamel.yaml.nodes import *  # NOQA
 from ruamel.yaml.compat import text_type, binary_type, to_unicode, PY2, PY3, ordereddict
 from ruamel.yaml.scalarstring import (
-    PreservedScalarString,
+    LiteralScalarString,
+    FoldedScalarString,
     SingleQuotedScalarString,
     DoubleQuotedScalarString,
 )
@@ -149,7 +150,7 @@ class BaseRepresenter(object):
         if style is None:
             style = self.default_style
         comment = None
-        if style and style[0] == '|':
+        if style and style[0] in '|>':
             comment = getattr(value, 'comment', None)
             if comment:
                 comment = [None, [comment]]
@@ -667,7 +668,7 @@ class RoundTripRepresenter(SafeRepresenter):
             return self.represent_scalar(u'tag:yaml.org,2002:null', u'null')
         return self.represent_scalar(u'tag:yaml.org,2002:null', "")
 
-    def represent_preserved_scalarstring(self, data):
+    def represent_literal_scalarstring(self, data):
         # type: (Any) -> Any
         tag = None
         style = '|'
@@ -676,6 +677,20 @@ class RoundTripRepresenter(SafeRepresenter):
         tag = u'tag:yaml.org,2002:str'
         return self.represent_scalar(tag, data, style=style)
 
+    represent_preserved_scalarstring = represent_literal_scalarstring
+
+    def represent_folded_scalarstring(self, data):
+        # type: (Any) -> Any
+        tag = None
+        style = '>'
+        for fold_pos in reversed(getattr(data, 'fold_pos', [])):
+            if data[fold_pos] == ' ':
+                data = data[:fold_pos] + '\a' + data[fold_pos:]
+        if PY2 and not isinstance(data, unicode):
+            data = unicode(data, 'ascii')
+        tag = u'tag:yaml.org,2002:str'
+        return self.represent_scalar(tag, data, style=style)
+
     def represent_single_quoted_scalarstring(self, data):
         # type: (Any) -> Any
         tag = None
@@ -1113,7 +1128,11 @@ class RoundTripRepresenter(SafeRepresenter):
 RoundTripRepresenter.add_representer(type(None), RoundTripRepresenter.represent_none)
 
 RoundTripRepresenter.add_representer(
-    PreservedScalarString, RoundTripRepresenter.represent_preserved_scalarstring
+    LiteralScalarString, RoundTripRepresenter.represent_literal_scalarstring
+)
+
+RoundTripRepresenter.add_representer(
+    FoldedScalarString, RoundTripRepresenter.represent_folded_scalarstring
 )
 
 RoundTripRepresenter.add_representer(
diff --git a/scalarstring.py b/scalarstring.py
index 0bc3c47..a701f11 100644
--- a/scalarstring.py
+++ b/scalarstring.py
@@ -9,9 +9,11 @@ if False:  # MYPY
 
 __all__ = [
     'ScalarString',
-    'PreservedScalarString',
+    'LiteralScalarString',
+    'FoldedScalarString',
     'SingleQuotedScalarString',
     'DoubleQuotedScalarString',
+    'PreservedScalarString',
 ]
 
 
@@ -27,7 +29,7 @@ class ScalarString(text_type):
         return type(self)((text_type.replace(self, old, new, maxreplace)))
 
 
-class PreservedScalarString(ScalarString):
+class LiteralScalarString(ScalarString):
     __slots__ = 'comment'  # the comment after the | on the first line
 
     style = '|'
@@ -37,6 +39,19 @@ class PreservedScalarString(ScalarString):
         return ScalarString.__new__(cls, value)
 
 
+PreservedScalarString = LiteralScalarString
+
+
+class FoldedScalarString(ScalarString):
+    __slots__ = ('fold_pos', 'comment')  # the comment after the > on the first line
+
+    style = '>'
+
+    def __new__(cls, value):
+        # type: (Text) -> Any
+        return ScalarString.__new__(cls, value)
+
+
 class SingleQuotedScalarString(ScalarString):
     __slots__ = ()
 
@@ -59,11 +74,11 @@ class DoubleQuotedScalarString(ScalarString):
 
 def preserve_literal(s):
     # type: (Text) -> Text
-    return PreservedScalarString(s.replace('\r\n', '\n').replace('\r', '\n'))
+    return LiteralScalarString(s.replace('\r\n', '\n').replace('\r', '\n'))
 
 
 def walk_tree(base, map=None):
-    # type: (Any) -> None
+    # type: (Any, Any) -> None
     """
     the routine here walks over a simple yaml tree (recursing in
     dict values and list items) and converts strings that
@@ -95,7 +110,7 @@ def walk_tree(base, map=None):
         for idx, elem in enumerate(base):
             if isinstance(elem, string_types):
                 for ch in map:
-                    if ch in elem:
+                    if ch in elem:  # type: ignore
                         base[idx] = map[ch](elem)
                         break
             else:
diff --git a/scanner.py b/scanner.py
index 5033339..ab20d7c 100644
--- a/scanner.py
+++ b/scanner.py
@@ -145,7 +145,7 @@ class Scanner(object):
     def reader(self):
         # type: () -> Any
         try:
-            return self._scanner_reader
+            return self._scanner_reader  # type: ignore
         except AttributeError:
             if hasattr(self.loader, 'typ'):
                 self._scanner_reader = self.loader.reader
@@ -1172,6 +1172,8 @@ class Scanner(object):
                 #
                 # This is the folding according to the specification:
 
+                if folded and line_break == '\n':
+                    chunks.append('\a')
                 if (
                     folded
                     and line_break == '\n'