fix #99: no RT on block scalar + empty line + comment0.13.14

4 files changed, 72 insertions, 57 deletions

diff --git a/README.rst b/README.rst
index 23b5afa..8b990b0 100644
--- a/README.rst
+++ b/README.rst
@@ -18,6 +18,11 @@ ChangeLog
 
 .. should insert NEXT: at the beginning of line for next key
 
+0.13.14 (2017-02-12):
+  - fix for issue 97: clipped block scalar followed by empty lines and comment
+    would result in two CommentTokens of which the first was dropped.
+    (reported by Colm O'Connor)
+
 0.13.13 (2017-01-28):
   - fix for issue 96: prevent insertion of extra empty line if indented mapping entries
     are separated by an empty line (reported by Derrick Sawyer)
diff --git a/__init__.py b/__init__.py
index b9b8d06..78f694e 100644
--- a/__init__.py
+++ b/__init__.py
@@ -7,39 +7,43 @@ from __future__ import print_function, absolute_import, division, unicode_litera
 # be installed at some point
 
 _package_data = dict(
-    full_package_name="ruamel.yaml",
-    version_info=(0, 13, 14, "dev"),
-    author="Anthon van der Neut",
-    author_email="a.van.der.neut@ruamel.eu",
-    description="ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order",  # NOQA
+    full_package_name='ruamel.yaml',
+    version_info=(0, 13, 14),
+    __version__='0.13.14',
+    author='Anthon van der Neut',
+    author_email='a.van.der.neut@ruamel.eu',
+    description='ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order',  # NOQA
     entry_points=None,
     install_requires=dict(
         any=[],
-        py33=["typing"],
-        py34=["typing"],
-        py27=["ruamel.ordereddict", "typing"],
-        pypy=["typing"],
+        py33=['typing'],
+        py34=['typing'],
+        py27=['ruamel.ordereddict', 'typing'],
+        pypy=['typing'],
     ),
     ext_modules=[dict(
-        name="_ruamel_yaml",
-        src=["ext/_ruamel_yaml.c", "ext/api.c", "ext/writer.c", "ext/dumper.c",
-                "ext/loader.c", "ext/reader.c", "ext/scanner.c", "ext/parser.c",
-                "ext/emitter.c"],
-        lib=[],
-        test='#include "ext/yaml.h"\n\nint main(int argc, char* argv[])\n{\nyaml_parser_t parser;\nparser = parser;  /* prevent warning */\nreturn 0;\n}\n'  # NOQA
-        )
-    ],
+            name='_ruamel_yaml',
+            src=['ext/_ruamel_yaml.c', 'ext/api.c', 'ext/writer.c', 'ext/dumper.c',
+                'ext/loader.c',
+                'ext/reader.c',
+                'ext/scanner.c',
+                'ext/parser.c',
+                'ext/emitter.c',
+            ],
+            lib=[],
+            test='#include "ext/yaml.h"\n\nint main(int argc, char* argv[])\n{\nyaml_parser_t parser;\nparser = parser;  /* prevent warning */\nreturn 0;\n}\n',  # NOQA
+        )],
     classifiers=[
-        "Programming Language :: Python :: 2.7",
-        "Programming Language :: Python :: 3.3",
-        "Programming Language :: Python :: 3.4",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: Implementation :: CPython",
-        "Programming Language :: Python :: Implementation :: PyPy",
-        "Programming Language :: Python :: Implementation :: Jython",
-        "Topic :: Software Development :: Libraries :: Python Modules",
-        "Topic :: Text Processing :: Markup"
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: Implementation :: CPython',
+        'Programming Language :: Python :: Implementation :: PyPy',
+        'Programming Language :: Python :: Implementation :: Jython',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+        'Topic :: Text Processing :: Markup',
     ],
     windows_wheels=True,
     read_the_docs='yaml',
@@ -48,29 +52,8 @@ _package_data = dict(
 )
 
 
-# < from ruamel.util.new import _convert_version
-def _convert_version(tup):
-    """create a PEP 386 pseudo-format conformant string from tuple tup"""
-    ret_val = str(tup[0])  # first is always digit
-    next_sep = "."  # separator for next extension, can be "" or "."
-    for x in tup[1:]:
-        if isinstance(x, int):
-            ret_val += next_sep + str(x)
-            next_sep = '.'
-            continue
-        first_letter = x[0].lower()
-        next_sep = ''
-        if first_letter in 'abcr':
-            ret_val += 'rc' if first_letter == 'r' else first_letter
-        elif first_letter in 'pd':
-            ret_val += '.post' if first_letter == 'p' else '.dev'
-    return ret_val
-
-# <
 version_info = _package_data['version_info']
-__version__ = _convert_version(version_info)  # type: ignore
-
-del _convert_version
+__version__ = _package_data['__version__']
 
 try:
     from .cyaml import *  # NOQA
diff --git a/_test/test_comments.py b/_test/test_comments.py
index 78daa5e..d1ef7fb 100644
--- a/_test/test_comments.py
+++ b/_test/test_comments.py
@@ -735,3 +735,24 @@ class TestEmptyValueBeforeComments:
           a: 1 # comment a
           b:   # comment b
         """)
+
+test_block_scalar_commented_line_template = """\
+y: p
+# Some comment
+
+a: |
+  x
+{}b: y
+"""
+
+
+class TestBlockScalarWithComments:
+    # issue 99 reported by Colm O'Connor
+
+    for x in ['', '\n', '\n# Another comment\n', '\n\n', '\n\n# abc\n#xyz\n',
+              '\n\n# abc\n#xyz\n', '# abc\n\n#xyz\n', '\n\n  # abc\n  #xyz\n']:
+
+        commented_line = test_block_scalar_commented_line_template.format(x)
+        data = ruamel.yaml.round_trip_load(commented_line)
+
+        assert ruamel.yaml.round_trip_dump(data) == commented_line
diff --git a/scanner.py b/scanner.py
index 95e9d0a..51f6cb9 100644
--- a/scanner.py
+++ b/scanner.py
@@ -1070,22 +1070,28 @@ class Scanner(object):
 
         # Process trailing line breaks. The 'chomping' setting determines
         # whether they are included in the value.
-        comment = []
+        trailing = []
         if chomping in [None, True]:
             chunks.append(line_break)
         if chomping is True:
             chunks.extend(breaks)
         elif chomping in [None, False]:
-            comment.extend(breaks)
+            trailing.extend(breaks)
 
         # We are done.
-        token = ScalarToken(u''.join(chunks), False, start_mark, end_mark,
-                            style)
-        if len(comment) > 0:
-            # Keep track of the trailing whitespace as a comment token, if
-            # isn't all included in the actual value.
+        token = ScalarToken(u''.join(chunks), False, start_mark, end_mark, style)
+        if len(trailing) > 0:
+            # print('trailing 1', trailing)  # XXXXX
+            # Eat whitespaces and comments until we reach the next token.
+            comment = self.scan_to_next_token()
+            while comment:
+                trailing.append(comment[0])
+                comment = self.scan_to_next_token()
+
+            # Keep track of the trailing whitespace and following comments
+            # as a comment token, if isn't all included in the actual value.
             comment_end_mark = self.get_mark()
-            comment = CommentToken(''.join(comment), end_mark,
+            comment = CommentToken(''.join(trailing), end_mark,
                                    comment_end_mark)
             token.add_post_comment(comment)
         return token