fix for indenting non-indented sequence after comment

- solves several expected to fail and fixes two incorrect tests (test_09-Sammy Sosa &
  test_before_nested_seq_from_scratch)
- still swaps foo and bar on test_issue_25_03

6 files changed, 163 insertions, 126 deletions

diff --git a/_test/test_comments.py b/_test/test_comments.py
index 9698fde..8717727 100644
--- a/_test/test_comments.py
+++ b/_test/test_comments.py
@@ -141,9 +141,9 @@ class TestComments:
           # 1998 rbi ranking
           - Sammy Sosa
           - Ken Griffey
-        """)
+        """, indent=4, block_seq_indent=2)
 
-    def test_09a(self):   
+    def test_09a(self):
         round_trip("""
         hr: # 1998 hr ranking
         - Mark McGwire
@@ -152,7 +152,7 @@ class TestComments:
           # 1998 rbi ranking
         - Sammy Sosa
         - Ken Griffey
-        """, indent=4, block_seq_indent=2)
+        """)
 
     def test_simple_map_middle_comment(self):
         round_trip("""
diff --git a/compat.py b/compat.py
index 6b05d5a..5eddc59 100644
--- a/compat.py
+++ b/compat.py
@@ -125,6 +125,13 @@ DBG_NODE = 4
 
 
 _debug = None  # type: Union[None, int]
+if 'RUAMELDEBUG' in os.environ:
+    _debugx = os.environ.get('RUAMELDEBUG')
+    if _debugx is None:
+        _debug = 0
+    else:
+        _debug = int(_debugx)
+
 
 if bool(_debug):
     class ObjectCounter(object):
diff --git a/emitter.py b/emitter.py
index a1b703d..fcf3004 100644
--- a/emitter.py
+++ b/emitter.py
@@ -318,10 +318,13 @@ class Emitter(object):
                 self.expect_scalar()
             elif isinstance(self.event, SequenceStartEvent):
                 if self.event.comment:
-                    self.write_pre_comment(self.event)
+                    if self.write_pre_comment(self.event):
+                        self.indention = False
+                        self.no_newline = True
                     if self.event.flow_style is False and self.event.comment:
-                        self.write_post_comment(self.event)
-                # print('seq event', self.event)
+                        if self.write_post_comment(self.event):
+                            self.indention = False
+                            self.no_newline = True
                 if self.flow_level or self.canonical or self.event.flow_style or \
                         self.check_empty_sequence():
                     self.expect_flow_sequence()
@@ -490,7 +493,9 @@ class Emitter(object):
         else:
             if self.event.comment and self.event.comment[1]:
                 self.write_pre_comment(self.event)
+            nonl = self.no_newline
             self.write_indent()
+            self.no_newline = nonl
             self.write_indicator((u' ' * self.block_seq_indent) + u'-', True, indention=True)
             if self.block_seq_indent + 2 > self.best_indent:
                 self.no_newline = True
@@ -1372,27 +1377,29 @@ class Emitter(object):
         self.write_line_break()
 
     def write_pre_comment(self, event):
-        # type: (Any) -> None
+        # type: (Any) -> bool
         comments = event.comment[1]
         if comments is None:
-            return
+            return False
         try:
             for comment in comments:
                 if isinstance(event, MappingStartEvent) and \
                    getattr(comment, 'pre_done', None):
                     continue
                 if self.column != 0:
-                        self.write_line_break()
+                    self.write_line_break()
                 self.write_comment(comment)
                 if isinstance(event, MappingStartEvent):
                     comment.pre_done = True
         except TypeError:
             print('eventtt', type(event), event)
             raise
+        return True
 
     def write_post_comment(self, event):
-        # type: (Any) -> None
+        # type: (Any) -> bool
         if self.event.comment[0] is None:
-            return
+            return False
         comment = event.comment[0]
         self.write_comment(comment)
+        return True
diff --git a/parser.py b/parser.py
index 860fdd9..7812628 100644
--- a/parser.py
+++ b/parser.py
@@ -80,7 +80,7 @@ from ruamel.yaml.error import MarkedYAMLError
 from ruamel.yaml.tokens import *                               # NOQA
 from ruamel.yaml.events import *                               # NOQA
 from ruamel.yaml.scanner import Scanner, RoundTripScanner, ScannerError      # NOQA
-from ruamel.yaml.compat import utf8                            # NOQA
+from ruamel.yaml.compat import utf8, nprint                    # NOQA
 
 if False:  # MYPY
     from typing import Any, Dict, Optional, List  # NOQA
@@ -342,121 +342,131 @@ class Parser(object):
             token = self.scanner.get_token()
             event = AliasEvent(token.value, token.start_mark, token.end_mark)  # type: Any
             self.state = self.states.pop()
-        else:
-            anchor = None
-            tag = None
-            start_mark = end_mark = tag_mark = None
-            if self.scanner.check_token(AnchorToken):
+            return event
+
+        anchor = None
+        tag = None
+        start_mark = end_mark = tag_mark = None
+        if self.scanner.check_token(AnchorToken):
+            token = self.scanner.get_token()
+            start_mark = token.start_mark
+            end_mark = token.end_mark
+            anchor = token.value
+            if self.scanner.check_token(TagToken):
                 token = self.scanner.get_token()
-                start_mark = token.start_mark
+                tag_mark = token.start_mark
                 end_mark = token.end_mark
-                anchor = token.value
-                if self.scanner.check_token(TagToken):
-                    token = self.scanner.get_token()
-                    tag_mark = token.start_mark
-                    end_mark = token.end_mark
-                    tag = token.value
-            elif self.scanner.check_token(TagToken):
+                tag = token.value
+        elif self.scanner.check_token(TagToken):
+            token = self.scanner.get_token()
+            start_mark = tag_mark = token.start_mark
+            end_mark = token.end_mark
+            tag = token.value
+            if self.scanner.check_token(AnchorToken):
                 token = self.scanner.get_token()
                 start_mark = tag_mark = token.start_mark
                 end_mark = token.end_mark
-                tag = token.value
-                if self.scanner.check_token(AnchorToken):
-                    token = self.scanner.get_token()
-                    end_mark = token.end_mark
-                    anchor = token.value
-            if tag is not None:
-                handle, suffix = tag
-                if handle is not None:
-                    if handle not in self.tag_handles:
-                        raise ParserError(
-                            "while parsing a node", start_mark,
-                            "found undefined tag handle %r" % utf8(handle),
-                            tag_mark)
-                    tag = self.transform_tag(handle, suffix)
-                else:
-                    tag = suffix
-            # if tag == u'!':
-            #     raise ParserError("while parsing a node", start_mark,
-            #             "found non-specific tag '!'", tag_mark,
-            #      "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag'
-            #     and share your opinion.")
-            if start_mark is None:
-                start_mark = end_mark = self.scanner.peek_token().start_mark
-            event = None
-            implicit = (tag is None or tag == u'!')
-            if indentless_sequence and self.scanner.check_token(BlockEntryToken):
-                end_mark = self.scanner.peek_token().end_mark
-                event = SequenceStartEvent(anchor, tag, implicit,
-                                           start_mark, end_mark)
-                self.state = self.parse_indentless_sequence_entry
-            else:
-                if self.scanner.check_token(ScalarToken):
-                    token = self.scanner.get_token()
-                    end_mark = token.end_mark
-                    if (token.plain and tag is None) or tag == u'!':
-                        implicit = (True, False)
-                    elif tag is None:
-                        implicit = (False, True)
-                    else:
-                        implicit = (False, False)
-                    event = ScalarEvent(
-                        anchor, tag, implicit, token.value,
-                        start_mark, end_mark, style=token.style,
-                        comment=token.comment
-                    )
-                    self.state = self.states.pop()
-                elif self.scanner.check_token(FlowSequenceStartToken):
-                    end_mark = self.scanner.peek_token().end_mark
-                    event = SequenceStartEvent(
-                        anchor, tag, implicit,
-                        start_mark, end_mark, flow_style=True)
-                    self.state = self.parse_flow_sequence_first_entry
-                elif self.scanner.check_token(FlowMappingStartToken):
-                    end_mark = self.scanner.peek_token().end_mark
-                    event = MappingStartEvent(
-                        anchor, tag, implicit,
-                        start_mark, end_mark, flow_style=True)
-                    self.state = self.parse_flow_mapping_first_key
-                elif block and self.scanner.check_token(BlockSequenceStartToken):
-                    end_mark = self.scanner.peek_token().start_mark
-                    # should inserting the comment be dependent on the
-                    # indentation?
-                    pt = self.scanner.peek_token()
-                    comment = pt.comment
-                    # print('pt0', type(pt))
-                    if comment is None or comment[1] is None:
-                        comment = pt.split_comment()
-                    # print('pt1', comment)
-                    event = SequenceStartEvent(
-                        anchor, tag, implicit, start_mark, end_mark,
-                        flow_style=False,
-                        comment=comment,
-                    )
-                    self.state = self.parse_block_sequence_first_entry
-                elif block and self.scanner.check_token(BlockMappingStartToken):
-                    end_mark = self.scanner.peek_token().start_mark
-                    comment = self.scanner.peek_token().comment
-                    event = MappingStartEvent(
-                        anchor, tag, implicit, start_mark, end_mark,
-                        flow_style=False, comment=comment)
-                    self.state = self.parse_block_mapping_first_key
-                elif anchor is not None or tag is not None:
-                    # Empty scalars are allowed even if a tag or an anchor is
-                    # specified.
-                    event = ScalarEvent(anchor, tag, (implicit, False), u'',
-                                        start_mark, end_mark)
-                    self.state = self.states.pop()
-                else:
-                    if block:
-                        node = 'block'
-                    else:
-                        node = 'flow'
-                    token = self.scanner.peek_token()
+                anchor = token.value
+        if tag is not None:
+            handle, suffix = tag
+            if handle is not None:
+                if handle not in self.tag_handles:
                     raise ParserError(
-                        "while parsing a %s node" % node, start_mark,
-                        "expected the node content, but found %r" % token.id,
-                        token.start_mark)
+                        "while parsing a node", start_mark,
+                        "found undefined tag handle %r" % utf8(handle),
+                        tag_mark)
+                tag = self.transform_tag(handle, suffix)
+            else:
+                tag = suffix
+        # if tag == u'!':
+        #     raise ParserError("while parsing a node", start_mark,
+        #             "found non-specific tag '!'", tag_mark,
+        #      "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag'
+        #     and share your opinion.")
+        if start_mark is None:
+            start_mark = end_mark = self.scanner.peek_token().start_mark
+        event = None
+        implicit = (tag is None or tag == u'!')
+        if indentless_sequence and self.scanner.check_token(BlockEntryToken):
+            comment = None
+            pt = self.scanner.peek_token()
+            if pt.comment and pt.comment[0]:
+                comment = [pt.comment[0], []]
+                pt.comment[0] = None
+            end_mark = self.scanner.peek_token().end_mark
+            event = SequenceStartEvent(anchor, tag, implicit,
+                                       start_mark, end_mark, comment=comment)
+            self.state = self.parse_indentless_sequence_entry
+            return event
+
+        if self.scanner.check_token(ScalarToken):
+            token = self.scanner.get_token()
+            # self.scanner.peek_token_same_line_comment(token)
+            end_mark = token.end_mark
+            if (token.plain and tag is None) or tag == u'!':
+                implicit = (True, False)
+            elif tag is None:
+                implicit = (False, True)
+            else:
+                implicit = (False, False)
+            # nprint('se', token.value, token.comment)
+            event = ScalarEvent(
+                anchor, tag, implicit, token.value,
+                start_mark, end_mark, style=token.style,
+                comment=token.comment
+            )
+            self.state = self.states.pop()
+        elif self.scanner.check_token(FlowSequenceStartToken):
+            end_mark = self.scanner.peek_token().end_mark
+            event = SequenceStartEvent(
+                anchor, tag, implicit,
+                start_mark, end_mark, flow_style=True)
+            self.state = self.parse_flow_sequence_first_entry
+        elif self.scanner.check_token(FlowMappingStartToken):
+            end_mark = self.scanner.peek_token().end_mark
+            event = MappingStartEvent(
+                anchor, tag, implicit,
+                start_mark, end_mark, flow_style=True)
+            self.state = self.parse_flow_mapping_first_key
+        elif block and self.scanner.check_token(BlockSequenceStartToken):
+            end_mark = self.scanner.peek_token().start_mark
+            # should inserting the comment be dependent on the
+            # indentation?
+            pt = self.scanner.peek_token()
+            comment = pt.comment
+            # nprint('pt0', type(pt))
+            if comment is None or comment[1] is None:
+                comment = pt.split_comment()
+            # nprint('pt1', comment)
+            event = SequenceStartEvent(
+                anchor, tag, implicit, start_mark, end_mark,
+                flow_style=False,
+                comment=comment,
+            )
+            self.state = self.parse_block_sequence_first_entry
+        elif block and self.scanner.check_token(BlockMappingStartToken):
+            end_mark = self.scanner.peek_token().start_mark
+            comment = self.scanner.peek_token().comment
+            event = MappingStartEvent(
+                anchor, tag, implicit, start_mark, end_mark,
+                flow_style=False, comment=comment)
+            self.state = self.parse_block_mapping_first_key
+        elif anchor is not None or tag is not None:
+            # Empty scalars are allowed even if a tag or an anchor is
+            # specified.
+            event = ScalarEvent(anchor, tag, (implicit, False), u'',
+                                start_mark, end_mark)
+            self.state = self.states.pop()
+        else:
+            if block:
+                node = 'block'
+            else:
+                node = 'flow'
+            token = self.scanner.peek_token()
+            raise ParserError(
+                "while parsing a %s node" % node, start_mark,
+                "expected the node content, but found %r" % token.id,
+                token.start_mark)
         return event
 
     # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)*
diff --git a/scanner.py b/scanner.py
index 70c66d3..5e6a5e5 100644
--- a/scanner.py
+++ b/scanner.py
@@ -32,7 +32,7 @@ from __future__ import print_function, absolute_import, division, unicode_litera
 
 from ruamel.yaml.error import MarkedYAMLError
 from ruamel.yaml.tokens import *                # NOQA
-from ruamel.yaml.compat import utf8, unichr, PY3, check_anchorname_char
+from ruamel.yaml.compat import utf8, unichr, PY3, check_anchorname_char, nprint  # NOQA
 
 if False:  # MYPY
     from typing import Any, Dict, Optional, List, Union, Text  # NOQA
diff --git a/tokens.py b/tokens.py
index e5bac6b..99f4b19 100644
--- a/tokens.py
+++ b/tokens.py
@@ -4,6 +4,7 @@
 if False:  # MYPY
     from typing import Any, Dict, Optional, List  # NOQA
 
+SHOWLINES = True
 
 class Token(object):
     __slots__ = 'start_mark', 'end_mark', '_comment',
@@ -15,12 +16,18 @@ class Token(object):
 
     def __repr__(self):
         # type: () -> Any
-        attributes = [key for key in self.__slots__ if not key.endswith('_mark') and
-                      hasattr('self', key)]
+        # attributes = [key for key in self.__slots__ if not key.endswith('_mark') and
+        #               hasattr('self', key)]
+        attributes = [key for key in self.__slots__ if not key.endswith('_mark')]
         attributes.sort()
         arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
                                for key in attributes])
-        return '%s(%s)' % (self.__class__.__name__, arguments)
+        if SHOWLINES:
+            try:
+                arguments += u', line: ' + str(self.start_mark.line)
+            except:
+                pass
+        return u'{}({})'.format(self.__class__.__name__, arguments)
 
     def add_post_comment(self, comment):
         # type: (Any) -> None
@@ -261,4 +268,10 @@ class CommentToken(Token):
 
     def __repr__(self):
         # type: () -> Any
-        return 'CommentToken({!r})'.format(self.value)
+        v = self.value
+        if SHOWLINES:
+            try:
+                v += u', line: ' + str(self.start_mark.line)
+            except:
+                pass
+        return 'CommentToken({!r})'.format(v)