removed (object), prepare new RTScanner

1 files changed, 115 insertions, 19 deletions

diff --git a/tokens.py b/tokens.py
index 19a2572..490866b 100644
--- a/tokens.py
+++ b/tokens.py
@@ -6,10 +6,10 @@ if False:  # MYPY
     from typing import Text, Any, Dict, Optional, List  # NOQA
     from .error import StreamMark  # NOQA
 
-SHOWLINES = True
+SHOW_LINES = True
 
 
-class Token(object):
+class Token:
     __slots__ = 'start_mark', 'end_mark', '_comment'
 
     def __init__(self, start_mark, end_mark):
@@ -23,19 +23,20 @@ class Token(object):
         #               hasattr('self', key)]
         attributes = [key for key in self.__slots__ if not key.endswith('_mark')]
         attributes.sort()
-        arguments = ', '.join(
-            [_F('{key!s}={gattr!r})', key=key, gattr=getattr(self, key)) for key in attributes]
-        )
-        if SHOWLINES:
+        #arguments = ', '.join(
+        #    [_F('{key!s}={gattr!r})', key=key, gattr=getattr(self, key)) for key in attributes]
+        #)
+        arguments = [_F('{key!s}={gattr!r}', key=key, gattr=getattr(self, key)) for key in attributes]
+        if SHOW_LINES:
             try:
-                arguments += ', line: ' + str(self.start_mark.line)
+                arguments.append('line: ' + str(self.start_mark.line))
             except:  # NOQA
                 pass
         try:
-            arguments += ', comment: ' + str(self._comment)
+            arguments.append('comment: ' + str(self._comment))
         except:  # NOQA
             pass
-        return '{}({})'.format(self.__class__.__name__, arguments)
+        return '{}({})'.format(self.__class__.__name__, ', '.join(arguments))
 
     @property
     def column(self):
@@ -45,29 +46,77 @@ class Token(object):
     def column(self, pos):
         self.start_mark.column = pos
 
+    # old style ( <= 0.17) is a TWO element list with first being the EOL 
+    # comment concatenated with following FLC/BLNK; and second being a list of FLC/BLNK
+    # preceding the token
+    # new style ( >= 0.17 ) is a THREE element list with the first being a list of
+    # preceding FLC/BLNK, the second EOL and the third following FLC/BLNK
+    # note that new style has differing order, and does not consist of CommentToken(s)
+    # but of CommentInfo instances
+    # any non-assigned values in new style are None, but first and last can be empty list
+    # new style routines add one comment at a time
+
+    # going to be deprecated in favour of add_comment_eol/post
     def add_post_comment(self, comment):
         # type: (Any) -> None
         if not hasattr(self, '_comment'):
             self._comment = [None, None]
+        else:
+            assert len(self._comment) == 2 # make sure it is version 0
+        #if isinstance(comment, CommentToken):
+        #    if comment.value.startswith('# C09'):
+        #        raise
         self._comment[0] = comment
 
+    # going to be deprecated in favour of add_comment_pre
     def add_pre_comments(self, comments):
         # type: (Any) -> None
         if not hasattr(self, '_comment'):
             self._comment = [None, None]
+        else:
+            assert len(self._comment) == 2 # make sure it is version 0
         assert self._comment[1] is None
         self._comment[1] = comments
+        return
 
-    def get_comment(self):
-        # type: () -> Any
-        return getattr(self, '_comment', None)
+    # new style
+    def add_comment_pre(self, comment):
+        if not hasattr(self, '_comment'):
+            self._comment = [[], None, None]
+        else:
+            assert len(self._comment) == 3
+            if self._comment[0] is None:
+                self._comment[0] = []
+        self._comment[0].append(comment)
+
+    def add_comment_eol(self, comment):
+        if not hasattr(self, '_comment'):
+            self._comment = [None, None, None]
+        else:
+            assert len(self._comment) == 3
+            assert self._comment[1] is None
+        self._comment[1] = comment
+
+    def add_comment_post(self, comment):
+        if not hasattr(self, '_comment'):
+            self._comment = [None, None, []]
+        else:
+            assert len(self._comment) == 3
+            if self._comment[2] is None:
+                self._comment[2] = []
+        self._comment[2].append(comment)
+
+
+    #def get_comment(self):
+    #    # type: () -> Any
+    #    return getattr(self, '_comment', None)
 
     @property
     def comment(self):
         # type: () -> Any
         return getattr(self, '_comment', None)
 
-    def move_comment(self, target, empty=False):
+    def move_old_comment(self, target, empty=False):
         # type: (Any, bool) -> Any
         """move a comment from this token to target (normally next token)
         used to combine e.g. comments before a BlockEntryToken to the
@@ -97,7 +146,7 @@ class Token(object):
             tc[1] = c[1]
         return self
 
-    def split_comment(self):
+    def split_old_comment(self):
         # type: () -> Any
         """ split the post part of a comment, and return it
         as comment to be added. Delete second part if [None, None]
@@ -113,6 +162,40 @@ class Token(object):
             delattr(self, '_comment')
         return ret_val
 
+    def move_new_comment(self, target, empty=False):
+        # type: (Any, bool) -> Any
+        """move a comment from this token to target (normally next token)
+        used to combine e.g. comments before a BlockEntryToken to the
+        ScalarToken that follows it
+        empty is a special for empty values -> comment after key
+        """
+        c = self.comment
+        if c is None:
+            return
+        # don't push beyond last element
+        if isinstance(target, (StreamEndToken, DocumentStartToken)):
+            return
+        delattr(self, '_comment')
+        tc = target.comment
+        if not tc:  # target comment, just insert
+            # special for empty value in key: value issue 25
+            if empty:
+                c = [c[0], c[1], c[2]]
+            target._comment = c
+            # nprint('mco2:', self, target, target.comment, empty)
+            return self
+        return
+        raise NotImplemtedError
+        # if self and target have both pre, eol or post comments, something seems wrong
+        for idx in range(3):
+            if c[idx] and tc[idx]:
+                raise NotImplementedError(_F('overlap in comment {c!r} {tc!r}', c=c, tc=tc))
+        # move the comment parts
+        for idx in range(3):
+            if c[idx]:
+                tc[idx] = c[idx]
+        return self
+
 
 # class BOMToken(Token):
 #     id = '<byte order mark>'
@@ -256,13 +339,26 @@ class ScalarToken(Token):
 
 
 class CommentToken(Token):
-    __slots__ = 'value', 'pre_done'
+    __slots__ = '_value', 'pre_done'
     id = '<comment>'
 
-    def __init__(self, value, start_mark, end_mark):
+    def __init__(self, value, start_mark=None, end_mark=None, column=None):
         # type: (Any, Any, Any) -> None
-        Token.__init__(self, start_mark, end_mark)
-        self.value = value
+        if start_mark is None:
+            assert columns is not None
+            self._column = column
+        Token.__init__(self, start_mark, None)
+        self._value = value
+
+    @property
+    def value(self):
+        if isinstance(self._value, str):
+            return self._value
+        return "".join(self._value)
+
+    @value.setter
+    def value(self, val):
+        self._value = val
 
     def reset(self):
         # type: () -> None
@@ -272,7 +368,7 @@ class CommentToken(Token):
     def __repr__(self):
         # type: () -> Any
         v = '{!r}'.format(self.value)
-        if SHOWLINES:
+        if SHOW_LINES:
             try:
                 v += ', line: ' + str(self.start_mark.line)
             except:  # NOQA