diff options
author | Anthon van der Neut <anthon@mnt.org> | 2021-04-19 08:17:05 +0200 |
---|---|---|
committer | Anthon van der Neut <anthon@mnt.org> | 2021-04-19 08:17:05 +0200 |
commit | 3d77f16e00124b74e150625396617b41e41da014 (patch) | |
tree | ed01337ba7c78a5a5e68255a7a574e00e78556fc /tokens.py | |
parent | 54cb146e541b557471a19fcd884abd7e441d85b8 (diff) | |
download | ruamel.yaml-3d77f16e00124b74e150625396617b41e41da014.tar.gz |
removed (object), prepare new RTScanner
Diffstat (limited to 'tokens.py')
-rw-r--r-- | tokens.py | 134 |
1 files changed, 115 insertions, 19 deletions
@@ -6,10 +6,10 @@ if False: # MYPY from typing import Text, Any, Dict, Optional, List # NOQA from .error import StreamMark # NOQA -SHOWLINES = True +SHOW_LINES = True -class Token(object): +class Token: __slots__ = 'start_mark', 'end_mark', '_comment' def __init__(self, start_mark, end_mark): @@ -23,19 +23,20 @@ class Token(object): # hasattr('self', key)] attributes = [key for key in self.__slots__ if not key.endswith('_mark')] attributes.sort() - arguments = ', '.join( - [_F('{key!s}={gattr!r})', key=key, gattr=getattr(self, key)) for key in attributes] - ) - if SHOWLINES: + #arguments = ', '.join( + # [_F('{key!s}={gattr!r})', key=key, gattr=getattr(self, key)) for key in attributes] + #) + arguments = [_F('{key!s}={gattr!r}', key=key, gattr=getattr(self, key)) for key in attributes] + if SHOW_LINES: try: - arguments += ', line: ' + str(self.start_mark.line) + arguments.append('line: ' + str(self.start_mark.line)) except: # NOQA pass try: - arguments += ', comment: ' + str(self._comment) + arguments.append('comment: ' + str(self._comment)) except: # NOQA pass - return '{}({})'.format(self.__class__.__name__, arguments) + return '{}({})'.format(self.__class__.__name__, ', '.join(arguments)) @property def column(self): @@ -45,29 +46,77 @@ class Token(object): def column(self, pos): self.start_mark.column = pos + # old style ( <= 0.17) is a TWO element list with first being the EOL + # comment concatenated with following FLC/BLNK; and second being a list of FLC/BLNK + # preceding the token + # new style ( >= 0.17 ) is a THREE element list with the first being a list of + # preceding FLC/BLNK, the second EOL and the third following FLC/BLNK + # note that new style has differing order, and does not consist of CommentToken(s) + # but of CommentInfo instances + # any non-assigned values in new style are None, but first and last can be empty list + # new style routines add one comment at a time + + # going to be deprecated in favour of add_comment_eol/post def add_post_comment(self, comment): # type: (Any) -> None if not hasattr(self, '_comment'): self._comment = [None, None] + else: + assert len(self._comment) == 2 # make sure it is version 0 + #if isinstance(comment, CommentToken): + # if comment.value.startswith('# C09'): + # raise self._comment[0] = comment + # going to be deprecated in favour of add_comment_pre def add_pre_comments(self, comments): # type: (Any) -> None if not hasattr(self, '_comment'): self._comment = [None, None] + else: + assert len(self._comment) == 2 # make sure it is version 0 assert self._comment[1] is None self._comment[1] = comments + return - def get_comment(self): - # type: () -> Any - return getattr(self, '_comment', None) + # new style + def add_comment_pre(self, comment): + if not hasattr(self, '_comment'): + self._comment = [[], None, None] + else: + assert len(self._comment) == 3 + if self._comment[0] is None: + self._comment[0] = [] + self._comment[0].append(comment) + + def add_comment_eol(self, comment): + if not hasattr(self, '_comment'): + self._comment = [None, None, None] + else: + assert len(self._comment) == 3 + assert self._comment[1] is None + self._comment[1] = comment + + def add_comment_post(self, comment): + if not hasattr(self, '_comment'): + self._comment = [None, None, []] + else: + assert len(self._comment) == 3 + if self._comment[2] is None: + self._comment[2] = [] + self._comment[2].append(comment) + + + #def get_comment(self): + # # type: () -> Any + # return getattr(self, '_comment', None) @property def comment(self): # type: () -> Any return getattr(self, '_comment', None) - def move_comment(self, target, empty=False): + def move_old_comment(self, target, empty=False): # type: (Any, bool) -> Any """move a comment from this token to target (normally next token) used to combine e.g. comments before a BlockEntryToken to the @@ -97,7 +146,7 @@ class Token(object): tc[1] = c[1] return self - def split_comment(self): + def split_old_comment(self): # type: () -> Any """ split the post part of a comment, and return it as comment to be added. Delete second part if [None, None] @@ -113,6 +162,40 @@ class Token(object): delattr(self, '_comment') return ret_val + def move_new_comment(self, target, empty=False): + # type: (Any, bool) -> Any + """move a comment from this token to target (normally next token) + used to combine e.g. comments before a BlockEntryToken to the + ScalarToken that follows it + empty is a special for empty values -> comment after key + """ + c = self.comment + if c is None: + return + # don't push beyond last element + if isinstance(target, (StreamEndToken, DocumentStartToken)): + return + delattr(self, '_comment') + tc = target.comment + if not tc: # target comment, just insert + # special for empty value in key: value issue 25 + if empty: + c = [c[0], c[1], c[2]] + target._comment = c + # nprint('mco2:', self, target, target.comment, empty) + return self + return + raise NotImplemtedError + # if self and target have both pre, eol or post comments, something seems wrong + for idx in range(3): + if c[idx] and tc[idx]: + raise NotImplementedError(_F('overlap in comment {c!r} {tc!r}', c=c, tc=tc)) + # move the comment parts + for idx in range(3): + if c[idx]: + tc[idx] = c[idx] + return self + # class BOMToken(Token): # id = '<byte order mark>' @@ -256,13 +339,26 @@ class ScalarToken(Token): class CommentToken(Token): - __slots__ = 'value', 'pre_done' + __slots__ = '_value', 'pre_done' id = '<comment>' - def __init__(self, value, start_mark, end_mark): + def __init__(self, value, start_mark=None, end_mark=None, column=None): # type: (Any, Any, Any) -> None - Token.__init__(self, start_mark, end_mark) - self.value = value + if start_mark is None: + assert columns is not None + self._column = column + Token.__init__(self, start_mark, None) + self._value = value + + @property + def value(self): + if isinstance(self._value, str): + return self._value + return "".join(self._value) + + @value.setter + def value(self, val): + self._value = val def reset(self): # type: () -> None @@ -272,7 +368,7 @@ class CommentToken(Token): def __repr__(self): # type: () -> Any v = '{!r}'.format(self.value) - if SHOWLINES: + if SHOW_LINES: try: v += ', line: ' + str(self.start_mark.line) except: # NOQA |