From 3d972c045bab860b93ee6c28c61dabeafe81d7e9 Mon Sep 17 00:00:00 2001 From: Anthon van der Neut Date: Sun, 12 Aug 2018 20:21:22 +0200 Subject: fix issue #172 compact JSON no longer parsing This was quite intrusive, because loosing the tests surrounding the colon made flow style lists with ::vector as key fail. Had to keept track of whether in a flow-style mapping or sequence to get this solved *When this change indeed resolves your problem, please **Close** this issue*. *(You can do so usingthe WorkFlow pull-down (close to the top right of this page)* --- _test/test_issues.py | 14 +++++++++++++- main.py | 17 ++++++++++++++++- scanner.py | 38 ++++++++++++++++++++++++++++---------- 3 files changed, 57 insertions(+), 12 deletions(-) diff --git a/_test/test_issues.py b/_test/test_issues.py index 7e95857..4ddec2a 100644 --- a/_test/test_issues.py +++ b/_test/test_issues.py @@ -44,7 +44,7 @@ class TestIssues: key-A:{} mapping-B: """) - for comment in ['', ' # no-newline', ' # some comment\n', '\n', ]: + for comment in ['', ' # no-newline', ' # some comment\n', '\n']: s = yaml_str.format(comment) res = round_trip(s) # NOQA @@ -65,3 +65,15 @@ class TestIssues: - {} """) x = round_trip(s, preserve_quotes=True) # NOQA + + json_str = ( + '{"sshKeys":[{"name":"AETROS\/google-k80-1","uses":0,"getLastUse":0,' + '"fingerprint":"MD5:19:dd:41:93:a1:a3:f5:91:4a:8e:9b:d0:ae:ce:66:4c",' + '"created":1509497961}]}' + ) + + json_str2 = '{"abc":[{"a":"1", "uses":0}]}' + + def test_issue_172(self): + x = round_trip_load(TestIssues.json_str2) + x = round_trip_load(TestIssues.json_str) diff --git a/main.py b/main.py index fa9db38..4afaa78 100644 --- a/main.py +++ b/main.py @@ -639,7 +639,6 @@ class YAML(object): self.constructor.add_constructor(tag, f_y) return cls - def parse(self, stream): # type: (StreamTextType, Any) -> Any """ @@ -920,6 +919,14 @@ def load(stream, Loader=None, version=None, preserve_quotes=None): return loader._constructor.get_single_data() finally: loader._parser.dispose() + try: + loader._reader.reset_reader() # type: ignore + except AttributeError: + pass + try: + loader._scanner.reset_scanner() # type: ignore + except AttributeError: + pass def load_all(stream, Loader=None, version=None, preserve_quotes=None): @@ -937,6 +944,14 @@ def load_all(stream, Loader=None, version=None, preserve_quotes=None): yield loader._constructor.get_data() finally: loader._parser.dispose() + try: + loader._reader.reset_reader() # type: ignore + except AttributeError: + pass + try: + loader._scanner.reset_scanner() # type: ignore + except AttributeError: + pass def safe_load(stream, version=None): diff --git a/scanner.py b/scanner.py index 5813c53..c0d376b 100644 --- a/scanner.py +++ b/scanner.py @@ -82,14 +82,25 @@ class Scanner(object): self.reset_scanner() self.first_time = False + @property + def flow_level(self): + return len(self.flow_context) + + # @flow_level.setter + # def flow_level(self): + # self.flow_context = [] + def reset_scanner(self): # type: () -> None # Had we reached the end of the stream? self.done = False + # flow_context is an expanding/shrinking list consisting of '{' and '[' + # in general len(flow_context) == flow_level + self.flow_context = [] # The number of unclosed '{' and '['. `flow_level == 0` means block # context. - self.flow_level = 0 + self.flow_level_org = 0 # List of processed tokens that are not yet emitted. self.tokens = [] # type: List[Any] @@ -489,18 +500,20 @@ class Scanner(object): def fetch_flow_sequence_start(self): # type: () -> None - self.fetch_flow_collection_start(FlowSequenceStartToken) + self.fetch_flow_collection_start(FlowSequenceStartToken, to_push='[') def fetch_flow_mapping_start(self): # type: () -> None - self.fetch_flow_collection_start(FlowMappingStartToken) + self.fetch_flow_collection_start(FlowMappingStartToken, to_push='{') - def fetch_flow_collection_start(self, TokenClass): + def fetch_flow_collection_start(self, TokenClass, to_push): # type: (Any) -> None # '[' and '{' may start a simple key. self.save_possible_simple_key() # Increase the flow level. - self.flow_level += 1 + self.flow_level_org += 1 + self.flow_context.append(to_push) + # assert self.flow_level == len(self.flow_context) # Simple keys are allowed after '[' and '{'. self.allow_simple_key = True # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START. @@ -511,18 +524,20 @@ class Scanner(object): def fetch_flow_sequence_end(self): # type: () -> None - self.fetch_flow_collection_end(FlowSequenceEndToken) + self.fetch_flow_collection_end(FlowSequenceEndToken, to_pop='[') def fetch_flow_mapping_end(self): # type: () -> None - self.fetch_flow_collection_end(FlowMappingEndToken) + self.fetch_flow_collection_end(FlowMappingEndToken, to_pop='{') - def fetch_flow_collection_end(self, TokenClass): + def fetch_flow_collection_end(self, TokenClass, to_pop): # type: (Any) -> None # Reset possible simple key on the current level. self.remove_possible_simple_key() # Decrease the flow level. - self.flow_level -= 1 + self.flow_level_org -= 1 + assert self.flow_context.pop() == to_pop + assert self.flow_level_org == len(self.flow_context) # No simple keys after ']' or '}'. self.allow_simple_key = False # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END. @@ -778,7 +793,10 @@ class Scanner(object): if bool(self.flow_level): return True else: - if bool(self.flow_level) and self.reader.peek(1) in '\'"{[]}': + if bool(self.flow_level): + if self.flow_context[-1] == '[': + return False + # if self.reader.peek(1) in '\'"{[]}': return True # VALUE(block context): ':' (' '|'\n') return self.reader.peek(1) in _THE_END_SPACE_TAB -- cgit v1.2.1