From 156336d846bff0b891687f483edc7113288e9b0d Mon Sep 17 00:00:00 2001 From: xi Date: Mon, 20 Feb 2006 20:30:09 +0000 Subject: Implement Composer and BaseResolver. git-svn-id: http://svn.pyyaml.org/branches/pyyaml3000@53 18f92427-320e-0410-9341-c67f048884a3 --- lib/yaml/__init__.py | 32 ++++++++++ lib/yaml/composer.py | 92 +++++++++++++++++++++++++++++ lib/yaml/error.py | 16 ++--- lib/yaml/events.py | 14 ++--- lib/yaml/nodes.py | 35 +++++++++++ lib/yaml/parser.py | 9 +++ lib/yaml/resolver.py | 68 +++++++++++++++++++++ tests/data/duplicate-anchor-1.error-message | 3 + tests/data/duplicate-anchor-2.error-message | 1 + tests/data/recursive-anchor.error-message | 4 ++ tests/data/undefined-anchor.error-message | 3 + tests/test_errors.py | 12 +++- tests/test_structure.py | 2 +- 13 files changed, 273 insertions(+), 18 deletions(-) create mode 100644 lib/yaml/composer.py create mode 100644 lib/yaml/nodes.py create mode 100644 lib/yaml/resolver.py create mode 100644 tests/data/duplicate-anchor-1.error-message create mode 100644 tests/data/duplicate-anchor-2.error-message create mode 100644 tests/data/recursive-anchor.error-message create mode 100644 tests/data/undefined-anchor.error-message diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index cae7cde..f1fcf10 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -1,7 +1,10 @@ +from error import YAMLError from reader import Reader from scanner import Scanner from parser import Parser +from composer import Composer +from resolver import Resolver from tokens import * from events import * @@ -17,3 +20,32 @@ def parse(data, Reader=Reader, Scanner=Scanner, Parser=Parser): parser = Parser(scanner) return iter(parser) +def compose(data, Reader=Reader, Scanner=Scanner, Parser=Parser, + Composer=Composer): + reader = Reader(data) + scanner = Scanner(reader) + parser = Parser(scanner) + composer = Composer(parser) + return iter(composer) + +def compose_document(*args, **kwds): + try: + return compose(*args, **kwds).next() + except StopIteration: + return None + +def resolve(data, Reader=Reader, Scanner=Scanner, Parser=Parser, + Composer=Composer, Resolver=Resolver): + reader = Reader(data) + scanner = Scanner(reader) + parser = Parser(scanner) + composer = Composer(parser) + resolver = Resolver(composer) + return iter(resolver) + +def resolve_document(*args, **kwds): + try: + return resolve(*args, **kwds).next() + except StopIteration: + return None + diff --git a/lib/yaml/composer.py b/lib/yaml/composer.py new file mode 100644 index 0000000..fcd93ab --- /dev/null +++ b/lib/yaml/composer.py @@ -0,0 +1,92 @@ + +from error import MarkedYAMLError +from events import * +from nodes import * + +class ComposerError(MarkedYAMLError): + pass + +class Composer: + + def __init__(self, parser): + self.parser = parser + self.all_anchors = {} + self.complete_anchors = {} + + def check(self): + # If there are more documents available? + return not self.parser.check(StreamEndEvent) + + def get(self): + # Get the root node of the next document. + if not self.parser.check(StreamEndEvent): + return self.compose_document() + + def __iter__(self): + # Iterator protocol. + while not self.parser.check(StreamEndEvent): + yield self.compose_document() + + def compose_document(self): + node = self.compose_node() + self.all_anchors = {} + self.complete_anchors = {} + return node + + def compose_node(self): + if self.parser.check(AliasEvent): + event = self.parser.get() + anchor = event.anchor + if anchor not in self.all_anchors: + raise ComposerError(None, None, "found undefined alias %r" + % anchor.encode('utf-8'), event.start_marker) + if anchor not in self.complete_anchors: + collection_event = self.all_anchors[anchor] + raise ComposerError("while composing a collection", + collection_event.start_marker, + "found recursive anchor %r" % anchor.encode('utf-8'), + event.start_marker) + return self.complete_anchors[anchor] + event = self.parser.peek() + anchor = event.anchor + if anchor is not None: + if anchor in self.all_anchors: + raise ComposerError("found duplicate anchor %r; first occurence" + % anchor.encode('utf-8'), self.all_anchors[anchor].start_marker, + "second occurence", event.start_marker) + self.all_anchors[anchor] = event + if self.parser.check(ScalarEvent): + node = self.compose_scalar_node() + elif self.parser.check(SequenceEvent): + node = self.compose_sequence_node() + elif self.parser.check(MappingEvent): + node = self.compose_mapping_node() + if anchor is not None: + self.complete_anchors[anchor] = node + return node + + def compose_scalar_node(self): + event = self.parser.get() + return ScalarNode(event.tag, event.value, + event.start_marker, event.end_marker) + + def compose_sequence_node(self): + start_event = self.parser.get() + value = [] + while not self.parser.check(CollectionEndEvent): + value.append(self.compose_node()) + end_event = self.parser.get() + return SequenceNode(start_event.tag, value, + start_event.start_marker, end_event.end_marker) + + def compose_mapping_node(self): + start_event = self.parser.get() + value = [] + while not self.parser.check(CollectionEndEvent): + item_key = self.compose_node() + item_value = self.compose_node() + value.append((item_key, item_value)) + end_event = self.parser.get() + return MappingNode(start_event.tag, value, + start_event.start_marker, end_event.end_marker) + diff --git a/lib/yaml/error.py b/lib/yaml/error.py index 2fae3eb..38f143e 100644 --- a/lib/yaml/error.py +++ b/lib/yaml/error.py @@ -63,16 +63,16 @@ class MarkedYAMLError(YAMLError): # lines.append(str(marker)) if self.context is not None: lines.append(self.context) - if self.context_marker is not None \ - and (self.problem is None or self.problem_marker is None - or self.context_marker.name != self.problem_marker.name - or self.context_marker.line != self.problem_marker.line - or self.context_marker.column != self.problem_marker.column): - lines.append(str(self.context_marker)) + if self.context_marker is not None \ + and (self.problem is None or self.problem_marker is None + or self.context_marker.name != self.problem_marker.name + or self.context_marker.line != self.problem_marker.line + or self.context_marker.column != self.problem_marker.column): + lines.append(str(self.context_marker)) if self.problem is not None: lines.append(self.problem) - if self.problem_marker is not None: - lines.append(str(self.problem_marker)) + if self.problem_marker is not None: + lines.append(str(self.problem_marker)) return '\n'.join(lines) diff --git a/lib/yaml/events.py b/lib/yaml/events.py index 6ecb772..d468c53 100644 --- a/lib/yaml/events.py +++ b/lib/yaml/events.py @@ -12,17 +12,13 @@ class Event: return '%s(%s)' % (self.__class__.__name__, arguments) class NodeEvent(Event): - def __init__(self, anchor, tag, start_marker, end_marker): + def __init__(self, anchor, start_marker, end_marker): self.anchor = anchor - self.tag = tag self.start_marker = start_marker self.end_marker = end_marker class AliasEvent(NodeEvent): - def __init__(self, name, start_marker, end_marker): - self.name = name - self.start_marker = start_marker - self.end_marker = end_marker + pass class ScalarEvent(NodeEvent): def __init__(self, anchor, tag, value, start_marker, end_marker): @@ -33,7 +29,11 @@ class ScalarEvent(NodeEvent): self.end_marker = end_marker class CollectionEvent(NodeEvent): - pass + def __init__(self, anchor, tag, start_marker, end_marker): + self.anchor = anchor + self.tag = tag + self.start_marker = start_marker + self.end_marker = end_marker class SequenceEvent(CollectionEvent): pass diff --git a/lib/yaml/nodes.py b/lib/yaml/nodes.py new file mode 100644 index 0000000..c3fd626 --- /dev/null +++ b/lib/yaml/nodes.py @@ -0,0 +1,35 @@ + +class Node: + def __init__(self, tag, value, start_marker, end_marker): + self.tag = tag + self.value = value + self.start_marker = start_marker + self.end_marker = end_marker + def __repr__(self): + value = self.value + if isinstance(value, list): + if len(value) == 0: + value = '' + elif len(value) == 1: + value = '<1 item>' + else: + value = '<%d items>' % len(value) + else: + if len(value) > 75: + value = repr(value[:70]+u' ... ') + else: + value = repr(value) + return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value) + +class ScalarNode(Node): + pass + +class CollectionNode(Node): + pass + +class SequenceNode(CollectionNode): + pass + +class MappingNode(CollectionNode): + pass + diff --git a/lib/yaml/parser.py b/lib/yaml/parser.py index d5dcd6d..0c3d616 100644 --- a/lib/yaml/parser.py +++ b/lib/yaml/parser.py @@ -99,6 +99,15 @@ class Parser: return True return False + def peek(self): + # Get the next event. + if self.current_event is None: + try: + self.current_event = self.event_generator.next() + except StopIteration: + pass + return self.current_event + def get(self): # Get the next event. if self.current_event is None: diff --git a/lib/yaml/resolver.py b/lib/yaml/resolver.py new file mode 100644 index 0000000..7cff321 --- /dev/null +++ b/lib/yaml/resolver.py @@ -0,0 +1,68 @@ + +from nodes import * + +class BaseResolver: + + DEFAULT_SCALAR_TAG = u'tag:yaml.org,2002:str' + DEFAULT_SEQUENCE_TAG = u'tag:yaml.org,2002:seq' + DEFAULT_MAPPING_TAG = u'tag:yaml.org,2002:map' + + def __init__(self, composer): + self.composer = composer + self.resolved_nodes = {} + + def check(self): + # If there are more documents available? + return self.composer.check() + + def get(self): + # Resolve and return the root node of the next document. + if self.composer.check(): + return self.resolve_document(self.composer.get()) + + def __iter__(self): + # Iterator protocol. + while self.composer.check(): + yield self.resolve_document(self.composer.get()) + + def resolve_document(self, node): + self.resolve_node([], node) + return node + self.resolved_nodes = {} + + def resolve_node(self, path, node): + if node in self.resolved_nodes: + return + self.resolved_nodes[node] = None + if isinstance(node, ScalarNode): + self.resolve_scalar(path, node) + elif isinstance(node, SequenceNode): + self.resolve_sequence(path, node) + for index in len(node.value): + self.resolve_node(path+[node, index], node.value[index]) + elif isinstance(node, MappingNode): + self.resolve_mapping(path, node) + for key, value in node.value: + self.resolve_node(path+[node, None], key) + self.resolve_node(path+[node, key], value) + + def resolve_scalar(self, node): + if node.tag is None: + node.tag = self.detect_scalar(node.value) + if node.tag is None or node.tag == u'!': + node.tag = self.DEFAULT_SCALAR_TAG + + def resolve_sequence(self, node): + if node.tag is None or node.tag == u'!': + node.tag = self.DEFAULT_SEQUENCE_TAG + + def resolve_mapping(self, node): + if node.tag is None or node.tag == u'!': + node.tag = self.DEFAULT_MAPPING_TAG + + def detect_scalar(self, value): + return None + +class Resolver(BaseResolver): + pass + diff --git a/tests/data/duplicate-anchor-1.error-message b/tests/data/duplicate-anchor-1.error-message new file mode 100644 index 0000000..906cf29 --- /dev/null +++ b/tests/data/duplicate-anchor-1.error-message @@ -0,0 +1,3 @@ +- &foo bar +- &bar bar +- &foo bar diff --git a/tests/data/duplicate-anchor-2.error-message b/tests/data/duplicate-anchor-2.error-message new file mode 100644 index 0000000..62b4389 --- /dev/null +++ b/tests/data/duplicate-anchor-2.error-message @@ -0,0 +1 @@ +&foo [1, 2, 3, &foo 4] diff --git a/tests/data/recursive-anchor.error-message b/tests/data/recursive-anchor.error-message new file mode 100644 index 0000000..661166c --- /dev/null +++ b/tests/data/recursive-anchor.error-message @@ -0,0 +1,4 @@ +- &foo [1 + 2, + 3, + *foo] diff --git a/tests/data/undefined-anchor.error-message b/tests/data/undefined-anchor.error-message new file mode 100644 index 0000000..9469103 --- /dev/null +++ b/tests/data/undefined-anchor.error-message @@ -0,0 +1,3 @@ +- foo +- &bar baz +- *bat diff --git a/tests/test_errors.py b/tests/test_errors.py index cf229a4..43834cb 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -5,6 +5,8 @@ from yaml.error import YAMLError from yaml.reader import * from yaml.scanner import * from yaml.parser import * +from yaml.composer import * +from yaml.resolver import * class TestErrors(test_appliance.TestAppliance): @@ -21,10 +23,13 @@ class TestErrors(test_appliance.TestAppliance): reader = Reader(file(filename, 'rb')) scanner = Scanner(reader) parser = Parser(scanner) - return list(parser) + composer = Composer(parser) + resolver = Resolver(composer) + return list(composer) except YAMLError, exc: #except ScannerError, exc: #except ParserError, exc: + #except ComposerError, exc: #print '.'*70 #print "%s:" % exc.__class__.__name__, exc raise @@ -34,10 +39,13 @@ class TestErrors(test_appliance.TestAppliance): reader = Reader(file(filename, 'rb').read()) scanner = Scanner(reader) parser = Parser(scanner) - return list(parser) + composer = Composer(parser) + resolver = Resolver(composer) + return list(composer) except YAMLError, exc: #except ScannerError, exc: #except ParserError, exc: + #except ComposerError, exc: #print '.'*70 #print "%s:" % filename #print "%s:" % exc.__class__.__name__, exc diff --git a/tests/test_structure.py b/tests/test_structure.py index 0bffbcd..06ff6f3 100644 --- a/tests/test_structure.py +++ b/tests/test_structure.py @@ -123,7 +123,7 @@ class TestParserOnCanonical(test_appliance.TestAppliance): for event1, event2 in zip(events1, events2): self.failUnlessEqual(event1.__class__, event2.__class__) if isinstance(event1, AliasEvent): - self.failUnlessEqual(event1.name, event2.name) + self.failUnlessEqual(event1.anchor, event2.anchor) elif isinstance(event1, ScalarEvent): self.failUnlessEqual(event1.anchor, event2.anchor) self.failUnlessEqual(event1.tag, event2.tag) -- cgit v1.2.1