From 428faafc50d046470da82af48fd3cf9a9854b088 Mon Sep 17 00:00:00 2001 From: xi Date: Mon, 30 May 2011 02:51:30 +0000 Subject: Clear cyclic references in the parser and the emitter to avoid extra GC calls. git-svn-id: http://svn.pyyaml.org/pyyaml/trunk@377 18f92427-320e-0410-9341-c67f048884a3 --- ext/_yaml.pyx | 6 +++++ lib/yaml/__init__.py | 67 +++++++++++++++++++++++++++++++++--------------- lib/yaml/constructor.py | 4 +-- lib/yaml/emitter.py | 5 ++++ lib/yaml/parser.py | 5 ++++ lib3/yaml/__init__.py | 67 +++++++++++++++++++++++++++++++++--------------- lib3/yaml/constructor.py | 4 +-- lib3/yaml/emitter.py | 5 ++++ lib3/yaml/parser.py | 5 ++++ tests/lib/canonical.py | 3 +++ tests/lib3/canonical.py | 3 +++ 11 files changed, 130 insertions(+), 44 deletions(-) diff --git a/ext/_yaml.pyx b/ext/_yaml.pyx index df19492..5158fb4 100644 --- a/ext/_yaml.pyx +++ b/ext/_yaml.pyx @@ -311,6 +311,9 @@ cdef class CParser: yaml_parser_delete(&self.parser) yaml_event_delete(&self.parsed_event) + def dispose(self): + pass + cdef object _parser_error(self): if self.parser.error == YAML_MEMORY_ERROR: return MemoryError @@ -993,6 +996,9 @@ cdef class CEmitter: def __dealloc__(self): yaml_emitter_delete(&self.emitter) + def dispose(self): + pass + cdef object _emitter_error(self): if self.emitter.error == YAML_MEMORY_ERROR: return MemoryError diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index c0fd1f3..d414be8 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -21,16 +21,22 @@ def scan(stream, Loader=Loader): Scan a YAML stream and produce scanning tokens. """ loader = Loader(stream) - while loader.check_token(): - yield loader.get_token() + try: + while loader.check_token(): + yield loader.get_token() + finally: + loader.dispose() def parse(stream, Loader=Loader): """ Parse a YAML stream and produce parsing events. """ loader = Loader(stream) - while loader.check_event(): - yield loader.get_event() + try: + while loader.check_event(): + yield loader.get_event() + finally: + loader.dispose() def compose(stream, Loader=Loader): """ @@ -38,7 +44,10 @@ def compose(stream, Loader=Loader): and produce the corresponding representation tree. """ loader = Loader(stream) - return loader.get_single_node() + try: + return loader.get_single_node() + finally: + loader.dispose() def compose_all(stream, Loader=Loader): """ @@ -46,8 +55,11 @@ def compose_all(stream, Loader=Loader): and produce corresponding representation trees. """ loader = Loader(stream) - while loader.check_node(): - yield loader.get_node() + try: + while loader.check_node(): + yield loader.get_node() + finally: + loader.dispose() def load(stream, Loader=Loader): """ @@ -55,7 +67,10 @@ def load(stream, Loader=Loader): and produce the corresponding Python object. """ loader = Loader(stream) - return loader.get_single_data() + try: + return loader.get_single_data() + finally: + loader.dispose() def load_all(stream, Loader=Loader): """ @@ -63,8 +78,11 @@ def load_all(stream, Loader=Loader): and produce corresponding Python objects. """ loader = Loader(stream) - while loader.check_data(): - yield loader.get_data() + try: + while loader.check_data(): + yield loader.get_data() + finally: + loader.dispose() def safe_load(stream): """ @@ -96,8 +114,11 @@ def emit(events, stream=None, Dumper=Dumper, getvalue = stream.getvalue dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, allow_unicode=allow_unicode, line_break=line_break) - for event in events: - dumper.emit(event) + try: + for event in events: + dumper.emit(event) + finally: + dumper.dispose() if getvalue: return getvalue() @@ -122,10 +143,13 @@ def serialize_all(nodes, stream=None, Dumper=Dumper, allow_unicode=allow_unicode, line_break=line_break, encoding=encoding, version=version, tags=tags, explicit_start=explicit_start, explicit_end=explicit_end) - dumper.open() - for node in nodes: - dumper.serialize(node) - dumper.close() + try: + dumper.open() + for node in nodes: + dumper.serialize(node) + dumper.close() + finally: + dumper.dispose() if getvalue: return getvalue() @@ -160,10 +184,13 @@ def dump_all(documents, stream=None, Dumper=Dumper, allow_unicode=allow_unicode, line_break=line_break, encoding=encoding, version=version, tags=tags, explicit_start=explicit_start, explicit_end=explicit_end) - dumper.open() - for data in documents: - dumper.represent(data) - dumper.close() + try: + dumper.open() + for data in documents: + dumper.represent(data) + dumper.close() + finally: + dumper.dispose() if getvalue: return getvalue() diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index 420c434..081d78a 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -58,11 +58,11 @@ class BaseConstructor(object): return data def construct_object(self, node, deep=False): + if node in self.constructed_objects: + return self.constructed_objects[node] if deep: old_deep = self.deep_construct self.deep_construct = True - if node in self.constructed_objects: - return self.constructed_objects[node] if node in self.recursive_objects: raise ConstructorError(None, None, "found unconstructable recursive node", node.start_mark) diff --git a/lib/yaml/emitter.py b/lib/yaml/emitter.py index 4cb2c8a..e5bcdcc 100644 --- a/lib/yaml/emitter.py +++ b/lib/yaml/emitter.py @@ -103,6 +103,11 @@ class Emitter(object): self.analysis = None self.style = None + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + def emit(self, event): self.events.append(event) while not self.need_more_events(): diff --git a/lib/yaml/parser.py b/lib/yaml/parser.py index b6a7416..f9e3057 100644 --- a/lib/yaml/parser.py +++ b/lib/yaml/parser.py @@ -86,6 +86,11 @@ class Parser(object): self.marks = [] self.state = self.parse_stream_start + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + def check_event(self, *choices): # Check the type of the next event. if self.current_event is None: diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index cd2ea5a..2d49ac8 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -22,16 +22,22 @@ def scan(stream, Loader=Loader): Scan a YAML stream and produce scanning tokens. """ loader = Loader(stream) - while loader.check_token(): - yield loader.get_token() + try: + while loader.check_token(): + yield loader.get_token() + finally: + loader.dispose() def parse(stream, Loader=Loader): """ Parse a YAML stream and produce parsing events. """ loader = Loader(stream) - while loader.check_event(): - yield loader.get_event() + try: + while loader.check_event(): + yield loader.get_event() + finally: + loader.dispose() def compose(stream, Loader=Loader): """ @@ -39,7 +45,10 @@ def compose(stream, Loader=Loader): and produce the corresponding representation tree. """ loader = Loader(stream) - return loader.get_single_node() + try: + return loader.get_single_node() + finally: + loader.dispose() def compose_all(stream, Loader=Loader): """ @@ -47,8 +56,11 @@ def compose_all(stream, Loader=Loader): and produce corresponding representation trees. """ loader = Loader(stream) - while loader.check_node(): - yield loader.get_node() + try: + while loader.check_node(): + yield loader.get_node() + finally: + loader.dispose() def load(stream, Loader=Loader): """ @@ -56,7 +68,10 @@ def load(stream, Loader=Loader): and produce the corresponding Python object. """ loader = Loader(stream) - return loader.get_single_data() + try: + return loader.get_single_data() + finally: + loader.dispose() def load_all(stream, Loader=Loader): """ @@ -64,8 +79,11 @@ def load_all(stream, Loader=Loader): and produce corresponding Python objects. """ loader = Loader(stream) - while loader.check_data(): - yield loader.get_data() + try: + while loader.check_data(): + yield loader.get_data() + finally: + loader.dispose() def safe_load(stream): """ @@ -96,8 +114,11 @@ def emit(events, stream=None, Dumper=Dumper, getvalue = stream.getvalue dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, allow_unicode=allow_unicode, line_break=line_break) - for event in events: - dumper.emit(event) + try: + for event in events: + dumper.emit(event) + finally: + dumper.dispose() if getvalue: return getvalue() @@ -121,10 +142,13 @@ def serialize_all(nodes, stream=None, Dumper=Dumper, allow_unicode=allow_unicode, line_break=line_break, encoding=encoding, version=version, tags=tags, explicit_start=explicit_start, explicit_end=explicit_end) - dumper.open() - for node in nodes: - dumper.serialize(node) - dumper.close() + try: + dumper.open() + for node in nodes: + dumper.serialize(node) + dumper.close() + finally: + dumper.dispose() if getvalue: return getvalue() @@ -158,10 +182,13 @@ def dump_all(documents, stream=None, Dumper=Dumper, allow_unicode=allow_unicode, line_break=line_break, encoding=encoding, version=version, tags=tags, explicit_start=explicit_start, explicit_end=explicit_end) - dumper.open() - for data in documents: - dumper.represent(data) - dumper.close() + try: + dumper.open() + for data in documents: + dumper.represent(data) + dumper.close() + finally: + dumper.dispose() if getvalue: return getvalue() diff --git a/lib3/yaml/constructor.py b/lib3/yaml/constructor.py index bd25b79..981543a 100644 --- a/lib3/yaml/constructor.py +++ b/lib3/yaml/constructor.py @@ -51,11 +51,11 @@ class BaseConstructor: return data def construct_object(self, node, deep=False): + if node in self.constructed_objects: + return self.constructed_objects[node] if deep: old_deep = self.deep_construct self.deep_construct = True - if node in self.constructed_objects: - return self.constructed_objects[node] if node in self.recursive_objects: raise ConstructorError(None, None, "found unconstructable recursive node", node.start_mark) diff --git a/lib3/yaml/emitter.py b/lib3/yaml/emitter.py index 8947d04..34cb145 100644 --- a/lib3/yaml/emitter.py +++ b/lib3/yaml/emitter.py @@ -103,6 +103,11 @@ class Emitter: self.analysis = None self.style = None + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + def emit(self, event): self.events.append(event) while not self.need_more_events(): diff --git a/lib3/yaml/parser.py b/lib3/yaml/parser.py index c147dd6..13a5995 100644 --- a/lib3/yaml/parser.py +++ b/lib3/yaml/parser.py @@ -86,6 +86,11 @@ class Parser: self.marks = [] self.state = self.parse_stream_start + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + def check_event(self, *choices): # Check the type of the next event. if self.current_event is None: diff --git a/tests/lib/canonical.py b/tests/lib/canonical.py index 41d111a..020e6db 100644 --- a/tests/lib/canonical.py +++ b/tests/lib/canonical.py @@ -211,6 +211,9 @@ class CanonicalParser: self.events = [] self.parsed = False + def dispose(self): + pass + # stream: STREAM-START document* STREAM-END def parse_stream(self): self.get_token(yaml.StreamStartToken) diff --git a/tests/lib3/canonical.py b/tests/lib3/canonical.py index e04477b..a8b4e3a 100644 --- a/tests/lib3/canonical.py +++ b/tests/lib3/canonical.py @@ -212,6 +212,9 @@ class CanonicalParser: self.events = [] self.parsed = False + def dispose(self): + pass + # stream: STREAM-START document* STREAM-END def parse_stream(self): self.get_token(yaml.StreamStartToken) -- cgit v1.2.1