summaryrefslogtreecommitdiff
path: root/test/test_readers/test_python
diff options
context:
space:
mode:
Diffstat (limited to 'test/test_readers/test_python')
-rw-r--r--test/test_readers/test_python/__init__.py14
-rwxr-xr-xtest/test_readers/test_python/showast57
-rwxr-xr-xtest/test_readers/test_python/showdoc33
-rwxr-xr-xtest/test_readers/test_python/showparse48
-rwxr-xr-xtest/test_readers/test_python/showtok40
-rwxr-xr-xtest/test_readers/test_python/test_functions.py56
-rwxr-xr-xtest/test_readers/test_python/test_parser.py989
-rwxr-xr-xtest/test_readers/test_python/test_token_parser.py46
8 files changed, 1283 insertions, 0 deletions
diff --git a/test/test_readers/test_python/__init__.py b/test/test_readers/test_python/__init__.py
new file mode 100644
index 000000000..46fc50e06
--- /dev/null
+++ b/test/test_readers/test_python/__init__.py
@@ -0,0 +1,14 @@
+import os
+import os.path
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
+prev = ''
+while sys.path[0] != prev:
+ try:
+ import DocutilsTestSupport
+ break
+ except ImportError:
+ prev = sys.path[0]
+ sys.path[0] = os.path.dirname(prev)
+sys.path.pop(0)
diff --git a/test/test_readers/test_python/showast b/test/test_readers/test_python/showast
new file mode 100755
index 000000000..e7d846307
--- /dev/null
+++ b/test/test_readers/test_python/showast
@@ -0,0 +1,57 @@
+#! /usr/bin/env python
+
+"""
+This is a tool for exploring abstract syntax trees generated by
+``compiler.parse()`` from test data in
+docutils/test/test_readers/test_python/test_parser or stdin.
+
+Usage::
+
+ showast <key> <index>
+
+ showast < <module.py>
+
+Where ``<key>`` is the key to the ``totest`` dictionary, and ``<index>`` is
+the index of the list ``totest[key]``. If no arguments are given, stdin is
+used for input.
+"""
+
+import sys
+import compiler
+from compiler.ast import Node
+import test_parser
+
+def pformat(ast, indent=' ', level=0):
+ assert isinstance(ast, Node), 'ast is not a Node: %r' % (ast,)
+ atts = {}
+ for name, value in vars(ast).items():
+ if not value or isinstance(value, Node):
+ continue
+ if isinstance(value, list):
+ if isinstance(value[0], Node):
+ continue
+ if isinstance(value[0], tuple) and value[0] \
+ and isinstance(value[0][0], Node):
+ continue
+ atts[name] = str(value).encode('unicode-escape')
+ attlist = atts.items()
+ attlist.sort()
+ parts = [ast.__class__.__name__]
+ for name, value in attlist:
+ parts.append('%s="%s"' % (name, value))
+ result = ['%s<%s>\n' % (indent * level, ' '.join(parts))]
+ for node in ast.getChildNodes():
+ result.extend(pformat(node, level=level+1))
+ return result
+
+if len(sys.argv) > 1:
+ key, caseno = sys.argv[1:]
+ print 'totest["%s"][%s][0]:\n' % (key, caseno)
+ input_text = test_parser.totest[key][int(caseno)][0]
+else:
+ input_text = sys.stdin.read()
+print input_text
+module = compiler.parse(input_text)
+print module
+print
+print ''.join(pformat(module)),
diff --git a/test/test_readers/test_python/showdoc b/test/test_readers/test_python/showdoc
new file mode 100755
index 000000000..6461960f8
--- /dev/null
+++ b/test/test_readers/test_python/showdoc
@@ -0,0 +1,33 @@
+#! /usr/bin/env python
+
+"""
+This is a tool for exploring module documentation trees generated by
+``docutils.readers.python.moduleparser.parse_module()`` from test data in
+docutils/test/test_readers/test_python/test_parser or stdin.
+
+Usage::
+
+ showdoc <key> <index>
+
+ showdoc < <module.py>
+
+Where ``<key>`` is the key to the ``totest`` dictionary, and ``<index>`` is
+the index of the list ``totest[key]``. If no arguments are given, stdin is
+used for input.
+"""
+
+import sys
+from docutils.readers.python.moduleparser import parse_module
+import test_parser
+
+if len(sys.argv) > 1:
+ key, caseno = sys.argv[1:]
+ print 'totest["%s"][%s][0]:\n' % (key, caseno)
+ input_text = test_parser.totest[key][int(caseno)][0]
+ input_source = "test_parser.totest['%s'][%s][0]" % (key, caseno)
+else:
+ input_text = sys.stdin.read()
+ input_source = '<stdin>'
+print input_text
+module = parse_module(input_text, input_source)
+print module,
diff --git a/test/test_readers/test_python/showparse b/test/test_readers/test_python/showparse
new file mode 100755
index 000000000..8144256d6
--- /dev/null
+++ b/test/test_readers/test_python/showparse
@@ -0,0 +1,48 @@
+#! /usr/bin/env python
+
+"""
+This is a tool for exploring abstract syntax trees generated by
+``parser.suite()`` from test data in
+docutils/test/test_readers/test_python/test_parser or stdin.
+
+Usage::
+
+ showparse <key> <index>
+
+ showparse < <module.py>
+
+Where ``<key>`` is the key to the ``totest`` dictionary, and ``<index>`` is
+the index of the list ``totest[key]``. If no arguments are given, stdin is
+used for input.
+"""
+
+import sys
+import types
+import parser
+import token
+import symbol
+import pprint
+import test_parser
+
+names = token.tok_name.copy()
+names.update(symbol.sym_name)
+
+def name_elements(ast):
+ if ast:
+ name = names[ast[0]]
+ ast[0] = '%s (%s)' % (name, ast[0])
+ for node in ast[1:]:
+ if type(node) == types.ListType:
+ name_elements(node)
+
+if len(sys.argv) > 1:
+ key, caseno = sys.argv[1:]
+ print 'totest["%s"][%s][0]:\n' % (key, caseno)
+ input_text = test_parser.totest[key][int(caseno)][0]
+else:
+ input_text = sys.stdin.read()
+print input_text
+module = parser.suite(input_text)
+ast = parser.ast2list(module, line_info=1)
+name_elements(ast)
+pprint.pprint(ast)
diff --git a/test/test_readers/test_python/showtok b/test/test_readers/test_python/showtok
new file mode 100755
index 000000000..efd250ce1
--- /dev/null
+++ b/test/test_readers/test_python/showtok
@@ -0,0 +1,40 @@
+#! /usr/bin/env python
+
+
+"""
+This is a tool for exploring token lists generated by
+``tokenize.generate_tokens()`` from test data in
+docutils/test/test_readers/test_python/test_parser or stdin.
+
+Usage::
+
+ showtok <key> <index>
+
+ showtok < <module.py>
+
+Where ``<key>`` is the key to the ``totest`` dictionary, and ``<index>`` is
+the index of the list ``totest[key]``. If no arguments are given, stdin is
+used for input.
+"""
+
+import sys
+import tokenize
+import pprint
+from token import tok_name
+import test_parser
+
+def name_tokens(tokens):
+ for i in range(len(tokens)):
+ tup = tokens[i]
+ tokens[i] = (tok_name[tup[0]], tup)
+
+if len(sys.argv) > 1:
+ key, caseno = sys.argv[1:]
+ print 'totest["%s"][%s][0]:\n' % (key, caseno)
+ input_text = test_parser.totest[key][int(caseno)][0]
+else:
+ input_text = sys.stdin.read()
+print input_text
+tokens = list(tokenize.generate_tokens(iter(input_text.splitlines(1)).next))
+name_tokens(tokens)
+pprint.pprint(tokens)
diff --git a/test/test_readers/test_python/test_functions.py b/test/test_readers/test_python/test_functions.py
new file mode 100755
index 000000000..d521b2203
--- /dev/null
+++ b/test/test_readers/test_python/test_functions.py
@@ -0,0 +1,56 @@
+#! /usr/bin/env python
+
+# Author: David Goodger
+# Contact: goodger@users.sourceforge.net
+# Revision: $Revision$
+# Date: $Date$
+# Copyright: This module has been placed in the public domain.
+
+"""
+Tests for PySource Reader functions.
+"""
+
+import unittest
+from __init__ import DocutilsTestSupport
+from docutils.readers.python.moduleparser import trim_docstring
+
+
+class MiscTests(unittest.TestCase):
+
+ docstrings = (
+ ("""""", """"""), # empty
+ ("""Begins on the first line.
+
+ Middle line indented.
+
+ Last line unindented.
+ """,
+ """\
+Begins on the first line.
+
+ Middle line indented.
+
+Last line unindented."""),
+ ("""
+ Begins on the second line.
+
+ Middle line indented.
+
+ Last line unindented.""",
+ """\
+Begins on the second line.
+
+ Middle line indented.
+
+Last line unindented."""),
+ ("""All on one line.""", """All on one line."""))
+
+ def test_trim_docstring(self):
+ for docstring, expected in self.docstrings:
+ self.assertEquals(trim_docstring(docstring), expected)
+ self.assertEquals(trim_docstring('\n ' + docstring),
+ expected)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/test_readers/test_python/test_parser.py b/test/test_readers/test_python/test_parser.py
new file mode 100755
index 000000000..1ccfbdc10
--- /dev/null
+++ b/test/test_readers/test_python/test_parser.py
@@ -0,0 +1,989 @@
+#! /usr/bin/env python
+
+# Author: David Goodger
+# Contact: goodger@users.sourceforge.net
+# Revision: $Revision$
+# Date: $Date$
+# Copyright: This module has been placed in the public domain.
+
+"""
+Tests for docutils/readers/python/moduleparser.py.
+"""
+
+from __init__ import DocutilsTestSupport
+
+
+def suite():
+ s = DocutilsTestSupport.PythonModuleParserTestSuite()
+ s.generateTests(totest)
+ return s
+
+totest = {}
+
+totest['module'] = [
+['''\
+''',
+'''\
+<module_section filename="test data">
+'''],
+['''\
+"""docstring"""
+''',
+'''\
+<module_section filename="test data">
+ <docstring>
+ docstring
+'''],
+['''\
+u"""Unicode docstring"""
+''',
+'''\
+<module_section filename="test data">
+ <docstring>
+ Unicode docstring
+'''],
+['''\
+"""docstring"""
+"""additional docstring"""
+''',
+'''\
+<module_section filename="test data">
+ <docstring>
+ docstring
+ <docstring lineno="2">
+ additional docstring
+'''],
+['''\
+"""docstring"""
+# comment
+"""additional docstring"""
+''',
+'''\
+<module_section filename="test data">
+ <docstring>
+ docstring
+ <docstring lineno="3">
+ additional docstring
+'''],
+['''\
+"""docstring"""
+1
+"""not an additional docstring"""
+''',
+'''\
+<module_section filename="test data">
+ <docstring>
+ docstring
+'''],
+]
+
+totest['import'] = [
+['''\
+import module
+''',
+'''\
+<module_section filename="test data">
+ <import_group lineno="1">
+ <import_name>
+ module
+'''],
+['''\
+import module as local
+''',
+'''\
+<module_section filename="test data">
+ <import_group lineno="1">
+ <import_name>
+ module
+ <import_alias>
+ local
+'''],
+['''\
+import module.name
+''',
+'''\
+<module_section filename="test data">
+ <import_group lineno="1">
+ <import_name>
+ module.name
+'''],
+['''\
+import module.name as local
+''',
+'''\
+<module_section filename="test data">
+ <import_group lineno="1">
+ <import_name>
+ module.name
+ <import_alias>
+ local
+'''],
+['''\
+import module
+"""not documentable"""
+''',
+'''\
+<module_section filename="test data">
+ <import_group lineno="1">
+ <import_name>
+ module
+'''],
+]
+
+totest['from'] = [
+['''\
+from module import name
+''',
+'''\
+<module_section filename="test data">
+ <import_group lineno="1">
+ <import_from>
+ module
+ <import_name>
+ name
+'''],
+['''\
+from module import name as local
+''',
+'''\
+<module_section filename="test data">
+ <import_group lineno="1">
+ <import_from>
+ module
+ <import_name>
+ name
+ <import_alias>
+ local
+'''],
+['''\
+from module import name1, name2 as local2
+''',
+'''\
+<module_section filename="test data">
+ <import_group lineno="1">
+ <import_from>
+ module
+ <import_name>
+ name1
+ <import_name>
+ name2
+ <import_alias>
+ local2
+'''],
+['''\
+from module.sub import name
+''',
+'''\
+<module_section filename="test data">
+ <import_group lineno="1">
+ <import_from>
+ module.sub
+ <import_name>
+ name
+'''],
+['''\
+from module.sub import name as local
+''',
+'''\
+<module_section filename="test data">
+ <import_group lineno="1">
+ <import_from>
+ module.sub
+ <import_name>
+ name
+ <import_alias>
+ local
+'''],
+['''\
+from module import *
+''',
+'''\
+<module_section filename="test data">
+ <import_group lineno="1">
+ <import_from>
+ module
+ <import_name>
+ *
+'''],
+['''\
+from __future__ import division
+''',
+'''\
+<module_section filename="test data">
+ <import_group lineno="1">
+ <import_from>
+ __future__
+ <import_name>
+ division
+'''],
+]
+
+totest['assign'] = [
+['''\
+a = 1
+''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ a
+ <expression_value>
+ 1
+'''],
+['''a = 1''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ a
+ <expression_value>
+ 1
+'''],
+['''\
+a = 1
+"""a docstring"""
+''', #"
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ a
+ <expression_value>
+ 1
+ <docstring lineno="2">
+ a docstring
+'''],
+['''\
+a = 1
+"""a docstring"""
+"""additional docstring"""
+''', #"
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ a
+ <expression_value>
+ 1
+ <docstring lineno="2">
+ a docstring
+ <docstring lineno="3">
+ additional docstring
+'''], #'
+['''\
+a = 1 + 2 * 3 / 4 ** 5
+''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ a
+ <expression_value>
+ 1 + 2 * 3 / 4 ** 5
+'''],
+['''\
+a = 1 \\
+ + 2
+''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ a
+ <expression_value>
+ 1 + 2
+'''],
+['''\
+a = not 1 and 2 or 3
+''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ a
+ <expression_value>
+ not 1 and 2 or 3
+'''],
+['''\
+a = ~ 1 & 2 | 3 ^ 4
+''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ a
+ <expression_value>
+ ~ 1 & 2 | 3 ^ 4
+'''],
+['''\
+a = `1 & 2`
+''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ a
+ <expression_value>
+ `1 & 2`
+'''],
+['''\
+very_long_name = \\
+ x
+''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ very_long_name
+ <expression_value>
+ x
+'''],
+['''\
+very_long_name \\
+ = x
+''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ very_long_name
+ <expression_value>
+ x
+'''],
+['''\
+very_long_name = \\
+ another_long_name = \\
+ x
+''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ very_long_name
+ <expression_value>
+ x
+ <attribute lineno="2">
+ <object_name>
+ another_long_name
+ <expression_value>
+ x
+'''],
+['''\
+a = (1
+ + 2)
+b = a.b[1 +
+ fn(x, y,
+ z, {'key': (1 + 2
+ + 3)})][4]
+c = """first line
+second line
+ third"""
+''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ a
+ <expression_value>
+ (1 + 2)
+ <attribute lineno="3">
+ <object_name>
+ b
+ <expression_value>
+ a.b[1 + fn(x, y, z, {'key': (1 + 2 + 3)})][4]
+ <attribute lineno="7">
+ <object_name>
+ c
+ <expression_value>
+ """first line
+ second line
+ third"""
+'''],
+['''\
+a, b, c = range(3)
+(d, e,
+ f) = a, b, c
+g, h, i = j = a, b, c
+k.a, k.b.c, k.d.e.f = a, b, c
+''',
+'''\
+<module_section filename="test data">
+ <attribute_tuple lineno="1">
+ <attribute lineno="1">
+ <object_name>
+ a
+ <attribute lineno="1">
+ <object_name>
+ b
+ <attribute lineno="1">
+ <object_name>
+ c
+ <expression_value>
+ range(3)
+ <attribute_tuple lineno="2">
+ <attribute lineno="2">
+ <object_name>
+ d
+ <attribute lineno="2">
+ <object_name>
+ e
+ <attribute lineno="3">
+ <object_name>
+ f
+ <expression_value>
+ a, b, c
+ <attribute_tuple lineno="4">
+ <attribute lineno="4">
+ <object_name>
+ g
+ <attribute lineno="4">
+ <object_name>
+ h
+ <attribute lineno="4">
+ <object_name>
+ i
+ <expression_value>
+ a, b, c
+ <attribute lineno="4">
+ <object_name>
+ j
+ <expression_value>
+ a, b, c
+ <attribute_tuple lineno="5">
+ <attribute lineno="5">
+ <object_name>
+ k.a
+ <attribute lineno="5">
+ <object_name>
+ k.b.c
+ <attribute lineno="5">
+ <object_name>
+ k.d.e.f
+ <expression_value>
+ a, b, c
+'''],
+['''\
+a = 1 ; b = 2
+print ; c = 3
+''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ a
+ <expression_value>
+ 1
+ <attribute lineno="1">
+ <object_name>
+ b
+ <expression_value>
+ 2
+ <attribute lineno="2">
+ <object_name>
+ c
+ <expression_value>
+ 3
+'''],
+['''\
+a.b = 1
+"""This assignment is noted but ignored unless ``a`` is a function."""
+''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ a.b
+ <expression_value>
+ 1
+ <docstring lineno="2">
+ This assignment is noted but ignored unless ``a`` is a function.
+'''],
+['''\
+a[b] = 1
+"""Subscript assignments are ignored."""
+''',
+'''\
+<module_section filename="test data">
+'''],
+['''\
+a = foo(b=1)
+''',
+'''\
+<module_section filename="test data">
+ <attribute lineno="1">
+ <object_name>
+ a
+ <expression_value>
+ foo(b=1)
+'''],
+# ['''\
+# a = 1
+#
+# """Because of the blank above, this is a module docstring."""
+# ''',
+# '''\
+# <module_section filename="test data">
+# <attribute lineno="1">
+# <object_name>
+# a
+# <expression_value>
+# 1
+# <docstring lineno="3">
+# Because of the blank above, this is a module docstring.
+# '''],
+]
+
+totest['def'] = [
+['''\
+def f():
+ """Function f docstring"""
+ """Additional docstring"""
+ local = 1
+ """Not a docstring, since ``local`` is local."""
+''', # "
+'''\
+<module_section filename="test data">
+ <function_section lineno="1">
+ <object_name>
+ f
+ <docstring lineno="1">
+ Function f docstring
+ <docstring lineno="3">
+ Additional docstring
+'''], # '
+['''\
+def f(a, b):
+ local = 1
+''',
+'''\
+<module_section filename="test data">
+ <function_section lineno="1">
+ <object_name>
+ f
+ <parameter_list>
+ <parameter>
+ <object_name>
+ a
+ <parameter>
+ <object_name>
+ b
+'''],
+['''\
+def f(a=None, b=1):
+ local = 1
+''',
+'''\
+<module_section filename="test data">
+ <function_section lineno="1">
+ <object_name>
+ f
+ <parameter_list>
+ <parameter>
+ <object_name>
+ a
+ <parameter_default>
+ None
+ <parameter>
+ <object_name>
+ b
+ <parameter_default>
+ 1
+'''],
+['''\
+def f(a, (b, c, d)=range(3),
+ e=None):
+ local = 1
+''',
+'''\
+<module_section filename="test data">
+ <function_section lineno="1">
+ <object_name>
+ f
+ <parameter_list>
+ <parameter>
+ <object_name>
+ a
+ <parameter_tuple>
+ <parameter>
+ <object_name>
+ b
+ <parameter>
+ <object_name>
+ c
+ <parameter>
+ <object_name>
+ d
+ <parameter_default>
+ range(3)
+ <parameter>
+ <object_name>
+ e
+ <parameter_default>
+ None
+'''],
+['''\
+def f(*args):
+ local = 1
+''',
+'''\
+<module_section filename="test data">
+ <function_section lineno="1">
+ <object_name>
+ f
+ <parameter_list>
+ <parameter excess_positional="1">
+ <object_name>
+ args
+'''],
+['''\
+def f(**kwargs):
+ local = 1
+''',
+'''\
+<module_section filename="test data">
+ <function_section lineno="1">
+ <object_name>
+ f
+ <parameter_list>
+ <parameter excess_keyword="1">
+ <object_name>
+ kwargs
+'''],
+['''\
+def f(a, b=None, *args, **kwargs):
+ local = 1
+''',
+'''\
+<module_section filename="test data">
+ <function_section lineno="1">
+ <object_name>
+ f
+ <parameter_list>
+ <parameter>
+ <object_name>
+ a
+ <parameter>
+ <object_name>
+ b
+ <parameter_default>
+ None
+ <parameter excess_positional="1">
+ <object_name>
+ args
+ <parameter excess_keyword="1">
+ <object_name>
+ kwargs
+'''],
+['''\
+def f():
+ pass
+f.attrib = 1
+"""f.attrib docstring"""
+''', # "
+# @@@ When should the attribute move inside the Function?
+'''\
+<module_section filename="test data">
+ <function_section lineno="1">
+ <object_name>
+ f
+ <attribute lineno="3">
+ <object_name>
+ f.attrib
+ <expression_value>
+ 1
+ <docstring lineno="4">
+ f.attrib docstring
+'''],
+['''\
+def f():
+ def g():
+ pass
+ """Not a docstring"""
+ local = 1
+''',
+'''\
+<module_section filename="test data">
+ <function_section lineno="1">
+ <object_name>
+ f
+'''],
+]
+
+totest['class'] = [
+['''\
+class C:
+ """class C docstring"""
+''',
+'''\
+<module_section filename="test data">
+ <class_section lineno="1">
+ <object_name>
+ C
+ <docstring lineno="1">
+ class C docstring
+'''],
+['''\
+class C(Super):
+ pass
+
+class D(SuperD, package.module.SuperD):
+ pass
+''',
+'''\
+<module_section filename="test data">
+ <class_section lineno="1">
+ <object_name>
+ C
+ <class_base>
+ <object_name>
+ Super
+ <class_section lineno="4">
+ <object_name>
+ D
+ <class_base>
+ <object_name>
+ SuperD
+ <class_base>
+ <object_name>
+ package.module.SuperD
+'''],
+['''\
+class C:
+ class D:
+ pass
+ """Not a docstring"""
+''',
+'''\
+<module_section filename="test data">
+ <class_section lineno="1">
+ <object_name>
+ C
+'''],
+['''\
+class C:
+ def f(self):
+ self.local = 1
+ local = 1
+''',
+'''\
+<module_section filename="test data">
+ <class_section lineno="1">
+ <object_name>
+ C
+ <method_section lineno="2">
+ <object_name>
+ f
+ <parameter_list>
+ <parameter>
+ <object_name>
+ self
+'''],
+['''\
+class C:
+ def __init__(self):
+ self.local = 1
+ local = 1
+''',
+'''\
+<module_section filename="test data">
+ <class_section lineno="1">
+ <object_name>
+ C
+ <method_section lineno="2">
+ <object_name>
+ __init__
+ <parameter_list>
+ <parameter>
+ <object_name>
+ self
+ <attribute lineno="3">
+ <object_name>
+ self.local
+ <expression_value>
+ 1
+ <attribute lineno="4">
+ <object_name>
+ local
+ <expression_value>
+ 1
+'''],
+['''\
+class C:
+ def __init__(self):
+ local = foo(a=1)
+''',
+'''\
+<module_section filename="test data">
+ <class_section lineno="1">
+ <object_name>
+ C
+ <method_section lineno="2">
+ <object_name>
+ __init__
+ <parameter_list>
+ <parameter>
+ <object_name>
+ self
+ <attribute lineno="3">
+ <object_name>
+ local
+ <expression_value>
+ foo(a=1)
+'''],
+]
+
+totest['ignore'] = [
+['''\
+1 + 2
+''',
+'''\
+<module_section filename="test data">
+'''],
+['''\
+del a
+''',
+'''\
+<module_section filename="test data">
+'''],
+]
+
+totest['comments'] = [
+# ['''\
+# # Comment
+# ''',
+# '''\
+# <module_section filename="test data">
+# <Comment lineno="1">
+# # Comment
+# '''],
+]
+
+# @@@ we don't parse comments yet
+totest['everything'] = [
+['''\
+# comment
+
+"""Docstring"""
+
+"""Additional docstring"""
+
+__docformat__ = 'reStructuredText'
+
+a = 1
+"""Attribute docstring"""
+
+class C(Super):
+
+ """C docstring"""
+
+ class_attribute = 1
+ """class_attribute docstring"""
+
+ def __init__(self, text=None):
+ """__init__ docstring"""
+
+ self.instance_attribute = (text * 7
+ + ' whaddyaknow')
+ """instance_attribute docstring"""
+
+
+def f(x, # parameter x
+ y=a*5, # parameter y
+ *args): # parameter args
+ """f docstring"""
+ return [x + item for item in args]
+
+f.function_attribute = 1
+"""f.function_attribute docstring"""
+''',
+'''\
+<module_section filename="test data">
+ <docstring>
+ Docstring
+ <docstring lineno="5">
+ Additional docstring
+ <attribute lineno="7">
+ <object_name>
+ __docformat__
+ <expression_value>
+ 'reStructuredText'
+ <attribute lineno="9">
+ <object_name>
+ a
+ <expression_value>
+ 1
+ <docstring lineno="10">
+ Attribute docstring
+ <class_section lineno="12">
+ <object_name>
+ C
+ <class_base>
+ <object_name>
+ Super
+ <docstring lineno="12">
+ C docstring
+ <attribute lineno="16">
+ <object_name>
+ class_attribute
+ <expression_value>
+ 1
+ <docstring lineno="17">
+ class_attribute docstring
+ <method_section lineno="19">
+ <object_name>
+ __init__
+ <docstring lineno="19">
+ __init__ docstring
+ <parameter_list>
+ <parameter>
+ <object_name>
+ self
+ <parameter>
+ <object_name>
+ text
+ <parameter_default>
+ None
+ <attribute lineno="22">
+ <object_name>
+ self.instance_attribute
+ <expression_value>
+ (text * 7 + ' whaddyaknow')
+ <docstring lineno="24">
+ instance_attribute docstring
+ <function_section lineno="27">
+ <object_name>
+ f
+ <docstring lineno="27">
+ f docstring
+ <parameter_list>
+ <parameter>
+ <object_name>
+ x
+ <parameter>
+ <object_name>
+ y
+ <parameter_default>
+ a * 5
+ <parameter excess_positional="1">
+ <object_name>
+ args
+ <attribute lineno="33">
+ <object_name>
+ f.function_attribute
+ <expression_value>
+ 1
+ <docstring lineno="34">
+ f.function_attribute docstring
+'''],
+]
+
+"""
+['''\
+''',
+'''\
+'''],
+"""
+
+if __name__ == '__main__':
+ import unittest
+ unittest.main(defaultTest='suite')
diff --git a/test/test_readers/test_python/test_token_parser.py b/test/test_readers/test_python/test_token_parser.py
new file mode 100755
index 000000000..23015a7dc
--- /dev/null
+++ b/test/test_readers/test_python/test_token_parser.py
@@ -0,0 +1,46 @@
+#! /usr/bin/env python
+
+# Author: David Goodger
+# Contact: goodger@users.sourceforge.net
+# Revision: $Revision$
+# Date: $Date$
+# Copyright: This module has been placed in the public domain.
+
+"""
+Tests for docutils/readers/python/moduleparser.py.
+"""
+
+from __init__ import DocutilsTestSupport
+
+
+def suite():
+ s = DocutilsTestSupport.PythonModuleParserTestSuite()
+ s.generateTests(totest, testmethod='test_token_parser_rhs')
+ return s
+
+totest = {}
+
+totest['expressions'] = [
+['''a = 1''', '''1'''],
+['''a = b = 1''', '''1'''],
+['''\
+a = (
+ 1 + 2
+ + 3
+ )
+''',
+'''(1 + 2 + 3)'''],
+['''\
+a = """\\
+line one
+line two"""
+''',
+'''"""\\\nline one\nline two"""'''],
+['''a = `1`''', '''`1`'''],
+['''a = `1`+`2`''', '''`1` + `2`'''],
+]
+
+
+if __name__ == '__main__':
+ import unittest
+ unittest.main(defaultTest='suite')