Release 0.4: tagging released revisiondocutils-0.4

git-svn-id: http://svn.code.sf.net/p/docutils/code/tags/docutils-0.4@4268 929543f6-e4f2-0310-98a6-ba3bd3dd1d04

8 files changed, 1283 insertions, 0 deletions

diff --git a/test/test_readers/test_python/__init__.py b/test/test_readers/test_python/__init__.py
new file mode 100644
index 000000000..46fc50e06
--- /dev/null
+++ b/test/test_readers/test_python/__init__.py
@@ -0,0 +1,14 @@
+import os
+import os.path
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
+prev = ''
+while sys.path[0] != prev:
+    try:
+        import DocutilsTestSupport
+        break
+    except ImportError:
+        prev = sys.path[0]
+        sys.path[0] = os.path.dirname(prev)
+sys.path.pop(0)
diff --git a/test/test_readers/test_python/showast b/test/test_readers/test_python/showast
new file mode 100755
index 000000000..e7d846307
--- /dev/null
+++ b/test/test_readers/test_python/showast
@@ -0,0 +1,57 @@
+#! /usr/bin/env python
+
+"""
+This is a tool for exploring abstract syntax trees generated by
+``compiler.parse()`` from test data in
+docutils/test/test_readers/test_python/test_parser or stdin.
+
+Usage::
+
+    showast <key> <index>
+
+    showast < <module.py>
+
+Where ``<key>`` is the key to the ``totest`` dictionary, and ``<index>`` is
+the index of the list ``totest[key]``.  If no arguments are given, stdin is
+used for input.
+"""
+
+import sys
+import compiler
+from compiler.ast import Node
+import test_parser
+
+def pformat(ast, indent='    ', level=0):
+    assert isinstance(ast, Node), 'ast is not a Node: %r' % (ast,)
+    atts = {}
+    for name, value in vars(ast).items():
+        if not value or isinstance(value, Node):
+            continue
+        if isinstance(value, list):
+            if isinstance(value[0], Node):
+                continue
+            if isinstance(value[0], tuple) and value[0] \
+                   and isinstance(value[0][0], Node):
+                continue
+        atts[name] = str(value).encode('unicode-escape')
+    attlist = atts.items()
+    attlist.sort()
+    parts = [ast.__class__.__name__]
+    for name, value in attlist:
+        parts.append('%s="%s"' % (name, value))
+    result = ['%s<%s>\n' % (indent * level, ' '.join(parts))]
+    for node in ast.getChildNodes():
+        result.extend(pformat(node, level=level+1))
+    return result
+
+if len(sys.argv) > 1:
+    key, caseno = sys.argv[1:]
+    print 'totest["%s"][%s][0]:\n' % (key, caseno)
+    input_text = test_parser.totest[key][int(caseno)][0]
+else:
+    input_text = sys.stdin.read()
+print input_text
+module = compiler.parse(input_text)
+print module
+print
+print ''.join(pformat(module)),
diff --git a/test/test_readers/test_python/showdoc b/test/test_readers/test_python/showdoc
new file mode 100755
index 000000000..6461960f8
--- /dev/null
+++ b/test/test_readers/test_python/showdoc
@@ -0,0 +1,33 @@
+#! /usr/bin/env python
+
+"""
+This is a tool for exploring module documentation trees generated by
+``docutils.readers.python.moduleparser.parse_module()`` from test data in
+docutils/test/test_readers/test_python/test_parser or stdin.
+
+Usage::
+
+    showdoc <key> <index>
+
+    showdoc < <module.py>
+
+Where ``<key>`` is the key to the ``totest`` dictionary, and ``<index>`` is
+the index of the list ``totest[key]``.  If no arguments are given, stdin is
+used for input.
+"""
+
+import sys
+from docutils.readers.python.moduleparser import parse_module
+import test_parser
+
+if len(sys.argv) > 1:
+    key, caseno = sys.argv[1:]
+    print 'totest["%s"][%s][0]:\n' % (key, caseno)
+    input_text = test_parser.totest[key][int(caseno)][0]
+    input_source = "test_parser.totest['%s'][%s][0]" % (key, caseno)
+else:
+    input_text = sys.stdin.read()
+    input_source = '<stdin>'
+print input_text
+module = parse_module(input_text, input_source)
+print module,
diff --git a/test/test_readers/test_python/showparse b/test/test_readers/test_python/showparse
new file mode 100755
index 000000000..8144256d6
--- /dev/null
+++ b/test/test_readers/test_python/showparse
@@ -0,0 +1,48 @@
+#! /usr/bin/env python
+
+"""
+This is a tool for exploring abstract syntax trees generated by
+``parser.suite()`` from test data in
+docutils/test/test_readers/test_python/test_parser or stdin.
+
+Usage::
+
+    showparse <key> <index>
+
+    showparse < <module.py>
+
+Where ``<key>`` is the key to the ``totest`` dictionary, and ``<index>`` is
+the index of the list ``totest[key]``.  If no arguments are given, stdin is
+used for input.
+"""
+
+import sys
+import types
+import parser
+import token
+import symbol
+import pprint
+import test_parser
+
+names = token.tok_name.copy()
+names.update(symbol.sym_name)
+
+def name_elements(ast):
+    if ast:
+        name = names[ast[0]]
+        ast[0] = '%s (%s)' % (name, ast[0])
+    for node in ast[1:]:
+        if type(node) == types.ListType:
+            name_elements(node)
+
+if len(sys.argv) > 1:
+    key, caseno = sys.argv[1:]
+    print 'totest["%s"][%s][0]:\n' % (key, caseno)
+    input_text = test_parser.totest[key][int(caseno)][0]
+else:
+    input_text = sys.stdin.read()
+print input_text
+module = parser.suite(input_text)
+ast = parser.ast2list(module, line_info=1)
+name_elements(ast)
+pprint.pprint(ast)
diff --git a/test/test_readers/test_python/showtok b/test/test_readers/test_python/showtok
new file mode 100755
index 000000000..efd250ce1
--- /dev/null
+++ b/test/test_readers/test_python/showtok
@@ -0,0 +1,40 @@
+#! /usr/bin/env python
+
+
+"""
+This is a tool for exploring token lists generated by
+``tokenize.generate_tokens()`` from test data in
+docutils/test/test_readers/test_python/test_parser or stdin.
+
+Usage::
+
+    showtok <key> <index>
+
+    showtok < <module.py>
+
+Where ``<key>`` is the key to the ``totest`` dictionary, and ``<index>`` is
+the index of the list ``totest[key]``.  If no arguments are given, stdin is
+used for input.
+"""
+
+import sys
+import tokenize
+import pprint
+from token import tok_name
+import test_parser
+
+def name_tokens(tokens):
+    for i in range(len(tokens)):
+        tup = tokens[i]
+        tokens[i] = (tok_name[tup[0]], tup)
+
+if len(sys.argv) > 1:
+    key, caseno = sys.argv[1:]
+    print 'totest["%s"][%s][0]:\n' % (key, caseno)
+    input_text = test_parser.totest[key][int(caseno)][0]
+else:
+    input_text = sys.stdin.read()
+print input_text
+tokens = list(tokenize.generate_tokens(iter(input_text.splitlines(1)).next))
+name_tokens(tokens)
+pprint.pprint(tokens)
diff --git a/test/test_readers/test_python/test_functions.py b/test/test_readers/test_python/test_functions.py
new file mode 100755
index 000000000..d521b2203
--- /dev/null
+++ b/test/test_readers/test_python/test_functions.py
@@ -0,0 +1,56 @@
+#! /usr/bin/env python
+
+# Author: David Goodger
+# Contact: goodger@users.sourceforge.net
+# Revision: $Revision$
+# Date: $Date$
+# Copyright: This module has been placed in the public domain.
+
+"""
+Tests for PySource Reader functions.
+"""
+
+import unittest
+from __init__ import DocutilsTestSupport
+from docutils.readers.python.moduleparser import trim_docstring
+
+
+class MiscTests(unittest.TestCase):
+
+    docstrings = (
+        ("""""", """"""), # empty
+        ("""Begins on the first line.
+
+        Middle line indented.
+
+    Last line unindented.
+    """,
+         """\
+Begins on the first line.
+
+    Middle line indented.
+
+Last line unindented."""),
+        ("""
+    Begins on the second line.
+
+        Middle line indented.
+
+    Last line unindented.""",
+         """\
+Begins on the second line.
+
+    Middle line indented.
+
+Last line unindented."""),
+        ("""All on one line.""", """All on one line."""))
+
+    def test_trim_docstring(self):
+        for docstring, expected in self.docstrings:
+            self.assertEquals(trim_docstring(docstring), expected)
+            self.assertEquals(trim_docstring('\n    ' + docstring),
+                              expected)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/test_readers/test_python/test_parser.py b/test/test_readers/test_python/test_parser.py
new file mode 100755
index 000000000..1ccfbdc10
--- /dev/null
+++ b/test/test_readers/test_python/test_parser.py
@@ -0,0 +1,989 @@
+#! /usr/bin/env python
+
+# Author: David Goodger
+# Contact: goodger@users.sourceforge.net
+# Revision: $Revision$
+# Date: $Date$
+# Copyright: This module has been placed in the public domain.
+
+"""
+Tests for docutils/readers/python/moduleparser.py.
+"""
+
+from __init__ import DocutilsTestSupport
+
+
+def suite():
+    s = DocutilsTestSupport.PythonModuleParserTestSuite()
+    s.generateTests(totest)
+    return s
+
+totest = {}
+
+totest['module'] = [
+['''\
+''',
+'''\
+<module_section filename="test data">
+'''],
+['''\
+"""docstring"""
+''',
+'''\
+<module_section filename="test data">
+    <docstring>
+        docstring
+'''],
+['''\
+u"""Unicode docstring"""
+''',
+'''\
+<module_section filename="test data">
+    <docstring>
+        Unicode docstring
+'''],
+['''\
+"""docstring"""
+"""additional docstring"""
+''',
+'''\
+<module_section filename="test data">
+    <docstring>
+        docstring
+    <docstring lineno="2">
+        additional docstring
+'''],
+['''\
+"""docstring"""
+# comment
+"""additional docstring"""
+''',
+'''\
+<module_section filename="test data">
+    <docstring>
+        docstring
+    <docstring lineno="3">
+        additional docstring
+'''],
+['''\
+"""docstring"""
+1
+"""not an additional docstring"""
+''',
+'''\
+<module_section filename="test data">
+    <docstring>
+        docstring
+'''],
+]
+
+totest['import'] = [
+['''\
+import module
+''',
+'''\
+<module_section filename="test data">
+    <import_group lineno="1">
+        <import_name>
+            module
+'''],
+['''\
+import module as local
+''',
+'''\
+<module_section filename="test data">
+    <import_group lineno="1">
+        <import_name>
+            module
+            <import_alias>
+                local
+'''],
+['''\
+import module.name
+''',
+'''\
+<module_section filename="test data">
+    <import_group lineno="1">
+        <import_name>
+            module.name
+'''],
+['''\
+import module.name as local
+''',
+'''\
+<module_section filename="test data">
+    <import_group lineno="1">
+        <import_name>
+            module.name
+            <import_alias>
+                local
+'''],
+['''\
+import module
+"""not documentable"""
+''',
+'''\
+<module_section filename="test data">
+    <import_group lineno="1">
+        <import_name>
+            module
+'''],
+]
+
+totest['from'] = [
+['''\
+from module import name
+''',
+'''\
+<module_section filename="test data">
+    <import_group lineno="1">
+        <import_from>
+            module
+        <import_name>
+            name
+'''],
+['''\
+from module import name as local
+''',
+'''\
+<module_section filename="test data">
+    <import_group lineno="1">
+        <import_from>
+            module
+        <import_name>
+            name
+            <import_alias>
+                local
+'''],
+['''\
+from module import name1, name2 as local2
+''',
+'''\
+<module_section filename="test data">
+    <import_group lineno="1">
+        <import_from>
+            module
+        <import_name>
+            name1
+        <import_name>
+            name2
+            <import_alias>
+                local2
+'''],
+['''\
+from module.sub import name
+''',
+'''\
+<module_section filename="test data">
+    <import_group lineno="1">
+        <import_from>
+            module.sub
+        <import_name>
+            name
+'''],
+['''\
+from module.sub import name as local
+''',
+'''\
+<module_section filename="test data">
+    <import_group lineno="1">
+        <import_from>
+            module.sub
+        <import_name>
+            name
+            <import_alias>
+                local
+'''],
+['''\
+from module import *
+''',
+'''\
+<module_section filename="test data">
+    <import_group lineno="1">
+        <import_from>
+            module
+        <import_name>
+            *
+'''],
+['''\
+from __future__ import division
+''',
+'''\
+<module_section filename="test data">
+    <import_group lineno="1">
+        <import_from>
+            __future__
+        <import_name>
+            division
+'''],
+]
+
+totest['assign'] = [
+['''\
+a = 1
+''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            a
+        <expression_value>
+            1
+'''],
+['''a = 1''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            a
+        <expression_value>
+            1
+'''],
+['''\
+a = 1
+"""a docstring"""
+''', #"
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            a
+        <expression_value>
+            1
+        <docstring lineno="2">
+            a docstring
+'''],
+['''\
+a = 1
+"""a docstring"""
+"""additional docstring"""
+''', #"
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            a
+        <expression_value>
+            1
+        <docstring lineno="2">
+            a docstring
+        <docstring lineno="3">
+            additional docstring
+'''], #'
+['''\
+a = 1 + 2 * 3 / 4 ** 5
+''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            a
+        <expression_value>
+            1 + 2 * 3 / 4 ** 5
+'''],
+['''\
+a = 1 \\
+    + 2
+''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            a
+        <expression_value>
+            1 + 2
+'''],
+['''\
+a = not 1 and 2 or 3
+''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            a
+        <expression_value>
+            not 1 and 2 or 3
+'''],
+['''\
+a = ~ 1 & 2 | 3 ^ 4
+''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            a
+        <expression_value>
+            ~ 1 & 2 | 3 ^ 4
+'''],
+['''\
+a = `1 & 2`
+''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            a
+        <expression_value>
+            `1 & 2`
+'''],
+['''\
+very_long_name = \\
+    x
+''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            very_long_name
+        <expression_value>
+            x
+'''],
+['''\
+very_long_name \\
+    = x
+''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            very_long_name
+        <expression_value>
+            x
+'''],
+['''\
+very_long_name = \\
+    another_long_name = \\
+    x
+''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            very_long_name
+        <expression_value>
+            x
+    <attribute lineno="2">
+        <object_name>
+            another_long_name
+        <expression_value>
+            x
+'''],
+['''\
+a = (1
+     + 2)
+b = a.b[1 +
+        fn(x, y,
+           z, {'key': (1 + 2
+                       + 3)})][4]
+c = """first line
+second line
+    third"""
+''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            a
+        <expression_value>
+            (1 + 2)
+    <attribute lineno="3">
+        <object_name>
+            b
+        <expression_value>
+            a.b[1 + fn(x, y, z, {'key': (1 + 2 + 3)})][4]
+    <attribute lineno="7">
+        <object_name>
+            c
+        <expression_value>
+            """first line
+            second line
+                third"""
+'''],
+['''\
+a, b, c = range(3)
+(d, e,
+ f) = a, b, c
+g, h, i = j = a, b, c
+k.a, k.b.c, k.d.e.f = a, b, c
+''',
+'''\
+<module_section filename="test data">
+    <attribute_tuple lineno="1">
+        <attribute lineno="1">
+            <object_name>
+                a
+        <attribute lineno="1">
+            <object_name>
+                b
+        <attribute lineno="1">
+            <object_name>
+                c
+        <expression_value>
+            range(3)
+    <attribute_tuple lineno="2">
+        <attribute lineno="2">
+            <object_name>
+                d
+        <attribute lineno="2">
+            <object_name>
+                e
+        <attribute lineno="3">
+            <object_name>
+                f
+        <expression_value>
+            a, b, c
+    <attribute_tuple lineno="4">
+        <attribute lineno="4">
+            <object_name>
+                g
+        <attribute lineno="4">
+            <object_name>
+                h
+        <attribute lineno="4">
+            <object_name>
+                i
+        <expression_value>
+            a, b, c
+    <attribute lineno="4">
+        <object_name>
+            j
+        <expression_value>
+            a, b, c
+    <attribute_tuple lineno="5">
+        <attribute lineno="5">
+            <object_name>
+                k.a
+        <attribute lineno="5">
+            <object_name>
+                k.b.c
+        <attribute lineno="5">
+            <object_name>
+                k.d.e.f
+        <expression_value>
+            a, b, c
+'''],
+['''\
+a = 1 ; b = 2
+print ; c = 3
+''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            a
+        <expression_value>
+            1
+    <attribute lineno="1">
+        <object_name>
+            b
+        <expression_value>
+            2
+    <attribute lineno="2">
+        <object_name>
+            c
+        <expression_value>
+            3
+'''],
+['''\
+a.b = 1
+"""This assignment is noted but ignored unless ``a`` is a function."""
+''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            a.b
+        <expression_value>
+            1
+        <docstring lineno="2">
+            This assignment is noted but ignored unless ``a`` is a function.
+'''],
+['''\
+a[b] = 1
+"""Subscript assignments are ignored."""
+''',
+'''\
+<module_section filename="test data">
+'''],
+['''\
+a = foo(b=1)
+''',
+'''\
+<module_section filename="test data">
+    <attribute lineno="1">
+        <object_name>
+            a
+        <expression_value>
+            foo(b=1)
+'''],
+# ['''\
+# a = 1
+# 
+# """Because of the blank above, this is a module docstring."""
+# ''',
+# '''\
+# <module_section filename="test data">
+#     <attribute lineno="1">
+#         <object_name>
+#             a
+#         <expression_value>
+#             1
+#     <docstring lineno="3">
+#         Because of the blank above, this is a module docstring.
+# '''],
+]
+
+totest['def'] = [
+['''\
+def f():
+    """Function f docstring"""
+    """Additional docstring"""
+    local = 1
+    """Not a docstring, since ``local`` is local."""
+''', # "
+'''\
+<module_section filename="test data">
+    <function_section lineno="1">
+        <object_name>
+            f
+        <docstring lineno="1">
+            Function f docstring
+        <docstring lineno="3">
+            Additional docstring
+'''], # '
+['''\
+def f(a, b):
+    local = 1
+''',
+'''\
+<module_section filename="test data">
+    <function_section lineno="1">
+        <object_name>
+            f
+        <parameter_list>
+            <parameter>
+                <object_name>
+                    a
+            <parameter>
+                <object_name>
+                    b
+'''],
+['''\
+def f(a=None, b=1):
+    local = 1
+''',
+'''\
+<module_section filename="test data">
+    <function_section lineno="1">
+        <object_name>
+            f
+        <parameter_list>
+            <parameter>
+                <object_name>
+                    a
+                <parameter_default>
+                    None
+            <parameter>
+                <object_name>
+                    b
+                <parameter_default>
+                    1
+'''],
+['''\
+def f(a, (b, c, d)=range(3),
+      e=None):
+    local = 1
+''',
+'''\
+<module_section filename="test data">
+    <function_section lineno="1">
+        <object_name>
+            f
+        <parameter_list>
+            <parameter>
+                <object_name>
+                    a
+            <parameter_tuple>
+                <parameter>
+                    <object_name>
+                        b
+                <parameter>
+                    <object_name>
+                        c
+                <parameter>
+                    <object_name>
+                        d
+                <parameter_default>
+                    range(3)
+            <parameter>
+                <object_name>
+                    e
+                <parameter_default>
+                    None
+'''],
+['''\
+def f(*args):
+    local = 1
+''',
+'''\
+<module_section filename="test data">
+    <function_section lineno="1">
+        <object_name>
+            f
+        <parameter_list>
+            <parameter excess_positional="1">
+                <object_name>
+                    args
+'''],
+['''\
+def f(**kwargs):
+    local = 1
+''',
+'''\
+<module_section filename="test data">
+    <function_section lineno="1">
+        <object_name>
+            f
+        <parameter_list>
+            <parameter excess_keyword="1">
+                <object_name>
+                    kwargs
+'''],
+['''\
+def f(a, b=None, *args, **kwargs):
+    local = 1
+''',
+'''\
+<module_section filename="test data">
+    <function_section lineno="1">
+        <object_name>
+            f
+        <parameter_list>
+            <parameter>
+                <object_name>
+                    a
+            <parameter>
+                <object_name>
+                    b
+                <parameter_default>
+                    None
+            <parameter excess_positional="1">
+                <object_name>
+                    args
+            <parameter excess_keyword="1">
+                <object_name>
+                    kwargs
+'''],
+['''\
+def f():
+    pass
+f.attrib = 1
+"""f.attrib docstring"""
+''', # "
+# @@@ When should the attribute move inside the Function?
+'''\
+<module_section filename="test data">
+    <function_section lineno="1">
+        <object_name>
+            f
+    <attribute lineno="3">
+        <object_name>
+            f.attrib
+        <expression_value>
+            1
+        <docstring lineno="4">
+            f.attrib docstring
+'''], 
+['''\
+def f():
+    def g():
+        pass
+    """Not a docstring"""
+    local = 1
+''',
+'''\
+<module_section filename="test data">
+    <function_section lineno="1">
+        <object_name>
+            f
+'''],
+]
+
+totest['class'] = [
+['''\
+class C:
+    """class C docstring"""
+''',
+'''\
+<module_section filename="test data">
+    <class_section lineno="1">
+        <object_name>
+            C
+        <docstring lineno="1">
+            class C docstring
+'''],
+['''\
+class C(Super):
+    pass
+
+class D(SuperD, package.module.SuperD):
+    pass
+''',
+'''\
+<module_section filename="test data">
+    <class_section lineno="1">
+        <object_name>
+            C
+        <class_base>
+            <object_name>
+                Super
+    <class_section lineno="4">
+        <object_name>
+            D
+        <class_base>
+            <object_name>
+                SuperD
+        <class_base>
+            <object_name>
+                package.module.SuperD
+'''],
+['''\
+class C:
+    class D:
+        pass
+    """Not a docstring"""
+''',
+'''\
+<module_section filename="test data">
+    <class_section lineno="1">
+        <object_name>
+            C
+'''],
+['''\
+class C:
+    def f(self):
+        self.local = 1
+        local = 1
+''',
+'''\
+<module_section filename="test data">
+    <class_section lineno="1">
+        <object_name>
+            C
+        <method_section lineno="2">
+            <object_name>
+                f
+            <parameter_list>
+                <parameter>
+                    <object_name>
+                        self
+'''],
+['''\
+class C:
+    def __init__(self):
+        self.local = 1
+        local = 1
+''',
+'''\
+<module_section filename="test data">
+    <class_section lineno="1">
+        <object_name>
+            C
+        <method_section lineno="2">
+            <object_name>
+                __init__
+            <parameter_list>
+                <parameter>
+                    <object_name>
+                        self
+            <attribute lineno="3">
+                <object_name>
+                    self.local
+                <expression_value>
+                    1
+            <attribute lineno="4">
+                <object_name>
+                    local
+                <expression_value>
+                    1
+'''],
+['''\
+class C:
+    def __init__(self):
+        local = foo(a=1)
+''',
+'''\
+<module_section filename="test data">
+    <class_section lineno="1">
+        <object_name>
+            C
+        <method_section lineno="2">
+            <object_name>
+                __init__
+            <parameter_list>
+                <parameter>
+                    <object_name>
+                        self
+            <attribute lineno="3">
+                <object_name>
+                    local
+                <expression_value>
+                    foo(a=1)
+'''],
+]
+
+totest['ignore'] = [
+['''\
+1 + 2
+''',
+'''\
+<module_section filename="test data">
+'''],
+['''\
+del a
+''',
+'''\
+<module_section filename="test data">
+'''],
+]
+
+totest['comments'] = [
+# ['''\
+# # Comment
+# ''',
+# '''\
+# <module_section filename="test data">
+#     <Comment lineno="1">
+#         # Comment
+# '''],
+]
+
+# @@@ we don't parse comments yet
+totest['everything'] = [
+['''\
+# comment
+
+"""Docstring"""
+
+"""Additional docstring"""
+
+__docformat__ = 'reStructuredText'
+
+a = 1
+"""Attribute docstring"""
+
+class C(Super):
+
+    """C docstring"""
+
+    class_attribute = 1
+    """class_attribute docstring"""
+
+    def __init__(self, text=None):
+        """__init__ docstring"""
+
+        self.instance_attribute = (text * 7
+                                   + ' whaddyaknow')
+        """instance_attribute docstring"""
+
+
+def f(x,                            # parameter x
+      y=a*5,                        # parameter y
+      *args):                       # parameter args
+    """f docstring"""
+    return [x + item for item in args]
+
+f.function_attribute = 1
+"""f.function_attribute docstring"""
+''',
+'''\
+<module_section filename="test data">
+    <docstring>
+        Docstring
+    <docstring lineno="5">
+        Additional docstring
+    <attribute lineno="7">
+        <object_name>
+            __docformat__
+        <expression_value>
+            'reStructuredText'
+    <attribute lineno="9">
+        <object_name>
+            a
+        <expression_value>
+            1
+        <docstring lineno="10">
+            Attribute docstring
+    <class_section lineno="12">
+        <object_name>
+            C
+        <class_base>
+            <object_name>
+                Super
+        <docstring lineno="12">
+            C docstring
+        <attribute lineno="16">
+            <object_name>
+                class_attribute
+            <expression_value>
+                1
+            <docstring lineno="17">
+                class_attribute docstring
+        <method_section lineno="19">
+            <object_name>
+                __init__
+            <docstring lineno="19">
+                __init__ docstring
+            <parameter_list>
+                <parameter>
+                    <object_name>
+                        self
+                <parameter>
+                    <object_name>
+                        text
+                    <parameter_default>
+                        None
+            <attribute lineno="22">
+                <object_name>
+                    self.instance_attribute
+                <expression_value>
+                    (text * 7 + ' whaddyaknow')
+                <docstring lineno="24">
+                    instance_attribute docstring
+    <function_section lineno="27">
+        <object_name>
+            f
+        <docstring lineno="27">
+            f docstring
+        <parameter_list>
+            <parameter>
+                <object_name>
+                    x
+            <parameter>
+                <object_name>
+                    y
+                <parameter_default>
+                    a * 5
+            <parameter excess_positional="1">
+                <object_name>
+                    args
+    <attribute lineno="33">
+        <object_name>
+            f.function_attribute
+        <expression_value>
+            1
+        <docstring lineno="34">
+            f.function_attribute docstring
+'''],
+]
+
+"""
+['''\
+''',
+'''\
+'''],
+"""
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main(defaultTest='suite')
diff --git a/test/test_readers/test_python/test_token_parser.py b/test/test_readers/test_python/test_token_parser.py
new file mode 100755
index 000000000..23015a7dc
--- /dev/null
+++ b/test/test_readers/test_python/test_token_parser.py
@@ -0,0 +1,46 @@
+#! /usr/bin/env python
+
+# Author: David Goodger
+# Contact: goodger@users.sourceforge.net
+# Revision: $Revision$
+# Date: $Date$
+# Copyright: This module has been placed in the public domain.
+
+"""
+Tests for docutils/readers/python/moduleparser.py.
+"""
+
+from __init__ import DocutilsTestSupport
+
+
+def suite():
+    s = DocutilsTestSupport.PythonModuleParserTestSuite()
+    s.generateTests(totest, testmethod='test_token_parser_rhs')
+    return s
+
+totest = {}
+
+totest['expressions'] = [
+['''a = 1''', '''1'''],
+['''a = b = 1''', '''1'''],
+['''\
+a = (
+     1 + 2
+     + 3
+     )
+''',
+'''(1 + 2 + 3)'''],
+['''\
+a = """\\
+line one
+line two"""
+''',
+'''"""\\\nline one\nline two"""'''],
+['''a = `1`''', '''`1`'''],
+['''a = `1`+`2`''', '''`1` + `2`'''],
+]
+
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main(defaultTest='suite')