merge in Ben's bundle with more py3k compatibility

2 files changed, 9 insertions, 1 deletions

diff --git a/sphinx/pycode/__init__.py b/sphinx/pycode/__init__.py
index 17dc6afb..c2086da5 100644
--- a/sphinx/pycode/__init__.py
+++ b/sphinx/pycode/__init__.py
@@ -210,7 +210,10 @@ class ModuleAnalyzer(object):
         if self.parsetree is not None:
             return
         self.tokenize()
-        self.parsetree = pydriver.parse_tokens(self.tokens)
+        try:
+            self.parsetree = pydriver.parse_tokens(self.tokens)
+        except parse.ParseError, err:
+            raise PycodeError('parsing failed', err)
         # find the source code encoding
         encoding = sys.getdefaultencoding()
         comments = self.parsetree.get_prefix()
diff --git a/sphinx/pycode/pgen2/tokenize.py b/sphinx/pycode/pgen2/tokenize.py
index 3a0478e0..4489db89 100644
--- a/sphinx/pycode/pgen2/tokenize.py
+++ b/sphinx/pycode/pgen2/tokenize.py
@@ -274,6 +274,11 @@ def generate_tokens(readline):
             line = readline()
         except StopIteration:
             line = ''
+        # if we are not at the end of the file make sure the
+        # line ends with a newline because the parser depends
+        # on that.
+        if line:
+            line = line.rstrip() + '\n'
         lnum = lnum + 1
         pos, max = 0, len(line)