Fix: pgen2 tokenizer doesn't recognize `...` literal (Ellipsis for py3). Closes #1547

I think pgen2 code derived from lib2to3 package. Basically, the package only support python2 code then it doesn't recognize `...` literal.

2 files changed, 10 insertions, 2 deletions

diff --git a/CHANGES b/CHANGES
index 2f905c05..92837b2c 100644
--- a/CHANGES
+++ b/CHANGES
@@ -215,6 +215,7 @@ Bugs fixed
   :rst:dir:`c:function`. Thanks to Takeshi Komiya.
 * PR#278: Fix section entries were shown twice if toctree has been put under
   only directive. Thanks to Takeshi Komiya.
+* #1547: pgen2 tokenizer doesn't recognize `...` literal (Ellipsis for py3).
 
 Documentation
 -------------
diff --git a/sphinx/pycode/pgen2/tokenize.py b/sphinx/pycode/pgen2/tokenize.py
index f516f78b..d6253505 100644
--- a/sphinx/pycode/pgen2/tokenize.py
+++ b/sphinx/pycode/pgen2/tokenize.py
@@ -33,6 +33,7 @@ __credits__ = \
     'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro'
 
 import string, re
+from six import PY3
 from sphinx.pycode.pgen2.token import *
 from sphinx.pycode.pgen2 import token
 
@@ -84,6 +85,9 @@ Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
 
 Bracket = '[][(){}]'
 Special = group(r'\r?\n', r'[:;.,`@]')
+if PY3:
+    Ellipsis_ = r'\.{3}'
+    Special = group(Ellipsis_, Special)
 Funny = group(Operator, Bracket, Special)
 
 PlainToken = group(Number, Funny, String, Name)
@@ -356,8 +360,9 @@ def generate_tokens(readline):
                 spos, epos, pos = (lnum, start), (lnum, end), end
                 token, initial = line[start:end], line[start]
 
-                if initial in numchars or \
-                   (initial == '.' and token != '.'):      # ordinary number
+                if initial in numchars or (
+                   initial == '.' and token not in ('.', '...')
+                   ):                                      # ordinary number
                     yield (NUMBER, token, spos, epos, line)
                 elif initial in '\r\n':
                     newline = NEWLINE
@@ -393,6 +398,8 @@ def generate_tokens(readline):
                         yield (STRING, token, spos, epos, line)
                 elif initial in namechars:                 # ordinary name
                     yield (NAME, token, spos, epos, line)
+                elif token in ('...',):                    # ordinary name
+                    yield (NAME, token, spos, epos, line)
                 elif initial == '\\':                      # continued stmt
                     # This yield is new; needed for better idempotency:
                     yield (NL, token, spos, (lnum, pos), line)