Refactor PythonConsoleLexer as a DelegatingLexer (#2412)

This is simpler and more reliable than hand-coding the state machine.

Fixes #2411

5 files changed, 70 insertions, 34 deletions

diff --git a/tests/examplefiles/pycon/pycon_ctrlc_traceback.output b/tests/examplefiles/pycon/pycon_ctrlc_traceback.output
index 6bdb3e01..efe5d35e 100644
--- a/tests/examplefiles/pycon/pycon_ctrlc_traceback.output
+++ b/tests/examplefiles/pycon/pycon_ctrlc_traceback.output
@@ -98,7 +98,9 @@
 'Exception'   Generic.Error
 '\n'          Text.Whitespace
 
-'>>>\n'       Generic.Output
+''            Text
+'>>>'         Generic.Prompt
+'\n'          Text.Whitespace
 
 '>>> '        Generic.Prompt
 'while'       Keyword
@@ -115,14 +117,22 @@
 '...'         Generic.Prompt
 '\n'          Text.Whitespace
 
-'^CTraceback (most recent call last):\n' Generic.Output
+'^CTraceback (most recent call last):\n' Generic.Traceback
 
-'  File "<stdin>", line 1, in <module>\n' Generic.Output
+'  File '     Text
+'"<stdin>"'   Name.Builtin
+', line '     Text
+'1'           Literal.Number
+', in '       Text
+'<module>'    Name
+'\n'          Text.Whitespace
 
-'KeyboardInterrupt\n' Name.Class
+'KeyboardInterrupt' Name.Class
+'\n'          Text.Whitespace
 
-'\n'          Generic.Output
+'\n'          Text.Whitespace
 
+''            Text
 '>>> '        Generic.Prompt
 'class'       Keyword
 ' '           Text
@@ -229,11 +239,9 @@
 'first'       Name
 '\n'          Text.Whitespace
 
-'\n'          Generic.Output
-
-'During handling of the above exception, another exception occurred:\n' Generic.Output
+'\n'          Text.Whitespace
 
-'\n'          Generic.Output
+'During handling of the above exception, another exception occurred:\n\n' Generic.Traceback
 
 'Traceback (most recent call last):\n' Generic.Traceback
 
@@ -250,15 +258,15 @@
 'second'      Name
 '\n'          Text.Whitespace
 
-'\n'          Generic.Output
+'\n'          Text.Whitespace
 
+''            Text
 '>>> '        Generic.Prompt
 'x'           Name
 ' '           Text
 '='           Operator
 '\n'          Text.Whitespace
 
-''            Generic.Traceback
 '  File '     Text
 '"<stdin>"'   Name.Builtin
 ', line '     Text
@@ -280,7 +288,9 @@
 'invalid syntax' Name
 '\n'          Text.Whitespace
 
-'>>>\n'       Generic.Output
+''            Text
+'>>>'         Generic.Prompt
+'\n'          Text.Whitespace
 
 '\n'          Generic.Output
 
@@ -360,12 +370,13 @@
 'multi'       Name
 '\n'          Text.Whitespace
 
-'    line\n'  Generic.Output
+'    line\n'  Other
 
-'detail\n'    Generic.Output
+'detail\n'    Other
 
-'\n'          Generic.Output
+'\n'          Text.Whitespace
 
+''            Text
 '>>> '        Generic.Prompt
 'raise'       Keyword
 ' '           Text
@@ -394,12 +405,13 @@
 'multi'       Name
 '\n'          Text.Whitespace
 
-'    line\n'  Generic.Output
+'    line\n'  Other
 
-'detail\n'    Generic.Output
+'detail\n'    Other
 
-'\n'          Generic.Output
+'\n'          Text.Whitespace
 
+''            Text
 '>>> '        Generic.Prompt
 'raise'       Keyword
 ' '           Text
@@ -429,12 +441,13 @@
 'multi'       Name
 '\n'          Text.Whitespace
 
-'    line\n'  Generic.Output
+'    line\n'  Other
 
-'detail\n'    Generic.Output
+'detail\n'    Other
 
-'\n'          Generic.Output
+'\n'          Text.Whitespace
 
+''            Text
 '>>> '        Generic.Prompt
 'raise'       Keyword
 ' '           Text
@@ -465,12 +478,13 @@
 'multi'       Name
 '\n'          Text.Whitespace
 
-'    line\n'  Generic.Output
+'    line\n'  Other
 
-'detail\n'    Generic.Output
+'detail\n'    Other
 
-'\n'          Generic.Output
+'\n'          Text.Whitespace
 
+''            Text
 '>>> '        Generic.Prompt
 'raise'       Keyword
 ' '           Text
@@ -497,12 +511,13 @@
 'multi'       Name
 '\n'          Text.Whitespace
 
-'    line\n'  Generic.Output
+'    line\n'  Other
 
-'detail\n'    Generic.Output
+'detail\n'    Other
 
-'\n'          Generic.Output
+'\n'          Text.Whitespace
 
+''            Text
 '>>> '        Generic.Prompt
 'raise'       Keyword
 ' '           Text
@@ -522,6 +537,7 @@
 'Exception'   Generic.Error
 '\n'          Text.Whitespace
 
+''            Text
 '>>> '        Generic.Prompt
 'import'      Keyword.Namespace
 ' '           Text
diff --git a/tests/examplefiles/pycon/pycon_test.pycon.output b/tests/examplefiles/pycon/pycon_test.pycon.output
index 30946876..66dcca03 100644
--- a/tests/examplefiles/pycon/pycon_test.pycon.output
+++ b/tests/examplefiles/pycon/pycon_test.pycon.output
@@ -2,7 +2,6 @@
 ':'           Punctuation
 '\n'          Text.Whitespace
 
-''            Generic.Traceback
 '  File '     Text
 '"<stdin>"'   Name.Builtin
 ', line '     Text
@@ -22,12 +21,14 @@
 'invalid syntax' Name
 '\n'          Text.Whitespace
 
+''            Text
 '>>> '        Generic.Prompt
 '\n'          Text.Whitespace
 
-'KeyboardInterrupt\n' Name.Class
+'KeyboardInterrupt\n' Generic.Output
 
-'>>>\n'       Generic.Output
+'>>>'         Generic.Prompt
+'\n'          Text.Whitespace
 
 '\n'          Generic.Output
 
@@ -46,8 +47,9 @@
 'ZeroDivisionError' Generic.Error
 '\n'          Text.Whitespace
 
-'\n'          Generic.Output
+'\n'          Text.Whitespace
 
+''            Text
 '>>> '        Generic.Prompt
 '1'           Literal.Number.Integer
 '/'           Operator
diff --git a/tests/snippets/pycon/broken_tb.txt b/tests/snippets/pycon/broken_tb.txt
index 6cecf761..f3f6f5e6 100644
--- a/tests/snippets/pycon/broken_tb.txt
+++ b/tests/snippets/pycon/broken_tb.txt
@@ -54,8 +54,9 @@ SyntaxError: EOL while scanning string literal
 'EOL while scanning string literal' Name
 '\n'          Text.Whitespace
 
-'\n'          Generic.Output
+'\n'          Text.Whitespace
 
+''            Text
 '>>> '        Generic.Prompt
 'exec'        Name
 '('           Punctuation
diff --git a/tests/snippets/pycon/multiple_tb.txt b/tests/snippets/pycon/multiple_tb.txt
index c8b71fc0..9461be0d 100644
--- a/tests/snippets/pycon/multiple_tb.txt
+++ b/tests/snippets/pycon/multiple_tb.txt
@@ -103,6 +103,6 @@ AttributeError: Can't get attribute 'f' on <module '__main__' (<class '_frozen_i
 "Can't get attribute 'f' on <module '__main__' (<class '_frozen_importlib.BuiltinImporter'>)>" Name
 '\n'          Text.Whitespace
 
-"AttributeError: Can't get attribute 'f' on <module '__main__' (<class '_frozen_importlib.BuiltinImporter'>)>\n" Generic.Output
+"AttributeError: Can't get attribute 'f' on <module '__main__' (<class '_frozen_importlib.BuiltinImporter'>)>\n" Other
 
-"AttributeError: Can't get attribute 'f' on <module '__main__' (<class '_frozen_importlib.BuiltinImporter'>)>\n" Generic.Output
+"AttributeError: Can't get attribute 'f' on <module '__main__' (<class '_frozen_importlib.BuiltinImporter'>)>\n" Other
diff --git a/tests/snippets/pycon/unterminated_tb.txt b/tests/snippets/pycon/unterminated_tb.txt
new file mode 100644
index 00000000..9173ec04
--- /dev/null
+++ b/tests/snippets/pycon/unterminated_tb.txt
@@ -0,0 +1,17 @@
+---input---
+>>> unterminated_traceback()
+Traceback (most recent call last):
+>>>
+
+---tokens---
+'>>> '        Generic.Prompt
+'unterminated_traceback' Name
+'('           Punctuation
+')'           Punctuation
+'\n'          Text.Whitespace
+
+'Traceback (most recent call last):\n' Generic.Traceback
+
+''            Text
+'>>>'         Generic.Prompt
+'\n'          Text.Whitespace