Support error locations in Python tracebacks (#1852)

Support both single carets for syntax errors (Python 2 and 3)
and fine-grained error locations with several carets (Python 3.11+).
Previously, the carets were highlighted as operators.  This uses
a new token, Token.Punctuation.Marker.  For now, no style supports
it specifically.  In the future, styles might start differentiating
it from Token.Punctuation.

[Closes #1850.]

1 files changed, 16 insertions, 2 deletions

diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py
index 145beede..9db7a2d8 100644
--- a/pygments/lexers/python.py
+++ b/pygments/lexers/python.py
@@ -727,7 +727,7 @@ class PythonTracebackLexer(RegexLexer):
             (r'^(  File )("[^"]+")(, line )(\d+)(\n)',
              bygroups(Text, Name.Builtin, Text, Number, Text)),
             (r'^(    )(.+)(\n)',
-             bygroups(Text, using(PythonLexer), Text)),
+             bygroups(Text, using(PythonLexer), Text), 'markers'),
             (r'^([ \t]*)(\.\.\.)(\n)',
              bygroups(Text, Comment, Text)),  # for doctests...
             (r'^([^:]+)(: )(.+)(\n)',
@@ -735,6 +735,15 @@ class PythonTracebackLexer(RegexLexer):
             (r'^([a-zA-Z_][\w.]*)(:?\n)',
              bygroups(Generic.Error, Text), '#pop')
         ],
+        'markers': [
+            # Either `PEP 657 <https://www.python.org/dev/peps/pep-0657/>`
+            # error locations in Python 3.11+, or single-caret markers
+            # for syntax errors before that.
+            (r'^( {4,})(\^+)(\n)',
+             bygroups(Text, Punctuation.Marker, Text),
+             '#pop'),
+            default('#pop'),
+        ],
     }
 
 
@@ -773,7 +782,7 @@ class Python2TracebackLexer(RegexLexer):
             (r'^(  File )("[^"]+")(, line )(\d+)(\n)',
              bygroups(Text, Name.Builtin, Text, Number, Text)),
             (r'^(    )(.+)(\n)',
-             bygroups(Text, using(Python2Lexer), Text)),
+             bygroups(Text, using(Python2Lexer), Text), 'marker'),
             (r'^([ \t]*)(\.\.\.)(\n)',
              bygroups(Text, Comment, Text)),  # for doctests...
             (r'^([^:]+)(: )(.+)(\n)',
@@ -781,6 +790,11 @@ class Python2TracebackLexer(RegexLexer):
             (r'^([a-zA-Z_]\w*)(:?\n)',
              bygroups(Generic.Error, Text), '#pop')
         ],
+        'marker': [
+            # For syntax errors.
+            (r'( {4,})(\^)', bygroups(Text, Punctuation.Marker), '#pop'),
+            default('#pop'),
+        ],
     }