diff options
| -rw-r--r-- | pygments/lexers/python.py | 27 | ||||
| -rw-r--r-- | tests/snippets/python/test_bytes_escape_codes.txt | 24 | ||||
| -rw-r--r-- | tests/snippets/python/test_string_escape_codes.txt | 20 |
3 files changed, 64 insertions, 7 deletions
diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py index 83c5ef4e..0a7e8f83 100644 --- a/pygments/lexers/python.py +++ b/pygments/lexers/python.py @@ -142,7 +142,7 @@ class PythonLexer(RegexLexer): combined('fstringescape', 'dqf')), ("([fF])(')", bygroups(String.Affix, String.Single), combined('fstringescape', 'sqf')), - # raw strings + # raw bytes and strings ('(?i)(rb|br|r)(""")', bygroups(String.Affix, String.Double), 'tdqs'), ("(?i)(rb|br|r)(''')", @@ -152,14 +152,24 @@ class PythonLexer(RegexLexer): ("(?i)(rb|br|r)(')", bygroups(String.Affix, String.Single), 'sqs'), # non-raw strings - ('([uUbB]?)(""")', bygroups(String.Affix, String.Double), + ('([uU]?)(""")', bygroups(String.Affix, String.Double), combined('stringescape', 'tdqs')), - ("([uUbB]?)(''')", bygroups(String.Affix, String.Single), + ("([uU]?)(''')", bygroups(String.Affix, String.Single), combined('stringescape', 'tsqs')), - ('([uUbB]?)(")', bygroups(String.Affix, String.Double), + ('([uU]?)(")', bygroups(String.Affix, String.Double), combined('stringescape', 'dqs')), - ("([uUbB]?)(')", bygroups(String.Affix, String.Single), + ("([uU]?)(')", bygroups(String.Affix, String.Single), combined('stringescape', 'sqs')), + # non-raw bytes + ('([bB])(""")', bygroups(String.Affix, String.Double), + combined('bytesescape', 'tdqs')), + ("([bB])(''')", bygroups(String.Affix, String.Single), + combined('bytesescape', 'tsqs')), + ('([bB])(")', bygroups(String.Affix, String.Double), + combined('bytesescape', 'dqs')), + ("([bB])(')", bygroups(String.Affix, String.Single), + combined('bytesescape', 'sqs')), + (r'[^\S\n]+', Text), include('numbers'), (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|.]', Operator), @@ -343,9 +353,12 @@ class PythonLexer(RegexLexer): include('rfstringescape'), include('stringescape'), ], + 'bytesescape': [ + (r'\\([\\abfnrtv"\']|\n|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) + ], 'stringescape': [ - (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|' - r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) + (r'\\(N\{.*?\}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8})', String.Escape), + include('bytesescape') ], 'fstrings-single': fstring_rules(String.Single), 'fstrings-double': fstring_rules(String.Double), diff --git a/tests/snippets/python/test_bytes_escape_codes.txt b/tests/snippets/python/test_bytes_escape_codes.txt new file mode 100644 index 00000000..19948d7e --- /dev/null +++ b/tests/snippets/python/test_bytes_escape_codes.txt @@ -0,0 +1,24 @@ +---input--- +b'\\ \n \x12 \777 \u1234 \U00010348 \N{Plus-Minus Sign}' + +---tokens--- +'b' Literal.String.Affix +"'" Literal.String.Single +'\\\\' Literal.String.Escape +' ' Literal.String.Single +'\\n' Literal.String.Escape +' ' Literal.String.Single +'\\x12' Literal.String.Escape +' ' Literal.String.Single +'\\777' Literal.String.Escape +' ' Literal.String.Single +'\\' Literal.String.Single +'u1234 ' Literal.String.Single +'\\' Literal.String.Single +'U00010348 ' Literal.String.Single +'\\' Literal.String.Single +'N' Literal.String.Single +'{' Literal.String.Single +'Plus-Minus Sign}' Literal.String.Single +"'" Literal.String.Single +'\n' Text diff --git a/tests/snippets/python/test_string_escape_codes.txt b/tests/snippets/python/test_string_escape_codes.txt new file mode 100644 index 00000000..5b06329c --- /dev/null +++ b/tests/snippets/python/test_string_escape_codes.txt @@ -0,0 +1,20 @@ +---input--- +'\\ \n \x12 \777 \u1234 \U00010348 \N{Plus-Minus Sign}' + +---tokens--- +"'" Literal.String.Single +'\\\\' Literal.String.Escape +' ' Literal.String.Single +'\\n' Literal.String.Escape +' ' Literal.String.Single +'\\x12' Literal.String.Escape +' ' Literal.String.Single +'\\777' Literal.String.Escape +' ' Literal.String.Single +'\\u1234' Literal.String.Escape +' ' Literal.String.Single +'\\U00010348' Literal.String.Escape +' ' Literal.String.Single +'\\N{Plus-Minus Sign}' Literal.String.Escape +"'" Literal.String.Single +'\n' Text |