Fix for lexing Python raw f-strings with backslashes (#1683)

* introduce and apply rfstringescape

* add unit test for raw f-strings

* add further tests

* fix comment

2 files changed, 60 insertions, 5 deletions

diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py
index f995d36f..8a9b7239 100644
--- a/pygments/lexers/python.py
+++ b/pygments/lexers/python.py
@@ -122,13 +122,17 @@ class PythonLexer(RegexLexer):
         'expr': [
             # raw f-strings
             ('(?i)(rf|fr)(""")',
-             bygroups(String.Affix, String.Double), 'tdqf'),
+             bygroups(String.Affix, String.Double),
+             combined('rfstringescape', 'tdqf')),
             ("(?i)(rf|fr)(''')",
-             bygroups(String.Affix, String.Single), 'tsqf'),
+             bygroups(String.Affix, String.Single),
+             combined('rfstringescape', 'tsqf')),
             ('(?i)(rf|fr)(")',
-             bygroups(String.Affix, String.Double), 'dqf'),
+             bygroups(String.Affix, String.Double),
+             combined('rfstringescape', 'dqf')),
             ("(?i)(rf|fr)(')",
-             bygroups(String.Affix, String.Single), 'sqf'),
+             bygroups(String.Affix, String.Single),
+             combined('rfstringescape', 'sqf')),
             # non-raw f-strings
             ('([fF])(""")', bygroups(String.Affix, String.Double),
              combined('fstringescape', 'tdqf')),
@@ -316,9 +320,12 @@ class PythonLexer(RegexLexer):
             (uni_name, Name.Namespace),
             default('#pop'),
         ],
-        'fstringescape': [
+        'rfstringescape': [
             (r'\{\{', String.Escape),
             (r'\}\}', String.Escape),
+        ],
+        'fstringescape': [
+            include('rfstringescape'),
             include('stringescape'),
         ],
         'stringescape': [
diff --git a/tests/test_python.py b/tests/test_python.py
index 188044fa..ee36a331 100644
--- a/tests/test_python.py
+++ b/tests/test_python.py
@@ -840,3 +840,51 @@ def test_fstring(lexer3):
             match = pattern.sub(lambda m: rep[m.group(0)], match)
             tokens[i] = (token, match)
         assert list(lexer3.get_tokens(fragment)) == tokens
+
+def test_raw_fstring(lexer3):
+    """
+    Tests that the lexer can parse raw f-strings
+    """
+    # Just raw
+    fragment = r'rf"m_\nu = x"'
+    tokens = [
+        (Token.Literal.String.Affix, 'rf'),
+        (Token.Literal.String.Double, '"'),
+        (Token.Literal.String.Double, 'm_'),
+        (Token.Literal.String.Double, '\\'),
+        (Token.Literal.String.Double, 'nu = x'),
+        (Token.Literal.String.Double, '"'),
+        (Token.Text, '\n')
+    ]
+    # Just f-string
+    fragment = r'f"m_\nu = {x}"'
+    tokens = [
+        (Token.Literal.String.Affix, 'f'),
+        (Token.Literal.String.Double, '"'),
+        (Token.Literal.String.Double, 'm_'),
+        (Token.Literal.String.Escape, '\\n'),
+        (Token.Literal.String.Double, 'u = '),
+        (Token.Literal.String.Interpol, '{'),
+        (Token.Name, 'x'),
+        (Token.Literal.String.Interpol, '}'),
+        (Token.Literal.String.Double, '"'),
+        (Token.Text, '\n'),
+    ]
+    # Raw behavior inside {{...}}
+    fragment = r'rf"m_{{\nu}} = {x}"'
+    tokens = [
+        (Token.Literal.String.Affix, 'rf'),
+        (Token.Literal.String.Double, '"'),
+        (Token.Literal.String.Double, 'm_'),
+        (Token.Literal.String.Escape, '{{'),
+        (Token.Literal.String.Double, '\\'),
+        (Token.Literal.String.Double, 'nu'),
+        (Token.Literal.String.Escape, '}}'),
+        (Token.Literal.String.Double, ' = '),
+        (Token.Literal.String.Interpol, '{'),
+        (Token.Name, 'x'),
+        (Token.Literal.String.Interpol, '}'),
+        (Token.Literal.String.Double, '"'),
+        (Token.Text, '\n'),
+    ]
+    assert list(lexer3.get_tokens(fragment)) == tokens