Python f-strings: highlight expressions in curly braces

Fixes #1228

1 files changed, 682 insertions, 0 deletions

diff --git a/tests/test_python.py b/tests/test_python.py
index b0922adf..23bb6ed1 100644
--- a/tests/test_python.py
+++ b/tests/test_python.py
@@ -12,6 +12,8 @@ import pytest
 from pygments.lexers import PythonLexer, Python3Lexer
 from pygments.token import Token
 
+import re
+
 
 @pytest.fixture(scope='module')
 def lexer2():
@@ -158,3 +160,683 @@ def test_walrus_operator(lexer3):
         (Token.Text, '\n'),
     ]
     assert list(lexer3.get_tokens(fragment)) == tokens
+
+
+def test_fstring(lexer3):
+    """
+    Tests that the lexer can parse f-strings
+    """
+    fragments_and_tokens = (
+        # examples from PEP-0498
+        (
+            "f'My name is {name}, my age next year is {age+1}, my anniversary is {anniversary:%A, %B %d, %Y}.'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'My name is '),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'name'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, ', my age next year is '),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'age'),
+                (Token.Operator, '+'),
+                (Token.Literal.Number.Integer, '1'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, ', my anniversary is '),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'anniversary'),
+                (Token.Literal.String.Interpol, ':'),
+                (Token.Literal.String.Single, '%A, %B %d, %Y'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, '.'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, u'\n')
+            ]
+        ), (
+            "f'He said his name is {name!r}.'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'He said his name is '),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'name'),
+                (Token.Literal.String.Interpol, '!r}'),
+                (Token.Literal.String.Single, '.'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+
+        ), (
+            "f'input={value:#06x}'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'input='),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'value'),
+                (Token.Literal.String.Interpol, ':'),
+                (Token.Literal.String.Single, '#06x'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            """f'{"quoted string"}'\n""",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Literal.String.Double, 'quoted string'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            """f'{f"{inner}"}'\n""",  # not in the PEP
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'inner'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            # SyntaxError: f-string expression part cannot include a backslash
+            "f'{\\'quoted string\\'}'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Error, '\\'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'quoted string'),
+                (Token.Literal.String.Escape, "\\'"),
+                (Token.Literal.String.Single, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'{{ {4*10} }}'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Escape, '{{'),
+                (Token.Literal.String.Single, ' '),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Literal.Number.Integer, '4'),
+                (Token.Operator, '*'),
+                (Token.Literal.Number.Integer, '10'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, ' '),
+                (Token.Literal.String.Escape, '}}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'{{{4*10}}}'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Escape, '{{'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Literal.Number.Integer, '4'),
+                (Token.Operator, '*'),
+                (Token.Literal.Number.Integer, '10'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Escape, '}}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "fr'x={4*10}'\n",
+            [
+                (Token.Literal.String.Affix, 'fr'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, "x="),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Literal.Number.Integer, '4'),
+                (Token.Operator, '*'),
+                (Token.Literal.Number.Integer, '10'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            """f'abc {a["x"]} def'\n""",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'abc '),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'a'),
+                (Token.Punctuation, '['),
+                (Token.Literal.String.Double, '"'),
+                (Token.Literal.String.Double, 'x'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Punctuation, ']'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, ' def'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'''abc {a['x']} def'''\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'''"),
+                (Token.Literal.String.Single, 'abc '),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'a'),
+                (Token.Punctuation, '['),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'x'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Punctuation, ']'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, ' def'),
+                (Token.Literal.String.Single, "'''"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            """f'''{x
++1}'''\n""",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'''"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'x'),
+                (Token.Text, '\n'),
+                (Token.Operator, '+'),
+                (Token.Literal.Number.Integer, '1'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'''"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            """f'''{d[0
+]}'''\n""",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'''"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'd'),
+                (Token.Punctuation, '['),
+                (Token.Literal.Number.Integer, '0'),
+                (Token.Text, '\n'),
+                (Token.Punctuation, ']'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'''"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'result: {value:{width}.{precision}}'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'result: '),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'value'),
+                (Token.Literal.String.Interpol, ':'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'width'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, '.'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'precision'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "'a' 'b' f'{x}' '{c}' f'str<{y:^4}>' 'd' 'e'\n",
+            [
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'a'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, ' '),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'b'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, ' '),
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'x'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, ' '),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, '{c}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, ' '),
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'str<'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'y'),
+                (Token.Literal.String.Interpol, ':'),
+                (Token.Literal.String.Single, '^4'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, '>'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, ' '),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'd'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, ' '),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'e'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'{i}:{d[i]}'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'i'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, ':'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'd'),
+                (Token.Punctuation, '['),
+                (Token.Name, 'i'),
+                (Token.Punctuation, ']'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'x = {x:+3}'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, "x = "),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'x'),
+                (Token.Literal.String.Interpol, ':'),
+                (Token.Literal.String.Single, '+3'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'{fn(lst,2)} {fn(lst,3)}'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'fn'),
+                (Token.Punctuation, '('),
+                (Token.Name, 'lst'),
+                (Token.Punctuation, ','),
+                (Token.Literal.Number.Integer, '2'),
+                (Token.Punctuation, ')'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, ' '),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'fn'),
+                (Token.Punctuation, '('),
+                (Token.Name, 'lst'),
+                (Token.Punctuation, ','),
+                (Token.Literal.Number.Integer, '3'),
+                (Token.Punctuation, ')'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'mapping is { {a:b for (a, b) in ((1, 2), (3, 4))} }'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'mapping is '),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Text, ' '),
+                (Token.Punctuation, '{'),
+                (Token.Name, 'a'),
+                (Token.Punctuation, ':'),
+                (Token.Name, 'b'),
+                (Token.Text, ' '),
+                (Token.Keyword, 'for'),
+                (Token.Text, ' '),
+                (Token.Punctuation, '('),
+                (Token.Name, 'a'),
+                (Token.Punctuation, ','),
+                (Token.Text, ' '),
+                (Token.Name, 'b'),
+                (Token.Punctuation, ')'),
+                (Token.Text, ' '),
+                (Token.Operator.Word, 'in'),
+                (Token.Text, ' '),
+                (Token.Punctuation, '('),
+                (Token.Punctuation, '('),
+                (Token.Literal.Number.Integer, '1'),
+                (Token.Punctuation, ','),
+                (Token.Text, ' '),
+                (Token.Literal.Number.Integer, '2'),
+                (Token.Punctuation, ')'),
+                (Token.Punctuation, ','),
+                (Token.Text, ' '),
+                (Token.Punctuation, '('),
+                (Token.Literal.Number.Integer, '3'),
+                (Token.Punctuation, ','),
+                (Token.Text, ' '),
+                (Token.Literal.Number.Integer, '4'),
+                (Token.Punctuation, ')'),
+                (Token.Punctuation, ')'),
+                (Token.Punctuation, '}'),
+                (Token.Text, ' '),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            """f'a={d["a"]}'\n""",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'a='),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'd'),
+                (Token.Punctuation, '['),
+                (Token.Literal.String.Double, '"'),
+                (Token.Literal.String.Double, 'a'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Punctuation, ']'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'a={d[a]}'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, 'a='),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'd'),
+                (Token.Punctuation, '['),
+                (Token.Name, 'a'),
+                (Token.Punctuation, ']'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "fr'{header}:\\s+'\n",
+            [
+                (Token.Literal.String.Affix, 'fr'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'header'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, ':'),
+                (Token.Literal.String.Single, '\\'),
+                (Token.Literal.String.Single, 's+'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'{a!r}'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'a'),
+                (Token.Literal.String.Interpol, '!r}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'{(lambda x: x*2)(3)}'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Punctuation, '('),
+                (Token.Keyword, 'lambda'),
+                (Token.Text, ' '),
+                (Token.Name, 'x'),
+                (Token.Punctuation, ':'),
+                (Token.Text, ' '),
+                (Token.Name, 'x'),
+                (Token.Operator, '*'),
+                (Token.Literal.Number.Integer, '2'),
+                (Token.Punctuation, ')'),
+                (Token.Punctuation, '('),
+                (Token.Literal.Number.Integer, '3'),
+                (Token.Punctuation, ')'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "extra = f'{extra},waiters:{len(self._waiters)}'\n",
+            [
+                (Token.Name, 'extra'),
+                (Token.Text, ' '),
+                (Token.Operator, '='),
+                (Token.Text, ' '),
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'extra'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, ',waiters:'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name.Builtin, 'len'),
+                (Token.Punctuation, '('),
+                (Token.Name.Builtin.Pseudo, 'self'),
+                (Token.Operator, '.'),
+                (Token.Name, '_waiters'),
+                (Token.Punctuation, ')'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            'message.append(f" [line {lineno:2d}]")\n',
+            [
+                (Token.Name, 'message'),
+                (Token.Operator, '.'),
+                (Token.Name, 'append'),
+                (Token.Punctuation, '('),
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Literal.String.Double, ' [line '),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'lineno'),
+                (Token.Literal.String.Interpol, ':'),
+                (Token.Literal.String.Double, '2d'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Double, ']'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Punctuation, ')'),
+                (Token.Text, '\n')
+            ]
+        ),
+        # Examples from https://bugs.python.org/issue36817
+        (
+            'f"{foo=}"\n',
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'foo'),
+                (Token.Literal.String.Interpol, '=}'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'{foo=!s}'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'foo'),
+                (Token.Literal.String.Interpol, '=!s}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            'f"{math.pi=!f:.2f}"\n',
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'math'),
+                (Token.Operator, '.'),
+                (Token.Name, 'pi'),
+                (Token.Literal.String.Interpol, '=!f:'),
+                (Token.Literal.String.Double, '.2f'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Text, '\n')
+            ]
+        ), (
+            'f"{ chr(65) =}"\n',
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Text, ' '),
+                (Token.Name.Builtin, 'chr'),
+                (Token.Punctuation, '('),
+                (Token.Literal.Number.Integer, '65'),
+                (Token.Punctuation, ')'),
+                (Token.Text, ' '),
+                (Token.Literal.String.Interpol, '=}'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Text, '\n')
+            ]
+        ), (
+            'f"{chr(65) = }"\n',
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name.Builtin, 'chr'),
+                (Token.Punctuation, '('),
+                (Token.Literal.Number.Integer, '65'),
+                (Token.Punctuation, ')'),
+                (Token.Text, ' '),
+                (Token.Literal.String.Interpol, '= }'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'*{n=:30}*'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, '*'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'n'),
+                (Token.Literal.String.Interpol, '=:'),
+                (Token.Literal.String.Single, '30'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, '*'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'*{n=!r:30}*'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, '*'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'n'),
+                (Token.Literal.String.Interpol, '=!r:'),
+                (Token.Literal.String.Single, '30'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, '*'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            """f"*{f'{n=}':30}*"\n""",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Literal.String.Double, '*'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'n'),
+                (Token.Literal.String.Interpol, '=}'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Interpol, ':'),
+                (Token.Literal.String.Double, '30'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Double, '*'),
+                (Token.Literal.String.Double, '"'),
+                (Token.Text, '\n')
+            ]
+        ), (
+            "f'*{n=:+<30}*'\n",
+            [
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Literal.String.Single, '*'),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'n'),
+                (Token.Literal.String.Interpol, '=:'),
+                (Token.Literal.String.Single, '+<30'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, '*'),
+                (Token.Literal.String.Single, "'"),
+                (Token.Text, '\n')
+            ]
+        ), (
+            """
+   f'''{foo
+         = !s:20}'''\n""",
+            [
+                (Token.Text, '   '),
+                (Token.Literal.String.Affix, 'f'),
+                (Token.Literal.String.Single, "'''"),
+                (Token.Literal.String.Interpol, '{'),
+                (Token.Name, 'foo'),
+                (Token.Text, '\n         '),
+                (Token.Literal.String.Interpol, '= !s:'),
+                (Token.Literal.String.Single, '20'),
+                (Token.Literal.String.Interpol, '}'),
+                (Token.Literal.String.Single, "'''"),
+                (Token.Text, '\n')
+            ]
+        )
+
+    )
+
+    for fragment,tokens in fragments_and_tokens:
+        assert list(lexer3.get_tokens(fragment)) == tokens
+
+    # Now switch between single and double quotes, to cover both cases equally
+    rep = {"'":'"', '"':"'"}
+    pattern = re.compile("|".join(rep.keys()))
+    for fragment,tokens in fragments_and_tokens:
+        fragment = pattern.sub(lambda m: rep[m.group(0)], fragment)
+        tokens = list(tokens)
+        for i,(token,match) in enumerate(tokens):
+            if token == Token.Literal.String.Single:
+                token = Token.Literal.String.Double
+            elif token == Token.Literal.String.Double:
+                token = Token.Literal.String.Single
+            match = pattern.sub(lambda m: rep[m.group(0)], match)
+            tokens[i] = (token, match)
+        assert list(lexer3.get_tokens(fragment)) == tokens