summaryrefslogtreecommitdiff
path: root/tests/test_python.py
diff options
context:
space:
mode:
Diffstat (limited to 'tests/test_python.py')
-rw-r--r--tests/test_python.py889
1 files changed, 0 insertions, 889 deletions
diff --git a/tests/test_python.py b/tests/test_python.py
deleted file mode 100644
index 12529d4a..00000000
--- a/tests/test_python.py
+++ /dev/null
@@ -1,889 +0,0 @@
-"""
- Python Tests
- ~~~~~~~~~~~~
-
- :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
- :license: BSD, see LICENSE for details.
-"""
-
-import pytest
-
-from pygments.lexers import PythonLexer, Python3Lexer
-from pygments.token import Token
-
-import re
-
-
-@pytest.fixture(scope='module')
-def lexer2():
- yield PythonLexer()
-
-
-@pytest.fixture(scope='module')
-def lexer3():
- yield Python3Lexer()
-
-
-def test_cls_builtin(lexer2):
- """
- Tests that a cls token gets interpreted as a Token.Name.Builtin.Pseudo
- """
- fragment = 'class TestClass():\n @classmethod\n def hello(cls):\n pass\n'
- tokens = [
- (Token.Keyword, 'class'),
- (Token.Text, ' '),
- (Token.Name.Class, 'TestClass'),
- (Token.Punctuation, '('),
- (Token.Punctuation, ')'),
- (Token.Punctuation, ':'),
- (Token.Text, '\n'),
- (Token.Text, ' '),
- (Token.Name.Decorator, '@classmethod'),
- (Token.Text, '\n'),
- (Token.Text, ' '),
- (Token.Keyword, 'def'),
- (Token.Text, ' '),
- (Token.Name.Function, 'hello'),
- (Token.Punctuation, '('),
- (Token.Name.Builtin.Pseudo, 'cls'),
- (Token.Punctuation, ')'),
- (Token.Punctuation, ':'),
- (Token.Text, '\n'),
- (Token.Text, ' '),
- (Token.Keyword, 'pass'),
- (Token.Text, '\n'),
- ]
- assert list(lexer2.get_tokens(fragment)) == tokens
-
-
-def test_needs_name(lexer3):
- """
- Tests that '@' is recognized as an Operator
- """
- fragment = 'S = (H @ beta - r).T @ inv(H @ V @ H.T) @ (H @ beta - r)\n'
- tokens = [
- (Token.Name, 'S'),
- (Token.Text, ' '),
- (Token.Operator, '='),
- (Token.Text, ' '),
- (Token.Punctuation, '('),
- (Token.Name, 'H'),
- (Token.Text, ' '),
- (Token.Operator, '@'),
- (Token.Text, ' '),
- (Token.Name, 'beta'),
- (Token.Text, ' '),
- (Token.Operator, '-'),
- (Token.Text, ' '),
- (Token.Name, 'r'),
- (Token.Punctuation, ')'),
- (Token.Operator, '.'),
- (Token.Name, 'T'),
- (Token.Text, ' '),
- (Token.Operator, '@'),
- (Token.Text, ' '),
- (Token.Name, 'inv'),
- (Token.Punctuation, '('),
- (Token.Name, 'H'),
- (Token.Text, ' '),
- (Token.Operator, '@'),
- (Token.Text, ' '),
- (Token.Name, 'V'),
- (Token.Text, ' '),
- (Token.Operator, '@'),
- (Token.Text, ' '),
- (Token.Name, 'H'),
- (Token.Operator, '.'),
- (Token.Name, 'T'),
- (Token.Punctuation, ')'),
- (Token.Text, ' '),
- (Token.Operator, '@'),
- (Token.Text, ' '),
- (Token.Punctuation, '('),
- (Token.Name, 'H'),
- (Token.Text, ' '),
- (Token.Operator, '@'),
- (Token.Text, ' '),
- (Token.Name, 'beta'),
- (Token.Text, ' '),
- (Token.Operator, '-'),
- (Token.Text, ' '),
- (Token.Name, 'r'),
- (Token.Punctuation, ')'),
- (Token.Text, '\n'),
- ]
- assert list(lexer3.get_tokens(fragment)) == tokens
-
-
-def test_pep_515(lexer3):
- """
- Tests that the lexer can parse numeric literals with underscores
- """
- fragments = (
- (Token.Literal.Number.Integer, '1_000_000'),
- (Token.Literal.Number.Float, '1_000.000_001'),
- (Token.Literal.Number.Float, '1_000e1_000j'),
- (Token.Literal.Number.Hex, '0xCAFE_F00D'),
- (Token.Literal.Number.Bin, '0b_0011_1111_0100_1110'),
- (Token.Literal.Number.Oct, '0o_777_123'),
- )
-
- for token, fragment in fragments:
- tokens = [
- (token, fragment),
- (Token.Text, '\n'),
- ]
- assert list(lexer3.get_tokens(fragment)) == tokens
-
-
-def test_walrus_operator(lexer3):
- """
- Tests that ':=' is recognized as an Operator
- """
- fragment = 'if (a := 2) > 4:'
- tokens = [
- (Token.Keyword, 'if'),
- (Token.Text, ' '),
- (Token.Punctuation, '('),
- (Token.Name, 'a'),
- (Token.Text, ' '),
- (Token.Operator, ':='),
- (Token.Text, ' '),
- (Token.Literal.Number.Integer, '2'),
- (Token.Punctuation, ')'),
- (Token.Text, ' '),
- (Token.Operator, '>'),
- (Token.Text, ' '),
- (Token.Literal.Number.Integer, '4'),
- (Token.Punctuation, ':'),
- (Token.Text, '\n'),
- ]
- assert list(lexer3.get_tokens(fragment)) == tokens
-
-
-def test_fstring(lexer3):
- """
- Tests that the lexer can parse f-strings
- """
- fragments_and_tokens = (
- # examples from PEP-0498
- (
- "f'My name is {name}, my age next year is {age+1}, my anniversary is {anniversary:%A, %B %d, %Y}.'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'My name is '),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'name'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, ', my age next year is '),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'age'),
- (Token.Operator, '+'),
- (Token.Literal.Number.Integer, '1'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, ', my anniversary is '),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'anniversary'),
- (Token.Literal.String.Interpol, ':'),
- (Token.Literal.String.Single, '%A, %B %d, %Y'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, '.'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "f'He said his name is {name!r}.'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'He said his name is '),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'name'),
- (Token.Literal.String.Interpol, '!r}'),
- (Token.Literal.String.Single, '.'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
-
- ), (
- "f'input={value:#06x}'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'input='),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'value'),
- (Token.Literal.String.Interpol, ':'),
- (Token.Literal.String.Single, '#06x'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- """f'{"quoted string"}'\n""",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Double, 'quoted string'),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- """f'{f"{inner}"}'\n""", # not in the PEP
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'inner'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- # SyntaxError: f-string expression part cannot include a backslash
- "f'{\\'quoted string\\'}'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Error, '\\'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'quoted string'),
- (Token.Literal.String.Escape, "\\'"),
- (Token.Literal.String.Single, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "f'{{ {4*10} }}'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Escape, '{{'),
- (Token.Literal.String.Single, ' '),
- (Token.Literal.String.Interpol, '{'),
- (Token.Literal.Number.Integer, '4'),
- (Token.Operator, '*'),
- (Token.Literal.Number.Integer, '10'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, ' '),
- (Token.Literal.String.Escape, '}}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "f'{{{4*10}}}'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Escape, '{{'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Literal.Number.Integer, '4'),
- (Token.Operator, '*'),
- (Token.Literal.Number.Integer, '10'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Escape, '}}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "fr'x={4*10}'\n",
- [
- (Token.Literal.String.Affix, 'fr'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, "x="),
- (Token.Literal.String.Interpol, '{'),
- (Token.Literal.Number.Integer, '4'),
- (Token.Operator, '*'),
- (Token.Literal.Number.Integer, '10'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- """f'abc {a["x"]} def'\n""",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'abc '),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'a'),
- (Token.Punctuation, '['),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Double, 'x'),
- (Token.Literal.String.Double, '"'),
- (Token.Punctuation, ']'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, ' def'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "f'''abc {a['x']} def'''\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'''"),
- (Token.Literal.String.Single, 'abc '),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'a'),
- (Token.Punctuation, '['),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'x'),
- (Token.Literal.String.Single, "'"),
- (Token.Punctuation, ']'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, ' def'),
- (Token.Literal.String.Single, "'''"),
- (Token.Text, '\n')
- ]
- ), (
- """f'''{x
-+1}'''\n""",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'''"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'x'),
- (Token.Text, '\n'),
- (Token.Operator, '+'),
- (Token.Literal.Number.Integer, '1'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'''"),
- (Token.Text, '\n')
- ]
- ), (
- """f'''{d[0
-]}'''\n""",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'''"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'd'),
- (Token.Punctuation, '['),
- (Token.Literal.Number.Integer, '0'),
- (Token.Text, '\n'),
- (Token.Punctuation, ']'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'''"),
- (Token.Text, '\n')
- ]
- ), (
- "f'result: {value:{width}.{precision}}'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'result: '),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'value'),
- (Token.Literal.String.Interpol, ':'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'width'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, '.'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'precision'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "'a' 'b' f'{x}' '{c}' f'str<{y:^4}>' 'd' 'e'\n",
- [
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'a'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, ' '),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'b'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, ' '),
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'x'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, ' '),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, '{c}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, ' '),
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'str<'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'y'),
- (Token.Literal.String.Interpol, ':'),
- (Token.Literal.String.Single, '^4'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, '>'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, ' '),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'd'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, ' '),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'e'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "f'{i}:{d[i]}'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'i'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, ':'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'd'),
- (Token.Punctuation, '['),
- (Token.Name, 'i'),
- (Token.Punctuation, ']'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "f'x = {x:+3}'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, "x = "),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'x'),
- (Token.Literal.String.Interpol, ':'),
- (Token.Literal.String.Single, '+3'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "f'{fn(lst,2)} {fn(lst,3)}'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'fn'),
- (Token.Punctuation, '('),
- (Token.Name, 'lst'),
- (Token.Punctuation, ','),
- (Token.Literal.Number.Integer, '2'),
- (Token.Punctuation, ')'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, ' '),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'fn'),
- (Token.Punctuation, '('),
- (Token.Name, 'lst'),
- (Token.Punctuation, ','),
- (Token.Literal.Number.Integer, '3'),
- (Token.Punctuation, ')'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "f'mapping is { {a:b for (a, b) in ((1, 2), (3, 4))} }'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'mapping is '),
- (Token.Literal.String.Interpol, '{'),
- (Token.Text, ' '),
- (Token.Punctuation, '{'),
- (Token.Name, 'a'),
- (Token.Punctuation, ':'),
- (Token.Name, 'b'),
- (Token.Text, ' '),
- (Token.Keyword, 'for'),
- (Token.Text, ' '),
- (Token.Punctuation, '('),
- (Token.Name, 'a'),
- (Token.Punctuation, ','),
- (Token.Text, ' '),
- (Token.Name, 'b'),
- (Token.Punctuation, ')'),
- (Token.Text, ' '),
- (Token.Operator.Word, 'in'),
- (Token.Text, ' '),
- (Token.Punctuation, '('),
- (Token.Punctuation, '('),
- (Token.Literal.Number.Integer, '1'),
- (Token.Punctuation, ','),
- (Token.Text, ' '),
- (Token.Literal.Number.Integer, '2'),
- (Token.Punctuation, ')'),
- (Token.Punctuation, ','),
- (Token.Text, ' '),
- (Token.Punctuation, '('),
- (Token.Literal.Number.Integer, '3'),
- (Token.Punctuation, ','),
- (Token.Text, ' '),
- (Token.Literal.Number.Integer, '4'),
- (Token.Punctuation, ')'),
- (Token.Punctuation, ')'),
- (Token.Punctuation, '}'),
- (Token.Text, ' '),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- """f'a={d["a"]}'\n""",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'a='),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'd'),
- (Token.Punctuation, '['),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Double, 'a'),
- (Token.Literal.String.Double, '"'),
- (Token.Punctuation, ']'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "f'a={d[a]}'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, 'a='),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'd'),
- (Token.Punctuation, '['),
- (Token.Name, 'a'),
- (Token.Punctuation, ']'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "fr'{header}:\\s+'\n",
- [
- (Token.Literal.String.Affix, 'fr'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'header'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, ':'),
- (Token.Literal.String.Single, '\\'),
- (Token.Literal.String.Single, 's+'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "f'{a!r}'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'a'),
- (Token.Literal.String.Interpol, '!r}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "f'{(lambda x: x*2)(3)}'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Punctuation, '('),
- (Token.Keyword, 'lambda'),
- (Token.Text, ' '),
- (Token.Name, 'x'),
- (Token.Punctuation, ':'),
- (Token.Text, ' '),
- (Token.Name, 'x'),
- (Token.Operator, '*'),
- (Token.Literal.Number.Integer, '2'),
- (Token.Punctuation, ')'),
- (Token.Punctuation, '('),
- (Token.Literal.Number.Integer, '3'),
- (Token.Punctuation, ')'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "extra = f'{extra},waiters:{len(self._waiters)}'\n",
- [
- (Token.Name, 'extra'),
- (Token.Text, ' '),
- (Token.Operator, '='),
- (Token.Text, ' '),
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'extra'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, ',waiters:'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name.Builtin, 'len'),
- (Token.Punctuation, '('),
- (Token.Name.Builtin.Pseudo, 'self'),
- (Token.Operator, '.'),
- (Token.Name, '_waiters'),
- (Token.Punctuation, ')'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- 'message.append(f" [line {lineno:2d}]")\n',
- [
- (Token.Name, 'message'),
- (Token.Operator, '.'),
- (Token.Name, 'append'),
- (Token.Punctuation, '('),
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Double, ' [line '),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'lineno'),
- (Token.Literal.String.Interpol, ':'),
- (Token.Literal.String.Double, '2d'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Double, ']'),
- (Token.Literal.String.Double, '"'),
- (Token.Punctuation, ')'),
- (Token.Text, '\n')
- ]
- ),
- # Examples from https://bugs.python.org/issue36817
- (
- 'f"{foo=}"\n',
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'foo'),
- (Token.Literal.String.Interpol, '=}'),
- (Token.Literal.String.Double, '"'),
- (Token.Text, '\n')
- ]
- ), (
- "f'{foo=!s}'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'foo'),
- (Token.Literal.String.Interpol, '=!s}'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- 'f"{math.pi=!f:.2f}"\n',
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'math'),
- (Token.Operator, '.'),
- (Token.Name, 'pi'),
- (Token.Literal.String.Interpol, '=!f:'),
- (Token.Literal.String.Double, '.2f'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Double, '"'),
- (Token.Text, '\n')
- ]
- ), (
- 'f"{ chr(65) =}"\n',
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Text, ' '),
- (Token.Name.Builtin, 'chr'),
- (Token.Punctuation, '('),
- (Token.Literal.Number.Integer, '65'),
- (Token.Punctuation, ')'),
- (Token.Text, ' '),
- (Token.Literal.String.Interpol, '=}'),
- (Token.Literal.String.Double, '"'),
- (Token.Text, '\n')
- ]
- ), (
- 'f"{chr(65) = }"\n',
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name.Builtin, 'chr'),
- (Token.Punctuation, '('),
- (Token.Literal.Number.Integer, '65'),
- (Token.Punctuation, ')'),
- (Token.Text, ' '),
- (Token.Literal.String.Interpol, '= }'),
- (Token.Literal.String.Double, '"'),
- (Token.Text, '\n')
- ]
- ), (
- "f'*{n=:30}*'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, '*'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'n'),
- (Token.Literal.String.Interpol, '=:'),
- (Token.Literal.String.Single, '30'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, '*'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- "f'*{n=!r:30}*'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, '*'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'n'),
- (Token.Literal.String.Interpol, '=!r:'),
- (Token.Literal.String.Single, '30'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, '*'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- """f"*{f'{n=}':30}*"\n""",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Double, '*'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'n'),
- (Token.Literal.String.Interpol, '=}'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Interpol, ':'),
- (Token.Literal.String.Double, '30'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Double, '*'),
- (Token.Literal.String.Double, '"'),
- (Token.Text, '\n')
- ]
- ), (
- "f'*{n=:+<30}*'\n",
- [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'"),
- (Token.Literal.String.Single, '*'),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'n'),
- (Token.Literal.String.Interpol, '=:'),
- (Token.Literal.String.Single, '+<30'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, '*'),
- (Token.Literal.String.Single, "'"),
- (Token.Text, '\n')
- ]
- ), (
- """
- f'''{foo
- = !s:20}'''\n""",
- [
- (Token.Text, ' '),
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Single, "'''"),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'foo'),
- (Token.Text, '\n '),
- (Token.Literal.String.Interpol, '= !s:'),
- (Token.Literal.String.Single, '20'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Single, "'''"),
- (Token.Text, '\n')
- ]
- )
-
- )
-
- for fragment,tokens in fragments_and_tokens:
- assert list(lexer3.get_tokens(fragment)) == tokens
-
- # Now switch between single and double quotes, to cover both cases equally
- rep = {"'":'"', '"':"'"}
- pattern = re.compile("|".join(rep.keys()))
- for fragment,tokens in fragments_and_tokens:
- fragment = pattern.sub(lambda m: rep[m.group(0)], fragment)
- tokens = list(tokens)
- for i,(token,match) in enumerate(tokens):
- if token == Token.Literal.String.Single:
- token = Token.Literal.String.Double
- elif token == Token.Literal.String.Double:
- token = Token.Literal.String.Single
- match = pattern.sub(lambda m: rep[m.group(0)], match)
- tokens[i] = (token, match)
- assert list(lexer3.get_tokens(fragment)) == tokens
-
-def test_raw_fstring(lexer3):
- """
- Tests that the lexer can parse raw f-strings
- """
- # Just raw
- fragment = r'rf"m_\nu = x"'
- tokens = [
- (Token.Literal.String.Affix, 'rf'),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Double, 'm_'),
- (Token.Literal.String.Double, '\\'),
- (Token.Literal.String.Double, 'nu = x'),
- (Token.Literal.String.Double, '"'),
- (Token.Text, '\n')
- ]
- # Just f-string
- fragment = r'f"m_\nu = {x}"'
- tokens = [
- (Token.Literal.String.Affix, 'f'),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Double, 'm_'),
- (Token.Literal.String.Escape, '\\n'),
- (Token.Literal.String.Double, 'u = '),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'x'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Double, '"'),
- (Token.Text, '\n'),
- ]
- # Raw behavior inside {{...}}
- fragment = r'rf"m_{{\nu}} = {x}"'
- tokens = [
- (Token.Literal.String.Affix, 'rf'),
- (Token.Literal.String.Double, '"'),
- (Token.Literal.String.Double, 'm_'),
- (Token.Literal.String.Escape, '{{'),
- (Token.Literal.String.Double, '\\'),
- (Token.Literal.String.Double, 'nu'),
- (Token.Literal.String.Escape, '}}'),
- (Token.Literal.String.Double, ' = '),
- (Token.Literal.String.Interpol, '{'),
- (Token.Name, 'x'),
- (Token.Literal.String.Interpol, '}'),
- (Token.Literal.String.Double, '"'),
- (Token.Text, '\n'),
- ]
- assert list(lexer3.get_tokens(fragment)) == tokens
View Lorry Controller Status