1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
# -*- coding: utf-8 -*-
"""
Pygments regex lexer tests
~~~~~~~~~~~~~~~~~~~~~~~~~~
:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import unittest
from pygments.token import Text
from pygments.lexer import RegexLexer
from pygments.lexer import default
class TestLexer(RegexLexer):
"""Test tuple state transitions including #pop."""
tokens = {
'root': [
('a', Text.Root, 'rag'),
('e', Text.Root),
('#', Text.Root, '#pop'),
('@', Text.Root, ('#pop', '#pop')),
default(('beer', 'beer'))
],
'beer': [
('d', Text.Beer, ('#pop', '#pop')),
],
'rag': [
('b', Text.Rag, '#push'),
('c', Text.Rag, ('#pop', 'beer')),
],
}
class TupleTransTest(unittest.TestCase):
def test(self):
lx = TestLexer()
toks = list(lx.get_tokens_unprocessed('abcde'))
self.assertEqual(toks, [
(0, Text.Root, 'a'), (1, Text.Rag, 'b'), (2, Text.Rag, 'c'),
(3, Text.Beer, 'd'), (4, Text.Root, 'e')])
def test_multiline(self):
lx = TestLexer()
toks = list(lx.get_tokens_unprocessed('a\ne'))
self.assertEqual(toks, [
(0, Text.Root, 'a'), (1, Text, u'\n'), (2, Text.Root, 'e')])
def test_default(self):
lx = TestLexer()
toks = list(lx.get_tokens_unprocessed('d'))
self.assertEqual(toks, [(0, Text.Beer, 'd')])
class PopEmptyTest(unittest.TestCase):
def test_regular(self):
lx = TestLexer()
toks = list(lx.get_tokens_unprocessed('#e'))
self.assertEqual(toks, [(0, Text.Root, '#'), (1, Text.Root, 'e')])
def test_tuple(self):
lx = TestLexer()
toks = list(lx.get_tokens_unprocessed('@e'))
self.assertEqual(toks, [(0, Text.Root, '@'), (1, Text.Root, 'e')])
|
