diff options
Diffstat (limited to 'pygments/lexer.py')
-rw-r--r-- | pygments/lexer.py | 486 |
1 files changed, 486 insertions, 0 deletions
diff --git a/pygments/lexer.py b/pygments/lexer.py new file mode 100644 index 00000000..3d3ee865 --- /dev/null +++ b/pygments/lexer.py @@ -0,0 +1,486 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexer + ~~~~~~~~~~~~~ + + Base lexer classes. + + :copyright: 2006 by Georg Brandl. + :license: GNU LGPL, see LICENSE for more details. +""" +import re + +from pygments.token import Error, Text, Other, _TokenType +from pygments.util import get_bool_opt, get_int_opt + + +__all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer', + 'LexerContext', 'include', 'flags', 'bygroups', 'using', 'this'] + + +class Lexer(object): + """ + Lexer for a specific language. + + Basic options recognized: + ``stripnl`` + Strip leading and trailing newlines from the input (default: True). + ``stripall`` + Strip all leading and trailing whitespace from the input + (default: False). + ``tabsize`` + If given and greater than 0, expand tabs in the input (default: 0). + """ + + #: Name of the lexer + name = None + + #: Shortcuts for the lexer + aliases = [] + + #: fn match rules + filenames = [] + + def __init__(self, **options): + self.options = options + self.stripnl = get_bool_opt(options, 'stripnl', True) + self.stripall = get_bool_opt(options, 'stripall', False) + self.tabsize = get_int_opt(options, 'tabsize', 0) + + def get_tokens(self, text): + """ + Return an iterable of (tokentype, value) pairs generated from ``text``. + + Also preprocess the text, i.e. expand tabs and strip it if wanted. + """ + text = type(text)('\n').join(text.splitlines()) + if self.stripall: + text = text.strip() + elif self.stripnl: + text = text.strip('\n') + if self.tabsize > 0: + text = text.expandtabs(self.tabsize) + if not text.endswith('\n'): + text += '\n' + + for i, t, v in self.get_tokens_unprocessed(text): + yield t, v + + def get_tokens_unprocessed(self, text): + """ + Return an iterable of (tokentype, value) pairs. + In subclasses, implement this method as a generator to + maximize effectiveness. + """ + raise NotImplementedError + + +class DelegatingLexer(Lexer): + """ + This lexer takes two lexer as arguments. A root lexer and + a language lexer. First everything is scanned using the language + lexer, afterwards all ``Other`` tokens are lexed using the root + lexer. + + The lexers from the ``template`` lexer package use this base lexer. + """ + + def __init__(self, _root_lexer, _language_lexer, _needle=Other, **options): + self.root_lexer = _root_lexer(**options) + self.language_lexer = _language_lexer(**options) + self.needle = _needle + Lexer.__init__(self, **options) + + def get_tokens_unprocessed(self, text): + buffered = '' + insertions = [] + lng_buffer = [] + for i, t, v in self.language_lexer.get_tokens_unprocessed(text): + if t is self.needle: + if lng_buffer: + insertions.append((len(buffered), lng_buffer)) + lng_buffer = [] + buffered += v + else: + lng_buffer.append((i, t, v)) + # the last "\n" should always be lexed by the root lexer + return do_insertions(insertions, + self.root_lexer.get_tokens_unprocessed(buffered)) + + +#------------------------------------------------------------------------------- +# RegexLexer and ExtendedRegexLexer +# + + +class include(str): + """ + Indicates that a state should include rules from another state. + """ + pass + + +class combined(tuple): + """ + Indicates a state combined from multiple states. + """ + + def __new__(cls, *args): + return tuple.__new__(cls, args) + + def __init__(self, *args): + tuple.__init__(self, args) + + +class _PseudoMatch(object): + """ + A pseudo match object constructed from a string. + """ + + def __init__(self, start, text): + self._text = text + self._start = start + + def start(self, arg=None): + return self._start + + def end(self, arg=None): + return self._start + len(self._text) + + def group(self, arg=None): + if arg: + raise IndexError('No such group') + return self._text + + def groups(self): + return (self._text,) + + def groupdict(self): + return {} + + +def bygroups(*args): + """ + Callback that yields multiple actions for each group in the match. + """ + def callback(lexer, match, ctx=None): + for i, action in enumerate(args): + if type(action) is _TokenType: + data = match.group(i + 1) + if data: + yield match.start(i + 1), action, data + else: + if ctx: + ctx.pos = match.start(i+1) + for item in action(lexer, _PseudoMatch(match.start(i + 1), + match.group(i + 1)), ctx): + if item: + yield item + if ctx: + ctx.pos = match.end() + return callback + + +class _This(object): + """ + Special singleton used for indicating the caller class. + Used by ``using``. + """ +this = _This() + + +def using(_other, **kwargs): + """ + Callback that processes the match with a different lexer. + + The keyword arguments are forwarded to the lexer. + """ + if _other is this: + def callback(lexer, match, ctx=None): + s = match.start() + for i, t, v in lexer.get_tokens_unprocessed(match.group()): + yield i + s, t, v + if ctx: + ctx.pos = match.end() + else: + def callback(lexer, match, ctx=None): + # XXX: cache that somehow + kwargs.update(lexer.options) + lx = _other(**kwargs) + + s = match.start() + for i, t, v in lx.get_tokens_unprocessed(match.group()): + yield i + s, t, v + if ctx: + ctx.pos = match.end() + return callback + + +class RegexLexerMeta(type): + """ + Metaclass for RegexLexer, creates the self._tokens attribute from + self.tokens on the first instantiation. + """ + + def _process_state(cls, state): + assert type(state) is str, "wrong state name %r" % state + assert state[0] != '#', "invalid state name %r" % state + if state in cls._tokens: + return cls._tokens[state] + tokens = cls._tokens[state] = [] + rflags = cls.flags + for tdef in cls.tokens[state]: + if isinstance(tdef, include): + # it's a state reference + assert tdef != state, "circular state reference %r" % state + tokens.extend(cls._process_state(str(tdef))) + continue + + assert type(tdef) is tuple, "wrong rule def %r" % tdef + + rex = re.compile(tdef[0], rflags) + + assert type(tdef[1]) is _TokenType or callable(tdef[1]), \ + 'token type must be simple type or callable, not %r' % tdef[1] + + if len(tdef) == 2: + new_state = None + else: + tdef2 = tdef[2] + if isinstance(tdef2, str): + # an existing state + if tdef2 == '#pop': + new_state = -1 + elif tdef2 in cls.tokens: + new_state = (tdef2,) + elif tdef2 == '#push': + new_state = tdef2 + elif tdef2[:5] == '#pop:': + new_state = -int(tdef2[5:]) + else: + assert False, 'unknown new state %r' % tdef2 + elif isinstance(tdef2, combined): + # combine a new state from existing ones + new_state = '_tmp_%d' % cls._tmpname + cls._tmpname += 1 + itokens = [] + for istate in tdef2: + assert istate != state, 'circular state ref %r' % istate + itokens.extend(cls._process_state(istate)) + cls._tokens[new_state] = itokens + new_state = (new_state,) + elif isinstance(tdef2, tuple): + # push more than one state + for state in tdef2: + assert state in cls.tokens, \ + 'unknown new state ' + state + new_state = tdef2 + else: + assert False, 'unknown new state def %r' % tdef2 + tokens.append((rex, tdef[1], new_state)) + return tokens + + def __call__(cls, *args, **kwds): + if not hasattr(cls, '_tokens'): + cls._tokens = {} + cls._tmpname = 0 + for state in cls.tokens.keys(): + cls._process_state(state) + + return type.__call__(cls, *args, **kwds) + + +class RegexLexer(Lexer): + """ + Base for simple stateful regular expression-based lexers. + Simplifies the lexing process so that you need only + provide a list of states and regular expressions. + """ + __metaclass__ = RegexLexerMeta + + #: Flags for compiling the regular expressions. + #: Defaults to MULTILINE. + flags = re.MULTILINE + + #: Dict of ``{'state': [(regex, tokentype, new_state), ...], ...}`` + #: + #: The initial state is 'root'. + #: ``new_state`` can be omitted to signify no state transition. + #: If it is a string, the state is pushed on the stack and changed. + #: If it is a tuple of strings, all states are pushed on the stack and + #: the current state will be the topmost. + #: It can also be ``combined('state1', 'state2', ...)`` + #: to signify a new, anonymous state combined from the rules of two + #: or more existing ones. + #: Furthermore, it can be '#pop' to signify going back one step in + #: the state stack, or '#push' to push the current state on the stack + #: again. + #: + #: The tuple can also be replaced with ``include('state')``, in which + #: case the rules from the state named by the string are included in the + #: current one. + tokens = {} + + def get_tokens_unprocessed(self, text, stack=['root']): + """ + Split ``text`` into (tokentype, text) pairs. + + ``stack`` is the inital stack (default: ``['root']``) + """ + pos = 0 + statestack = stack[:] + statetokens = self._tokens[statestack[-1]] + while 1: + for rex, action, new_state in statetokens: + m = rex.match(text, pos) + if m: + if type(action) is _TokenType: + yield pos, action, m.group() + else: + for item in action(self, m): + yield item + pos = m.end() + if new_state is not None: + # state transition + if isinstance(new_state, tuple): + statestack.extend(new_state) + elif isinstance(new_state, int): + # pop + del statestack[new_state:] + elif new_state == '#push': + statestack.append(statestack[-1]) + else: + assert False, "wrong state def: %r" % new_state + statetokens = self._tokens[statestack[-1]] + break + else: + try: + if text[pos] == '\n': + # at EOL, reset state to "root" + pos += 1 + statestack = ['root'] + statetokens = self._tokens['root'] + yield pos, Text, '\n' + continue + yield pos, Error, text[pos] + pos += 1 + except IndexError: + break + + +class LexerContext(object): + """ + A helper object that holds lexer position data. + """ + + def __init__(self, text, pos, stack=None, end=None): + self.text = text + self.pos = pos + self.end = end or len(text) # end=0 not supported ;-) + self.stack = stack or ['root'] + + def __repr__(self): + return 'LexerContext(%r, %r, %r)' % ( + self.text, self.pos, self.stack) + + +class ExtendedRegexLexer(RegexLexer): + """ + A RegexLexer which additionally allows functions to be + specified as "token types", in which case the function will + be called. + """ + + def get_tokens_unprocessed(self, text=None, context=None): + """ + Split ``text`` into (tokentype, text) pairs. + If ``context`` is given, use this lexer context instead. + """ + if not context: + ctx = LexerContext(text, 0) + statetokens = self._tokens['root'] + else: + ctx = context + statetokens = self._tokens[ctx.stack[-1]] + text = ctx.text + while 1: + for rex, action, new_state in statetokens: + m = rex.match(text, ctx.pos, ctx.end) + if m: + if type(action) is _TokenType: + yield ctx.pos, action, m.group() + ctx.pos = m.end() + else: + for item in action(self, m, ctx): + yield item + if not new_state: + # altered the state stack? + statetokens = self._tokens[ctx.stack[-1]] + # CAUTION: callback must set ctx.pos! + if new_state is not None: + # state transition + if isinstance(new_state, tuple): + ctx.stack.extend(new_state) + elif isinstance(new_state, int): + # pop + del ctx.stack[new_state:] + elif new_state == '#push': + ctx.stack.append(ctx.stack[-1]) + else: + assert False, "wrong state def: %r" % new_state + statetokens = self._tokens[ctx.stack[-1]] + break + else: + try: + if ctx.pos >= ctx.end: + break + if text[ctx.pos] == '\n': + # at EOL, reset state to "root" + ctx.pos += 1 + ctx.stack = ['root'] + statetokens = self._tokens['root'] + yield ctx.pos, Text, '\n' + continue + yield ctx.pos, Error, text[ctx.pos] + ctx.pos += 1 + except IndexError: + break + + +def do_insertions(insertions, tokens): + """ + Helper for lexers which must combine the results of several + sublexers. + + ``insertions`` is a list of ``(index, itokens)`` pairs. + Each ``itokens`` iterable should be inserted at position + ``index`` into the token stream given by the ``tokens`` + argument. + + The result is a combined token stream. + + XXX: The indices yielded by this function are not correct! + """ + insertions = iter(insertions) + try: + index, itokens = insertions.next() + except StopIteration: + # no insertions + for item in tokens: + yield item + return + + insleft = True + for i, t, v in tokens: + oldi = 0 + while insleft and i + len(v) >= index: + yield i, t, v[oldi:index-i] + for item in itokens: + yield item + oldi = index-i + try: + index, itokens = insertions.next() + except StopIteration: + insleft = False + break # not strictly necessary + yield i, t, v[oldi:] |