diff options
| -rw-r--r-- | pygments/lexer.py | 29 | ||||
| -rw-r--r-- | pygments/lexers/text.py | 55 |
2 files changed, 74 insertions, 10 deletions
diff --git a/pygments/lexer.py b/pygments/lexer.py index bffec0f1..2648b07b 100644 --- a/pygments/lexer.py +++ b/pygments/lexer.py @@ -588,7 +588,7 @@ def do_insertions(insertions, tokens): The result is a combined token stream. - FIXME: The indices yielded by this function are not correct! + TODO: clean up the code here. """ insertions = iter(insertions) try: @@ -599,21 +599,34 @@ def do_insertions(insertions, tokens): yield item return + realpos = None insleft = True + + # iterate over the token stream where we want to insert + # the tokens from the insertion list. for i, t, v in tokens: + # first iteration. store the postition of first item + if realpos is None: + realpos = i oldi = 0 while insleft and i + len(v) >= index: - yield i, t, v[oldi:index-i] - for item in itokens: - yield item - oldi = index-i + tmpval = v[oldi:index - i] + yield realpos, t, tmpval + realpos += len(tmpval) + for it_index, it_token, it_value in itokens: + yield realpos, it_token, it_value + realpos += len(it_value) + oldi = index - i try: index, itokens = insertions.next() except StopIteration: insleft = False break # not strictly necessary - yield i, t, v[oldi:] + yield realpos, t, v[oldi:] + realpos += len(v) - oldi + # leftover tokens if insleft: - for item in itokens: - yield item + for p, t, v in itokens: + yield realpos, t, v + realpos += len(v) diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py index 09ae3ed5..806a32b3 100644 --- a/pygments/lexers/text.py +++ b/pygments/lexers/text.py @@ -18,10 +18,11 @@ try: except NameError: from sets import Set as set -from pygments.lexer import RegexLexer, bygroups, include, using, this +from pygments.lexer import RegexLexer, bygroups, include, using, this, \ + do_insertions from pygments.token import Punctuation, \ Text, Comment, Keyword, Name, String, Generic, Operator, Number -from pygments.util import ClassNotFound +from pygments.util import get_bool_opt __all__ = ['IniLexer', 'SourcesListLexer', 'MakefileLexer', 'DiffLexer', @@ -459,6 +460,48 @@ class RstLexer(RegexLexer): mimetypes = ["text/x-rst"] flags = re.MULTILINE + def _handle_sourcecode(self, match): + from pygments.lexers import get_lexer_by_name + from pygments.util import ClassNotFound + + # section header + yield match.start(1), Punctuation, match.group(1) + yield match.start(2), Text, match.group(2) + yield match.start(3), Operator.Word, match.group(3) + yield match.start(4), Punctuation, match.group(4) + yield match.start(5), Text, match.group(5) + yield match.start(6), Keyword, match.group(6) + yield match.start(7), Text, match.group(7) + + # lookup lexer if wanted and existing + lexer = None + if self.handlecodeblocks: + try: + lexer = get_lexer_by_name(match.group(6).strip()) + except ClassNotFound: + pass + indention = match.group(8) + indention_size = len(indention) + code = (indention + match.group(9) + match.group(10) + match.group(11)) + + # no lexer for this language. handle it like it was a code block + if lexer is None: + yield match.start(8), String, code + return + + # highlight the lines with the lexer. + ins = [] + codelines = code.splitlines(True) + code = '' + for line in codelines: + if len(line) > indention_size: + ins.append((len(code), [(0, Text, line[:indention_size])])) + code += line[indention_size:] + else: + code += line + for item in do_insertions(ins, lexer.get_tokens_unprocessed(code)): + yield item + tokens = { 'root': [ # Heading with overline @@ -482,6 +525,10 @@ class RstLexer(RegexLexer): bygroups(Text, Number, using(this, state='inline'))), (r'^(\s*)(\(?[A-Za-z]+\))( .+\n(?:\1 .+\n)+)', bygroups(Text, Number, using(this, state='inline'))), + # sourcecode sections + (r'^( *\.\.)(\s*)((?:source)?code)(::)([ \t]*)([^\n]+)' + r'(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\8.*|)\n)+)', + _handle_sourcecode), # Introducing a section (r'^( *\.\.)(\s*)(\w+)(::)(?:([ \t]*)(.+))?', bygroups(Punctuation, Text, Operator.Word, Punctuation, Text, Keyword)), @@ -527,3 +574,7 @@ class RstLexer(RegexLexer): (r'[`\\]', String), ] } + + def __init__(self, **options): + self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True) + RegexLexer.__init__(self, **options) |