diff options
Diffstat (limited to 'sandbox/code-block-directive/pygments_code_block_directive.py')
| -rwxr-xr-x | sandbox/code-block-directive/pygments_code_block_directive.py | 143 |
1 files changed, 95 insertions, 48 deletions
diff --git a/sandbox/code-block-directive/pygments_code_block_directive.py b/sandbox/code-block-directive/pygments_code_block_directive.py index 0a115d905..e2abe30b6 100755 --- a/sandbox/code-block-directive/pygments_code_block_directive.py +++ b/sandbox/code-block-directive/pygments_code_block_directive.py @@ -1,6 +1,7 @@ #!/usr/bin/python +# coding: utf-8 -# :Author: the Pygments team; Felix Wiemann; Guenter Milde +# :Author: Georg Brandl; Felix Wiemann; Günter Milde # :Date: $Date$ # :Copyright: This module has been placed in the public domain. # @@ -9,7 +10,7 @@ # # .. class:: borderless # -# ========== =========================================================== +# ========== ============================================================= # 2007-06-01 Removed redundancy from class values. # 2007-06-04 Merge of successive tokens of same type # (code taken from pygments.formatters.others). @@ -20,26 +21,29 @@ # (misnamed as docutils_formatter) as class DocutilsInterface # 2007-06-08 Failsave implementation (fallback to a standard literal block # if pygments not found) -# ========== =========================================================== +# 2010-11-27 Rename directive and class from "code-block" to "code". +# Fix fallback if pygments not found. +# Use class-based interface. +# Add "number-lines" option. +# ========== ============================================================= # # :: -"""Define and register a code-block directive using pygments -""" +"""Define and register a code directive using pygments""" # Requirements # ------------ # :: from docutils import nodes -from docutils.parsers.rst import directives +from docutils.parsers.rst import directives, Directive try: import pygments from pygments.lexers import get_lexer_by_name from pygments.formatters.html import _get_ttype_class + with_pygments = True except ImportError: - pass - + with_pygments = False # Customisation @@ -50,16 +54,13 @@ except ImportError: unstyled_tokens = [''] -# DocutilsInterface +# Tokenizer # ----------------- # # This interface class combines code from # pygments.formatters.html and pygments.formatters.others. -# -# It does not require anything of docutils and could also become a part of -# pygments:: -class DocutilsInterface(object): +class Tokenizer(object): """Parse `code` string and yield "classified" tokens. Arguments @@ -72,7 +73,6 @@ class DocutilsInterface(object): Yields the tokens as ``(ttype_class, value)`` tuples, where ttype_class is taken from pygments.token.STANDARD_TYPES and corresponds to the class argument used in pygments html output. - """ def __init__(self, code, language): @@ -84,13 +84,15 @@ class DocutilsInterface(object): try: lexer = get_lexer_by_name(self.language) except ValueError: - # info: "no pygments lexer for %s, using 'text'"%self.language + # info: 'no pygments lexer for %s, using "text"' % self.language lexer = get_lexer_by_name('text') return pygments.lex(self.code, lexer) def join(self, tokens): - """join subsequent tokens of same token-type + """Join subsequent tokens of same token-type. + + Also, leave out the final '\n' (added by pygments). """ tokens = iter(tokens) (lasttype, lastval) = tokens.next() @@ -100,54 +102,100 @@ class DocutilsInterface(object): else: yield(lasttype, lastval) (lasttype, lastval) = (ttype, value) - yield(lasttype, lastval) + if lastval != '\n': + yield(lasttype, lastval) def __iter__(self): - """parse code string and yield "clasified" tokens + """parse code string and yield "classified" tokens """ - try: - tokens = self.lex() - except IOError: - print "INFO: Pygments lexer not found, using fallback" - # TODO: write message to INFO - yield ('', self.code) - return - + tokens = self.lex() for ttype, value in self.join(tokens): + # yield (ttype, value) yield (_get_ttype_class(ttype), value) +class NumberLines(object): + """Insert linenumber-tokens in front of every newline + + Nontrivial, as we need to weave these into the possibly + multi-line tokens from pygments. + """ + + def __init__(self, tokens, startline, fmt_str): + self.tokens = tokens + self.lineno = startline + self.fmt_str = fmt_str -# code_block_directive + def __iter__(self): + yield ('ln', self.fmt_str % self.lineno) + for ttype, value in self.tokens: + lines = value.split('\n') + for line in lines[:-1]: + yield (ttype, line + '\n') + self.lineno += 1 + yield ('ln', self.fmt_str % self.lineno) + yield (ttype, lines[-1]) + + +# CodeBlock directive # -------------------- # :: -def code_block_directive(name, arguments, options, content, lineno, - content_offset, block_text, state, state_machine): - """parse and classify content of a code_block +class CodeBlock(Directive): + """Parse and mark up content of a code block. """ - language = arguments[0] - # create a literal block element and set class argument - code_block = nodes.literal_block(classes=["code-block", language]) - - # parse content with pygments and add to code_block element - for cls, value in DocutilsInterface(u'\n'.join(content), language): - if cls in unstyled_tokens: - # insert as Text to decrease the verbosity of the output. - code_block += nodes.Text(value, value) + required_arguments = 1 + option_spec = {'class': directives.class_option, + 'number-lines': directives.unchanged + } + has_content = True + + def run(self): + language = self.arguments[0] + # Process number-lines with optional argument `startline` + startline = self.options.get('number-lines', '1') + try: + startline = int(startline or 1) # default to 1 for empty str + except ValueError: + raise self.error( + ':number-lines: option with non-integer start value') + self.assert_has_content() + + # create a literal block element and set class argument + code_block = nodes.literal_block(classes=['code', language] + + self.options['class']) + + # iterator returning code tokens + if with_pygments: + tokens = Tokenizer(u'\n'.join(self.content), language) else: - code_block += nodes.inline(value, value, classes=[cls]) + # TODO: warning or info? + self.warning('Cannot highlight code, Pygments lexer not found.') + tokens = [('', u'\n'.join(self.content))] + + if 'number-lines' in self.options: + # pad linenumbers, e.g. endline == 100 -> fmt_str = '%3d ' + endline = startline + len(self.content) + fmt_str = "%%%dd " % len(str(endline)) + # print startline, '...', endline, repr(fmt_str) + tokens = NumberLines(tokens, startline, fmt_str) + + # parse content with pygments and add to code_block element + for cls, value in tokens: + if cls in unstyled_tokens: + # insert as Text to decrease the verbosity of the output. + code_block += nodes.Text(value, value) + else: + code_block += nodes.inline(value, value, classes=[cls]) - return [code_block] + return [code_block] # Register Directive # ------------------ # :: -code_block_directive.arguments = (1, 0, 1) -code_block_directive.content = 1 -directives.register_directive('code-block', code_block_directive) +directives.register_directive('code', CodeBlock) # .. _doctutils: http://docutils.sf.net/ # .. _pygments: http://pygments.org/ @@ -163,15 +211,14 @@ directives.register_directive('code-block', code_block_directive) if __name__ == '__main__': from docutils.core import publish_cmdline, default_description - description = "code-block directive test output" + default_description + description = 'code-block directive test output' + default_description try: import locale locale.setlocale(locale.LC_ALL, '') except: pass # Uncomment the desired output format: - publish_cmdline(writer_name='pseudoxml', description=description) + # publish_cmdline(writer_name='pseudoxml', description=description) # publish_cmdline(writer_name='xml', description=description) - # publish_cmdline(writer_name='html', description=description) + publish_cmdline(writer_name='html', description=description) # publish_cmdline(writer_name='latex', description=description) - # publish_cmdline(writer_name='newlatex2e', description=description) |