diff options
author | Georg Brandl <georg@python.org> | 2016-01-17 17:00:26 +0100 |
---|---|---|
committer | Georg Brandl <georg@python.org> | 2016-01-17 17:00:26 +0100 |
commit | fc55dc2e95bcea03fbc0d8e1d130c9e53f3f1dad (patch) | |
tree | 2a06d2fac7321452513bda7739e297a7e9848ec4 /external | |
download | pygments-git-fc55dc2e95bcea03fbc0d8e1d130c9e53f3f1dad.tar.gz |
merge default into stable
Diffstat (limited to 'external')
-rwxr-xr-x | external/autopygmentize | 83 | ||||
-rwxr-xr-x | external/lasso-builtins-generator-9.lasso | 162 | ||||
-rw-r--r-- | external/markdown-processor.py | 67 | ||||
-rw-r--r-- | external/moin-parser.py | 112 | ||||
-rw-r--r-- | external/pygments.bashcomp | 38 | ||||
-rw-r--r-- | external/rst-directive.py | 82 |
6 files changed, 544 insertions, 0 deletions
diff --git a/external/autopygmentize b/external/autopygmentize new file mode 100755 index 00000000..d2f969a1 --- /dev/null +++ b/external/autopygmentize @@ -0,0 +1,83 @@ +#!/bin/bash +# Best effort auto-pygmentization with transparent decompression +# by Reuben Thomas 2008-2015 +# This program is in the public domain. + +# Strategy: first see if pygmentize can find a lexer; if not, ask file; if that finds nothing, fail +# Set the environment variable PYGMENTIZE_OPTS or pass options before the file path to configure pygments. + +# This program can be used as a .lessfilter for the less pager to auto-color less's output + +file="${!#}" # last argument +options=${@:1:$(($#-1))} # handle others args as options to pass to pygmentize + +file_common_opts="--brief --dereference" + +lexer=$(pygmentize -N "$file") +if [[ "$lexer" == text ]]; then + unset lexer + case $(file --mime-type --uncompress $file_common_opts "$file") in + application/xml|image/svg+xml) lexer=xml;; + application/javascript) lexer=javascript;; + text/html) lexer=html;; + text/troff) lexer=nroff;; + text/x-asm) lexer=nasm;; + text/x-awk) lexer=awk;; + text/x-c) lexer=c;; + text/x-c++) lexer=cpp;; + text/x-diff) lexer=diff;; + text/x-fortran) lexer=fortran;; + text/x-gawk) lexer=gawk;; + text/x-java) lexer=java;; + text/x-lisp) lexer=common-lisp;; + text/x-lua) lexer=lua;; + text/x-makefile) lexer=make;; + text/x-msdos-batch) lexer=bat;; + text/x-nawk) lexer=nawk;; + text/x-pascal) lexer=pascal;; + text/x-perl) lexer=perl;; + text/x-php) lexer=php;; + text/x-po) lexer=po;; + text/x-python) lexer=python;; + text/x-ruby) lexer=ruby;; + text/x-shellscript) lexer=sh;; + text/x-tcl) lexer=tcl;; + text/x-tex|text/x-texinfo) lexer=latex;; # FIXME: texinfo really needs its own lexer + + # Types that file outputs which pygmentize didn't support as of file 5.20, pygments 2.0 + # text/calendar + # text/inf + # text/PGP + # text/rtf + # text/texmacs + # text/vnd.graphviz + # text/x-bcpl + # text/x-info + # text/x-m4 + # text/x-vcard + # text/x-xmcd + + text/plain) # special filenames. TODO: insert more + case $(basename "$file") in + .zshrc) lexer=sh;; + esac + ;; + esac +fi + +encoding=$(file --mime-encoding --uncompress $file_common_opts "$file") +if [[ $encoding == "us-asciibinarybinary" ]]; then + encoding="us-ascii" +fi + +if [[ -n "$lexer" ]]; then + concat=cat + case $(file $file_common_opts --mime-type "$file") in + application/x-gzip) concat=zcat;; + application/x-bzip2) concat=bzcat;; + application/x-xz) concat=xzcat;; + esac + exec $concat "$file" | pygmentize -O inencoding=$encoding $PYGMENTIZE_OPTS $options -l $lexer +fi + +exit 1 diff --git a/external/lasso-builtins-generator-9.lasso b/external/lasso-builtins-generator-9.lasso new file mode 100755 index 00000000..01562995 --- /dev/null +++ b/external/lasso-builtins-generator-9.lasso @@ -0,0 +1,162 @@ +#!/usr/bin/lasso9 + +/* + Builtins Generator for Lasso 9 + + This is the shell script that was used to extract Lasso 9's built-in keywords + and generate most of the _lasso_builtins.py file. When run, it creates a file + containing the types, traits, methods, and members of the currently-installed + version of Lasso 9. + + A list of tags in Lasso 8 can be generated with this code: + + <?LassoScript + local('l8tags' = list, + 'l8libs' = array('Cache','ChartFX','Client','Database','File','HTTP', + 'iCal','Lasso','Link','List','PDF','Response','Stock','String', + 'Thread','Valid','WAP','XML')); + iterate(#l8libs, local('library')); + local('result' = namespace_load(#library)); + /iterate; + iterate(tags_list, local('i')); + #l8tags->insert(string_removeleading(#i, -pattern='_global_')); + /iterate; + #l8tags->sort; + iterate(#l8tags, local('i')); + string_lowercase(#i)+"<br>"; + /iterate; + +*/ + +output("This output statement is required for a complete list of methods.") +local(f) = file("_lasso_builtins-9.py") +#f->doWithClose => { + +#f->openTruncate +#f->writeString('# -*- coding: utf-8 -*- +""" + pygments.lexers._lasso_builtins + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Built-in Lasso types, traits, methods, and members. + + :copyright: Copyright 2006-'+date->year+' by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +') + +// Load and register contents of $LASSO9_MASTER_HOME/LassoModules/ +database_initialize + +// Load all of the libraries from builtins and lassoserver +// This forces all possible available types and methods to be registered +local(srcs = + (: + dir(sys_masterHomePath + '/LassoLibraries/builtins/')->eachFilePath, + dir(sys_masterHomePath + '/LassoLibraries/lassoserver/')->eachFilePath + ) +) + +with topLevelDir in delve(#srcs) +where not #topLevelDir->lastComponent->beginsWith('.') +do protect => { + handle_error => { + stdoutnl('Unable to load: ' + #topLevelDir + ' ' + error_msg) + } + library_thread_loader->loadLibrary(#topLevelDir) + stdoutnl('Loaded: ' + #topLevelDir) +} + +email_initialize +log_initialize +session_initialize + +local( + typesList = set(), + traitsList = set(), + unboundMethodsList = set(), + memberMethodsList = set() +) + +// types +with type in sys_listTypes +where not #type->asString->endsWith('$') // skip threads +do { + #typesList->insert(#type) +} + +// traits +with trait in sys_listTraits +where not #trait->asString->beginsWith('$') // skip combined traits +do { + #traitsList->insert(#trait) +} + +// member methods +with type in #typesList +do { + with method in #type->getType->listMethods + where #method->typeName == #type // skip inherited methods + let name = #method->methodName + where not #name->asString->endsWith('=') // skip setter methods + where #name->asString->isAlpha(1) // skip unpublished methods + do { + #memberMethodsList->insert(#name) + } +} +with trait in #traitsList +do { + with method in #trait->getType->provides + where #method->typeName == #trait // skip inherited methods + let name = #method->methodName + where not #name->asString->endsWith('=') // skip setter methods + where #name->asString->isAlpha(1) // skip unpublished methods + do { + #memberMethodsList->insert(#name) + } +} + +// unbound methods +with method in sys_listUnboundMethods +let name = #method->methodName +where not #name->asString->endsWith('=') // skip setter methods +where #name->asString->isAlpha(1) // skip unpublished methods +where #typesList !>> #name +where #traitsList !>> #name +do { + #unboundMethodsList->insert(#name) +} + +// write to file +with i in (: + pair(#typesList, "BUILTINS = { + 'Types': ( +"), + pair(#traitsList, " ), + 'Traits': ( +"), + pair(#unboundMethodsList, " ), + 'Unbound Methods': ( +"), + pair(#memberMethodsList, " ) +} +MEMBERS = { + 'Member Methods': ( +") +) +do { + #f->writeString(#i->second) + with t in (#i->first) + let ts = #t->asString + order by #ts + do { + #f->writeString(" '"+#ts->lowercase&asString+"',\n") + } +} + +#f->writeString(" ) +} +") + +} diff --git a/external/markdown-processor.py b/external/markdown-processor.py new file mode 100644 index 00000000..a3e178ec --- /dev/null +++ b/external/markdown-processor.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" + The Pygments Markdown Preprocessor + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This fragment is a Markdown_ preprocessor that renders source code + to HTML via Pygments. To use it, invoke Markdown like so:: + + import markdown + + html = markdown.markdown(someText, extensions=[CodeBlockExtension()]) + + This uses CSS classes by default, so use + ``pygmentize -S <some style> -f html > pygments.css`` + to create a stylesheet to be added to the website. + + You can then highlight source code in your markdown markup:: + + [sourcecode:lexer] + some code + [/sourcecode] + + .. _Markdown: https://pypi.python.org/pypi/Markdown + + :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +# Options +# ~~~~~~~ + +# Set to True if you want inline CSS styles instead of classes +INLINESTYLES = False + + +import re + +from markdown.preprocessors import Preprocessor +from markdown.extensions import Extension + +from pygments import highlight +from pygments.formatters import HtmlFormatter +from pygments.lexers import get_lexer_by_name, TextLexer + + +class CodeBlockPreprocessor(Preprocessor): + + pattern = re.compile(r'\[sourcecode:(.+?)\](.+?)\[/sourcecode\]', re.S) + + formatter = HtmlFormatter(noclasses=INLINESTYLES) + + def run(self, lines): + def repl(m): + try: + lexer = get_lexer_by_name(m.group(1)) + except ValueError: + lexer = TextLexer() + code = highlight(m.group(2), lexer, self.formatter) + code = code.replace('\n\n', '\n \n').replace('\n', '<br />') + return '\n\n<div class="code">%s</div>\n\n' % code + joined_lines = "\n".join(lines) + joined_lines = self.pattern.sub(repl, joined_lines) + return joined_lines.split("\n") + +class CodeBlockExtension(Extension): + def extendMarkdown(self, md, md_globals): + md.preprocessors.add('CodeBlockPreprocessor', CodeBlockPreprocessor(), '_begin') diff --git a/external/moin-parser.py b/external/moin-parser.py new file mode 100644 index 00000000..9cb082a2 --- /dev/null +++ b/external/moin-parser.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +""" + The Pygments MoinMoin Parser + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This is a MoinMoin parser plugin that renders source code to HTML via + Pygments; you need Pygments 0.7 or newer for this parser to work. + + To use it, set the options below to match your setup and put this file in + the data/plugin/parser subdirectory of your Moin instance, and give it the + name that the parser directive should have. For example, if you name the + file ``code.py``, you can get a highlighted Python code sample with this + Wiki markup:: + + {{{ + #!code python + [...] + }}} + + Additionally, if you set ATTACHMENTS below to True, Pygments will also be + called for all attachments for whose filenames there is no other parser + registered. + + You are responsible for including CSS rules that will map the Pygments CSS + classes to colors. You can output a stylesheet file with `pygmentize`, put + it into the `htdocs` directory of your Moin instance and then include it in + the `stylesheets` configuration option in the Moin config, e.g.:: + + stylesheets = [('screen', '/htdocs/pygments.css')] + + If you do not want to do that and are willing to accept larger HTML + output, you can set the INLINESTYLES option below to True. + + :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +# Options +# ~~~~~~~ + +# Set to True if you want to highlight attachments, in addition to +# {{{ }}} blocks. +ATTACHMENTS = True + +# Set to True if you want inline CSS styles instead of classes +INLINESTYLES = False + + +import sys + +from pygments import highlight +from pygments.lexers import get_lexer_by_name, get_lexer_for_filename, TextLexer +from pygments.formatters import HtmlFormatter +from pygments.util import ClassNotFound + + +# wrap lines in <span>s so that the Moin-generated line numbers work +class MoinHtmlFormatter(HtmlFormatter): + def wrap(self, source, outfile): + for line in source: + yield 1, '<span class="line">' + line[1] + '</span>' + +htmlformatter = MoinHtmlFormatter(noclasses=INLINESTYLES) +textlexer = TextLexer() +codeid = [0] + + +class Parser: + """ + MoinMoin Pygments parser. + """ + if ATTACHMENTS: + extensions = '*' + else: + extensions = [] + + Dependencies = [] + + def __init__(self, raw, request, **kw): + self.raw = raw + self.req = request + if "format_args" in kw: + # called from a {{{ }}} block + try: + self.lexer = get_lexer_by_name(kw['format_args'].strip()) + except ClassNotFound: + self.lexer = textlexer + return + if "filename" in kw: + # called for an attachment + filename = kw['filename'] + else: + # called for an attachment by an older moin + # HACK: find out the filename by peeking into the execution + # frame which might not always work + try: + frame = sys._getframe(1) + filename = frame.f_locals['filename'] + except: + filename = 'x.txt' + try: + self.lexer = get_lexer_for_filename(filename) + except ClassNotFound: + self.lexer = textlexer + + def format(self, formatter): + codeid[0] += 1 + id = "pygments_%s" % codeid[0] + w = self.req.write + w(formatter.code_area(1, id, start=1, step=1)) + w(formatter.rawHTML(highlight(self.raw, self.lexer, htmlformatter))) + w(formatter.code_area(0, id)) diff --git a/external/pygments.bashcomp b/external/pygments.bashcomp new file mode 100644 index 00000000..1299fdb9 --- /dev/null +++ b/external/pygments.bashcomp @@ -0,0 +1,38 @@ +#!bash +# +# Bash completion support for Pygments (the 'pygmentize' command). +# + +_pygmentize() +{ + local cur prev + + COMPREPLY=() + cur=`_get_cword` + prev=${COMP_WORDS[COMP_CWORD-1]} + + case "$prev" in + -f) + FORMATTERS=`pygmentize -L formatters | grep '* ' | cut -c3- | sed -e 's/,//g' -e 's/:$//'` + COMPREPLY=( $( compgen -W '$FORMATTERS' -- "$cur" ) ) + return 0 + ;; + -l) + LEXERS=`pygmentize -L lexers | grep '* ' | cut -c3- | sed -e 's/,//g' -e 's/:$//'` + COMPREPLY=( $( compgen -W '$LEXERS' -- "$cur" ) ) + return 0 + ;; + -S) + STYLES=`pygmentize -L styles | grep '* ' | cut -c3- | sed s/:$//` + COMPREPLY=( $( compgen -W '$STYLES' -- "$cur" ) ) + return 0 + ;; + esac + + if [[ "$cur" == -* ]]; then + COMPREPLY=( $( compgen -W '-f -l -S -L -g -O -P -F \ + -N -H -h -V -o' -- "$cur" ) ) + return 0 + fi +} +complete -F _pygmentize -o default pygmentize diff --git a/external/rst-directive.py b/external/rst-directive.py new file mode 100644 index 00000000..f81677b6 --- /dev/null +++ b/external/rst-directive.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +""" + The Pygments reStructuredText directive + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This fragment is a Docutils_ 0.5 directive that renders source code + (to HTML only, currently) via Pygments. + + To use it, adjust the options below and copy the code into a module + that you import on initialization. The code then automatically + registers a ``sourcecode`` directive that you can use instead of + normal code blocks like this:: + + .. sourcecode:: python + + My code goes here. + + If you want to have different code styles, e.g. one with line numbers + and one without, add formatters with their names in the VARIANTS dict + below. You can invoke them instead of the DEFAULT one by using a + directive option:: + + .. sourcecode:: python + :linenos: + + My code goes here. + + Look at the `directive documentation`_ to get all the gory details. + + .. _Docutils: http://docutils.sf.net/ + .. _directive documentation: + http://docutils.sourceforge.net/docs/howto/rst-directives.html + + :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +# Options +# ~~~~~~~ + +# Set to True if you want inline CSS styles instead of classes +INLINESTYLES = False + +from pygments.formatters import HtmlFormatter + +# The default formatter +DEFAULT = HtmlFormatter(noclasses=INLINESTYLES) + +# Add name -> formatter pairs for every variant you want to use +VARIANTS = { + # 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True), +} + + +from docutils import nodes +from docutils.parsers.rst import directives, Directive + +from pygments import highlight +from pygments.lexers import get_lexer_by_name, TextLexer + +class Pygments(Directive): + """ Source code syntax hightlighting. + """ + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = True + option_spec = dict([(key, directives.flag) for key in VARIANTS]) + has_content = True + + def run(self): + self.assert_has_content() + try: + lexer = get_lexer_by_name(self.arguments[0]) + except ValueError: + # no lexer found - use the text one instead of an exception + lexer = TextLexer() + # take an arbitrary option if more than one is given + formatter = self.options and VARIANTS[list(self.options)[0]] or DEFAULT + parsed = highlight(u'\n'.join(self.content), lexer, formatter) + return [nodes.raw('', parsed, format='html')] + +directives.register_directive('sourcecode', Pygments) |