From 1632edd8fbb7d824252bd12e52fe275234f45ec9 Mon Sep 17 00:00:00 2001 From: rrt Date: Tue, 7 Jun 2016 22:32:42 +0100 Subject: autopygmentize: put MIME types back in sorted order text/x-crystal was added out of order. Also bump the copyright year. --- external/autopygmentize | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/external/autopygmentize b/external/autopygmentize index f18cac09..07cfdaa9 100755 --- a/external/autopygmentize +++ b/external/autopygmentize @@ -1,6 +1,6 @@ #!/bin/bash # Best effort auto-pygmentization with transparent decompression -# by Reuben Thomas 2008-2015 +# by Reuben Thomas 2008-2016 # This program is in the public domain. # Strategy: first see if pygmentize can find a lexer; if not, ask file; if that finds nothing, fail @@ -25,6 +25,7 @@ if [[ "$lexer" == text ]]; then text/x-awk) lexer=awk;; text/x-c) lexer=c;; text/x-c++) lexer=cpp;; + text/x-crystal) lexer=crystal;; text/x-diff) lexer=diff;; text/x-fortran) lexer=fortran;; text/x-gawk) lexer=gawk;; @@ -40,7 +41,6 @@ if [[ "$lexer" == text ]]; then text/x-po) lexer=po;; text/x-python) lexer=python;; text/x-ruby) lexer=ruby;; - text/x-crystal) lexer=crystal;; text/x-shellscript) lexer=sh;; text/x-tcl) lexer=tcl;; text/x-tex|text/x-texinfo) lexer=latex;; # FIXME: texinfo really needs its own lexer -- cgit v1.2.1 From 1996fd14d14b71a0cc336b3fed83c905fc63467c Mon Sep 17 00:00:00 2001 From: rrt Date: Tue, 7 Jun 2016 22:54:32 +0100 Subject: autopygmentize: use hexdump on binary files Not strictly using autopygmentize, but makes things more legible. --- external/autopygmentize | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/external/autopygmentize b/external/autopygmentize index 07cfdaa9..9b030fac 100755 --- a/external/autopygmentize +++ b/external/autopygmentize @@ -66,19 +66,26 @@ if [[ "$lexer" == text ]]; then esac fi +# Find a preprocessor for compressed files +concat=cat +case $(file $file_common_opts --mime-type "$file") in + application/x-gzip) concat=zcat;; + application/x-bzip2) concat=bzcat;; + application/x-xz) concat=xzcat;; +esac + +# Find a reader: either a suitable lexer, or hd for binary files +reader="" encoding=$(file --mime-encoding --uncompress $file_common_opts "$file") if [[ $encoding == "binary" ]]; then - encoding="latin1" + reader=hd +elif [[ -n "$lexer" ]]; then + reader="pygmentize -O inencoding=$encoding $PYGMENTIZE_OPTS $options -l $lexer" fi -if [[ -n "$lexer" ]]; then - concat=cat - case $(file $file_common_opts --mime-type "$file") in - application/x-gzip) concat=zcat;; - application/x-bzip2) concat=bzcat;; - application/x-xz) concat=xzcat;; - esac - exec $concat "$file" | pygmentize -O inencoding=$encoding $PYGMENTIZE_OPTS $options -l $lexer +# If we found a reader, run it +if [[ -n "$reader" ]]; then + exec $concat "$file" | $reader fi exit 1 -- cgit v1.2.1 From e10e63b7cdb19e95bf7cef3502268521cfc81235 Mon Sep 17 00:00:00 2001 From: cocoatomo Date: Tue, 14 Jun 2016 01:33:42 +0900 Subject: Insert a missing argument "self" --- doc/docs/lexerdevelopment.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/docs/lexerdevelopment.rst b/doc/docs/lexerdevelopment.rst index fd6e76b9..9109180d 100644 --- a/doc/docs/lexerdevelopment.rst +++ b/doc/docs/lexerdevelopment.rst @@ -361,7 +361,7 @@ There are a few more things you can do with states: tokens = {...} def get_tokens_unprocessed(self, text, stack=('root', 'otherstate')): - for item in RegexLexer.get_tokens_unprocessed(text, stack): + for item in RegexLexer.get_tokens_unprocessed(self, text, stack): yield item Some lexers like the `PhpLexer` use this to make the leading `` Date: Mon, 20 Jun 2016 23:10:46 +0100 Subject: Fall back to od -x, and use hexdump lexer --- external/autopygmentize | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/external/autopygmentize b/external/autopygmentize index 9b030fac..26fe365c 100755 --- a/external/autopygmentize +++ b/external/autopygmentize @@ -74,14 +74,22 @@ case $(file $file_common_opts --mime-type "$file") in application/x-xz) concat=xzcat;; esac -# Find a reader: either a suitable lexer, or hd for binary files -reader="" +# Find a suitable lexer, preceded by a hex dump for binary files +prereader="" encoding=$(file --mime-encoding --uncompress $file_common_opts "$file") if [[ $encoding == "binary" ]]; then - reader=hd -elif [[ -n "$lexer" ]]; then + prereader="od -x" # POSIX fallback + if [[ -n $(which hd) ]]; then + prereader="hd" # preferred + fi + lexer=hexdump +fi +if [[ -n "$lexer" ]]; then reader="pygmentize -O inencoding=$encoding $PYGMENTIZE_OPTS $options -l $lexer" fi +if [[ -n "$prereader" ]]; then + reader="$prereader | $reader" +fi # If we found a reader, run it if [[ -n "$reader" ]]; then -- cgit v1.2.1 From 4564fe0139b6fb5cac8f0c6faf5c977c6accf739 Mon Sep 17 00:00:00 2001 From: rrt Date: Mon, 20 Jun 2016 23:13:33 +0100 Subject: Fix a couple of errors in the previous commit --- external/autopygmentize | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/external/autopygmentize b/external/autopygmentize index 26fe365c..d2d05970 100755 --- a/external/autopygmentize +++ b/external/autopygmentize @@ -83,17 +83,19 @@ if [[ $encoding == "binary" ]]; then prereader="hd" # preferred fi lexer=hexdump + encoding=latin1 fi if [[ -n "$lexer" ]]; then reader="pygmentize -O inencoding=$encoding $PYGMENTIZE_OPTS $options -l $lexer" fi -if [[ -n "$prereader" ]]; then - reader="$prereader | $reader" -fi # If we found a reader, run it if [[ -n "$reader" ]]; then - exec $concat "$file" | $reader + if [[ -n "$prereader" ]]; then + exec $concat "$file" | $prereader | $reader + else + exec $concat "$file" | $reader + fi fi exit 1 -- cgit v1.2.1 From 346229c9c36d45086db283f1c5612355e55d6efe Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 12 Jul 2016 20:00:52 +0100 Subject: Add '1e6' as float token for Python3Lexer The parent class (PythonLexer) has this, but it's missing from the subclass. 1e6 is a valid float token in Python 3 as well. --- pygments/lexers/python.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py index 35635ed1..d41dd08d 100644 --- a/pygments/lexers/python.py +++ b/pygments/lexers/python.py @@ -362,6 +362,7 @@ class Python3Lexer(RegexLexer): ] tokens['numbers'] = [ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), + (r'\d+[eE][+-]?[0-9]+j?', Number.Float), (r'0[oO][0-7]+', Number.Oct), (r'0[bB][01]+', Number.Bin), (r'0[xX][a-fA-F0-9]+', Number.Hex), -- cgit v1.2.1 From e425a9405f1fc05937a06df6b25815156c5a592d Mon Sep 17 00:00:00 2001 From: David Corbett Date: Wed, 13 Jul 2016 21:24:07 -0400 Subject: Fix Batch variables after IF --- pygments/lexers/shell.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py index a5933afb..e9a7595b 100644 --- a/pygments/lexers/shell.py +++ b/pygments/lexers/shell.py @@ -478,13 +478,16 @@ class BatchLexer(RegexLexer): using(this, state='variable')), '#pop'), (r'(exist%s)(%s%s)' % (_token_terminator, _space, _stoken), bygroups(Keyword, using(this, state='text')), '#pop'), - (r'(%s%s?)(==)(%s?%s)' % (_stoken, _space, _space, _stoken), - bygroups(using(this, state='text'), Operator, - using(this, state='text')), '#pop'), (r'(%s%s)(%s)(%s%s)' % (_number, _space, _opword, _space, _number), bygroups(using(this, state='arithmetic'), Operator.Word, using(this, state='arithmetic')), '#pop'), - (r'(%s%s)(%s)(%s%s)' % (_stoken, _space, _opword, _space, _stoken), + (_stoken, using(this, state='text'), ('#pop', 'if2')), + ], + 'if2': [ + (r'(%s?)(==)(%s?%s)' % (_space, _space, _stoken), + bygroups(using(this, state='text'), Operator, + using(this, state='text')), '#pop'), + (r'(%s)(%s)(%s%s)' % (_space, _opword, _space, _stoken), bygroups(using(this, state='text'), Operator.Word, using(this, state='text')), '#pop') ], -- cgit v1.2.1 From 8e0c2365df9bbb51c3f17b935eb864b1b8e23c86 Mon Sep 17 00:00:00 2001 From: Ren? Schwaiger Date: Mon, 18 Jul 2016 15:44:15 +0200 Subject: Add ?Rainbow Dash? syntax highlighting style This bright and colorful style is a port of my TextMate theme [1]. It is based on ?Mac Classic? by Chris Thomas. [1]: http://sanssecours.github.io/Rainbow-Dash.tmbundle --- AUTHORS | 1 + pygments/styles/__init__.py | 3 +- pygments/styles/rainbow_dash.py | 89 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 pygments/styles/rainbow_dash.py diff --git a/AUTHORS b/AUTHORS index 66377ead..efbad179 100644 --- a/AUTHORS +++ b/AUTHORS @@ -173,6 +173,7 @@ Other contributors, listed alphabetically, are: * Matteo Sasso -- Common Lisp lexer * Joe Schafer -- Ada lexer * Ken Schutte -- Matlab lexers +* René Schwaiger -- Rainbow Dash style * Sebastian Schweizer -- Whiley lexer * Tassilo Schweyer -- Io, MOOCode lexers * Ted Shaw -- AutoIt lexer diff --git a/pygments/styles/__init__.py b/pygments/styles/__init__.py index 4efd196e..d1e7bb50 100644 --- a/pygments/styles/__init__.py +++ b/pygments/styles/__init__.py @@ -41,7 +41,8 @@ STYLE_MAP = { 'lovelace': 'lovelace::LovelaceStyle', 'algol': 'algol::AlgolStyle', 'algol_nu': 'algol_nu::Algol_NuStyle', - 'arduino': 'arduino::ArduinoStyle' + 'arduino': 'arduino::ArduinoStyle', + 'rainbow_dash': 'rainbow_dash::RainbowDashStyle' } diff --git a/pygments/styles/rainbow_dash.py b/pygments/styles/rainbow_dash.py new file mode 100644 index 00000000..ac8d56d2 --- /dev/null +++ b/pygments/styles/rainbow_dash.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +""" + pygments.styles.rainbow_dash + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + A bright and colorful syntax highlighting `theme`. + + .. _theme: http://sanssecours.github.io/Rainbow-Dash.tmbundle + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import (Comment, Error, Generic, Name, Number, Operator, + String, Text, Whitespace, Keyword) + +BLUE_LIGHT = '#0080ff' +BLUE = '#2c5dcd' +GREEN = '#00cc66' +GREEN_LIGHT = '#ccffcc' +GREEN_NEON = '#00cc00' +GREY = '#aaaaaa' +GREY_LIGHT = '#cbcbcb' +GREY_DARK = '#4d4d4d' +PURPLE = '#5918bb' +RED = '#cc0000' +RED_DARK = '#c5060b' +RED_LIGHT = '#ffcccc' +RED_BRIGHT = '#ff0000' +WHITE = '#ffffff' +TURQUOISE = '#318495' +ORANGE = '#ff8000' + + +class RainbowDashStyle(Style): + """ + A bright and colorful syntax highlighting theme. + """ + + background_color = WHITE + + styles = { + Comment: 'italic {}'.format(BLUE_LIGHT), + Comment.Preproc: 'noitalic', + Comment.Special: 'bold', + + Error: 'bg:{} {}'.format(RED, WHITE), + + Generic.Deleted: 'border:{} bg:{}'.format(RED_DARK, RED_LIGHT), + Generic.Emph: 'italic', + Generic.Error: RED_BRIGHT, + Generic.Heading: 'bold {}'.format(BLUE), + Generic.Inserted: 'border:{} bg:{}'.format(GREEN_NEON, GREEN_LIGHT), + Generic.Output: GREY, + Generic.Prompt: 'bold {}'.format(BLUE), + Generic.Strong: 'bold', + Generic.Subheading: 'bold {}'.format(BLUE), + Generic.Traceback: RED_DARK, + + Keyword: 'bold {}'.format(BLUE), + Keyword.Pseudo: 'nobold', + Keyword.Type: PURPLE, + + Name.Attribute: 'italic {}'.format(BLUE), + Name.Builtin: 'bold {}'.format(PURPLE), + Name.Class: 'underline', + Name.Constant: TURQUOISE, + Name.Decorator: 'bold {}'.format(ORANGE), + Name.Entity: 'bold {}'.format(PURPLE), + Name.Exception: 'bold {}'.format(PURPLE), + Name.Function: 'bold {}'.format(ORANGE), + Name.Tag: 'bold {}'.format(BLUE), + + Number: 'bold {}'.format(PURPLE), + + Operator: BLUE, + Operator.Word: 'bold', + + String: GREEN, + String.Doc: 'italic', + String.Escape: 'bold {}'.format(RED_DARK), + String.Other: TURQUOISE, + String.Symbol: 'bold {}'.format(RED_DARK), + + Text: GREY_DARK, + + Whitespace: GREY_LIGHT + } -- cgit v1.2.1 From 4b40a6445cbb7aae8a04321f1a8b394e9fcff363 Mon Sep 17 00:00:00 2001 From: daisuzu Date: Sat, 30 Jul 2016 21:17:05 +0900 Subject: Add map support to ProtoBufLexer --- pygments/lexers/dsls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/dsls.py b/pygments/lexers/dsls.py index 312d5f5e..4044b7c5 100644 --- a/pygments/lexers/dsls.py +++ b/pygments/lexers/dsls.py @@ -36,7 +36,7 @@ class ProtoBufLexer(RegexLexer): tokens = { 'root': [ (r'[ \t]+', Text), - (r'[,;{}\[\]()]', Punctuation), + (r'[,;{}\[\]()<>]', Punctuation), (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single), (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment.Multiline), (words(( -- cgit v1.2.1 -- cgit v1.2.1 From e9a803bbaf3fee217f64f7bf32a18278770dfe36 Mon Sep 17 00:00:00 2001 From: Josiah Schwab Date: Thu, 18 Aug 2016 18:44:44 -0700 Subject: Add exponent-letter D to Fortran lexer In the Fortran specification, the exponent-letter within a signed-real-literal-constant is E or D. Therefore, both 3.14159e0 and 3.14159d0 should be classified as a Number.Float. The current behavior is >>> print(highlight("3.14159d0", FortranLexer(), HtmlFormatter()))
3.14159d0
where the only the significand is classified as a Number.Float while the exponent-letter and exponent are classified as Name.Variable. After this patch the behavior is >>> print(highlight("3.14159d0", FortranLexer(), HtmlFormatter()))
3.14159d0
which is the expected behavior. --- pygments/lexers/fortran.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygments/lexers/fortran.py b/pygments/lexers/fortran.py index e2f95b11..6cc8f2c8 100644 --- a/pygments/lexers/fortran.py +++ b/pygments/lexers/fortran.py @@ -156,8 +156,8 @@ class FortranLexer(RegexLexer): 'nums': [ (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer), - (r'[+-]?\d*\.\d+(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float), - (r'[+-]?\d+\.\d*(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float), + (r'[+-]?\d*\.\d+([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), + (r'[+-]?\d+\.\d*([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), ], } -- cgit v1.2.1 From b0b0ff2a0c3dc3990c409501d1ca38596d031939 Mon Sep 17 00:00:00 2001 From: devoncarew1 Date: Sun, 28 Aug 2016 03:30:42 +0000 Subject: javascript.py edited online with Bitbucket --- pygments/lexers/javascript.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pygments/lexers/javascript.py b/pygments/lexers/javascript.py index a23ba184..6bd97dbe 100644 --- a/pygments/lexers/javascript.py +++ b/pygments/lexers/javascript.py @@ -366,9 +366,10 @@ class DartLexer(RegexLexer): (r'\b(assert|break|case|catch|continue|default|do|else|finally|for|' r'if|in|is|new|return|super|switch|this|throw|try|while)\b', Keyword), - (r'\b(abstract|const|extends|factory|final|get|implements|' - r'native|operator|set|static|typedef|var)\b', Keyword.Declaration), - (r'\b(bool|double|Dynamic|int|num|Object|String|void)\b', Keyword.Type), + (r'\b(abstract|async|await|const|extends|factory|final|get|' + r'implements|native|operator|set|static|sync|typedef|var|with|' + r'yield)\b', Keyword.Declaration), + (r'\b(bool|double|dynamic|int|num|Object|String|void)\b', Keyword.Type), (r'\b(false|null|true)\b', Keyword.Constant), (r'[~!%^&*+=|?:<>/-]|as\b', Operator), (r'[a-zA-Z_$]\w*:', Name.Label), -- cgit v1.2.1 From fc88fabbffb273f0109a52a74d600d2ae567fe4b Mon Sep 17 00:00:00 2001 From: Ash Searle Date: Thu, 1 Sep 2016 20:39:24 +0100 Subject: Fix for floats with leading/trailing ., reclassify es2015 arrow notation as punctuation and add new es2015 regex flags (sticky and unicode) --- pygments/lexers/javascript.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pygments/lexers/javascript.py b/pygments/lexers/javascript.py index a23ba184..4b47b15a 100644 --- a/pygments/lexers/javascript.py +++ b/pygments/lexers/javascript.py @@ -53,7 +53,7 @@ class JavascriptLexer(RegexLexer): 'slashstartsregex': [ include('commentsandwhitespace'), (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' - r'([gim]+\b|\B)', String.Regex, '#pop'), + r'([gimuy]+\b|\B)', String.Regex, '#pop'), (r'(?=/)', Text, ('#pop', 'badregex')), default('#pop') ], @@ -64,9 +64,14 @@ class JavascriptLexer(RegexLexer): (r'\A#! ?/.*?\n', Comment.Hashbang), # recognized by node.js (r'^(?=\s|/|