diff options
98 files changed, 4033 insertions, 2914 deletions
@@ -8,6 +8,7 @@ syntax: glob .project .tags .tox +.cache/ Pygments.egg-info/* TAGS build/* @@ -72,7 +72,7 @@ Other contributors, listed alphabetically, are: * Alex Gosse -- TrafficScript lexer * Patrick Gotthardt -- PHP namespaces support * Olivier Guibe -- Asymptote lexer -* Jordi Gutiérrez Hermoso -- Octave lexer +* Phil Hagelberg -- Fennel lexer * Florian Hahn -- Boogie lexer * Martin Harriman -- SNOBOL lexer * Matthew Harrison -- SVG formatter @@ -81,6 +81,7 @@ Other contributors, listed alphabetically, are: * Aslak Hellesøy -- Gherkin lexer * Greg Hendershott -- Racket lexer * Justin Hendrick -- ParaSail lexer +* Jordi Gutiérrez Hermoso -- Octave lexer * David Hess, Fish Software, Inc. -- Objective-J lexer * Varun Hiremath -- Debian control lexer * Rob Hoelz -- Perl 6 lexer @@ -160,6 +161,7 @@ Other contributors, listed alphabetically, are: * Elias Rabel -- Fortran fixed form lexer * raichoo -- Idris lexer * Kashif Rasul -- CUDA lexer +* Nathan Reed -- HLSL lexer * Justin Reidy -- MXML lexer * Norman Richards -- JSON lexer * Corey Richardson -- Rust lexer updates @@ -6,9 +6,49 @@ Issue numbers refer to the tracker at pull request numbers to the requests at <https://bitbucket.org/birkenfeld/pygments-main/pull-requests/merged>. +Version 2.3.1 +------------- +(released Dec 16, 2018) + +- Updated lexers: + + * ASM (PR#784) + * Chapel (PR#735) + * Clean (PR#621) + * CSound (PR#684) + * Elm (PR#744) + * Fortran (PR#747) + * GLSL (PR#740) + * Hy (PR#754) + * PowerShell (PR#705) + * Python (PR#720, #1299, PR#715) + * SLexer (PR#680) + * YAML (PR#762, PR#724) + +Version 2.3.0 +------------- +(released Nov 25, 2018) + +- Added lexers: + + * Fennel (PR#783) + * HLSL (PR#675) + +- Updated lexers: + + * Dockerfile (PR#714) + +- Minimum Python versions changed to 2.7 and 3.5 +- Added support for Python 3.7 generator changes (PR#772) +- Fix incorrect token type in SCSS for single-quote strings (#1322) +- Use `terminal256` formatter if `TERM` contains `256` (PR#666) +- Fix incorrect handling of GitHub style fences in Markdown (PR#741, #1389) +- Fix `%a` not being highlighted in Python3 strings (PR#727) + + Version 2.2.0 ------------- -(release Jan 22, 2017) +(released Jan 22, 2017) - Added lexers: @@ -63,3 +63,9 @@ tox-test: tox-test-coverage: @tox -- --with-coverage --cover-package=pygments --cover-erase $(TEST) + +RLMODULES = pygments.lexers + +regexlint: + @if [ -z "$(REGEXLINT)" ]; then echo "Please set REGEXLINT=checkout path"; exit 1; fi + PYTHONPATH=`pwd`:$(REGEXLINT) $(REGEXLINT)/regexlint/cmdline.py $(RLMODULES) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml new file mode 100644 index 00000000..fc745f0f --- /dev/null +++ b/bitbucket-pipelines.yml @@ -0,0 +1,34 @@ +pipelines: + default: + - step: + name: Test on Python 2.7 + image: python:2.7 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py27 + - step: + name: Test on Python 3.4 + image: python:3.4 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py34 + - step: + name: Test on Python 3.6 + image: python:3.6 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py36 + - step: + name: Test on Python 3.7 + image: python:3.7 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py37 diff --git a/doc/docs/quickstart.rst b/doc/docs/quickstart.rst index dba7698a..3a823e7f 100644 --- a/doc/docs/quickstart.rst +++ b/doc/docs/quickstart.rst @@ -39,7 +39,7 @@ Here is a small example for highlighting Python code: from pygments.formatters import HtmlFormatter code = 'print "Hello World"' - print highlight(code, PythonLexer(), HtmlFormatter()) + print(highlight(code, PythonLexer(), HtmlFormatter())) which prints something like this: @@ -56,7 +56,7 @@ can be produced by: .. sourcecode:: python - print HtmlFormatter().get_style_defs('.highlight') + print(HtmlFormatter().get_style_defs('.highlight')) The argument to :func:`get_style_defs` is used as an additional CSS selector: the output may look like this: diff --git a/doc/faq.rst b/doc/faq.rst index f375828b..172929e0 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -35,8 +35,8 @@ and in this case, source code! What are the system requirements? --------------------------------- -Pygments only needs a standard Python install, version 2.6 or higher or version -3.3 or higher for Python 3. No additional libraries are needed. +Pygments only needs a standard Python install, version 2.7 or higher or version +3.5 or higher for Python 3. No additional libraries are needed. How can I use Pygments? ----------------------- diff --git a/doc/languages.rst b/doc/languages.rst index 7fa8eb2f..e5399403 100644 --- a/doc/languages.rst +++ b/doc/languages.rst @@ -37,6 +37,7 @@ Programming languages * `Ezhil <http://ezhillang.org>`_ Ezhil - A Tamil programming language * Factor * Fancy +* `Fennel <https://fennel-lang.org/>`_ * Fortran * F# * GAP @@ -44,6 +45,7 @@ Programming languages * GL shaders * Groovy * `Haskell <http://www.haskell.org>`_ (incl. Literate Haskell) +* HLSL * IDL * Io * Java diff --git a/pygments/__init__.py b/pygments/__init__.py index 394a85f2..19aafdeb 100644 --- a/pygments/__init__.py +++ b/pygments/__init__.py @@ -29,7 +29,7 @@ import sys from pygments.util import StringIO, BytesIO -__version__ = '2.2.0' +__version__ = '2.3.0' __docformat__ = 'restructuredtext' __all__ = ['lex', 'format', 'highlight'] diff --git a/pygments/formatters/img.py b/pygments/formatters/img.py index 2fb0dea5..6fafc476 100644 --- a/pygments/formatters/img.py +++ b/pygments/formatters/img.py @@ -237,7 +237,7 @@ class ImageFormatter(Formatter): bold and italic fonts will be generated. This really should be a monospace font to look sane. - Default: "Bitstream Vera Sans Mono" on Windows, Courier New on \*nix + Default: "Bitstream Vera Sans Mono" on Windows, Courier New on \\*nix `font_size` The font size in points to be used. diff --git a/pygments/lexers/_cocoa_builtins.py b/pygments/lexers/_cocoa_builtins.py index 064167ff..f55e9dd7 100644 --- a/pygments/lexers/_cocoa_builtins.py +++ b/pygments/lexers/_cocoa_builtins.py @@ -41,23 +41,23 @@ if __name__ == '__main__': # pragma: no cover headerFilePath = frameworkHeadersDir + f content = open(headerFilePath).read() - res = re.findall('(?<=@interface )\w+', content) + res = re.findall(r'(?<=@interface )\w+', content) for r in res: all_interfaces.add(r) - res = re.findall('(?<=@protocol )\w+', content) + res = re.findall(r'(?<=@protocol )\w+', content) for r in res: all_protocols.add(r) - res = re.findall('(?<=typedef enum )\w+', content) + res = re.findall(r'(?<=typedef enum )\w+', content) for r in res: all_primitives.add(r) - res = re.findall('(?<=typedef struct )\w+', content) + res = re.findall(r'(?<=typedef struct )\w+', content) for r in res: all_primitives.add(r) - res = re.findall('(?<=typedef const struct )\w+', content) + res = re.findall(r'(?<=typedef const struct )\w+', content) for r in res: all_primitives.add(r) diff --git a/pygments/lexers/_csound_builtins.py b/pygments/lexers/_csound_builtins.py index e5a9aaf7..56b5a452 100644 --- a/pygments/lexers/_csound_builtins.py +++ b/pygments/lexers/_csound_builtins.py @@ -3,1344 +3,1658 @@ pygments.lexers._csound_builtins ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2018 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -# Opcodes in Csound 6.05 from -# csound --list-opcodes -# except -# cggoto <http://www.csounds.com/manual/html/cggoto.html> -# cigoto <http://www.csounds.com/manual/html/cigoto.html> -# cingoto (undocumented) -# ckgoto <http://www.csounds.com/manual/html/ckgoto.html> -# cngoto <http://www.csounds.com/manual/html/cngoto.html> -# endin <http://www.csounds.com/manual/html/endin.html -# endop <http://www.csounds.com/manual/html/endop.html -# goto <http://www.csounds.com/manual/html/goto.html> -# igoto <http://www.csounds.com/manual/html/igoto.html> -# instr <http://www.csounds.com/manual/html/instr.html> -# kgoto <http://www.csounds.com/manual/html/kgoto.html> -# loop_ge <http://www.csounds.com/manual/html/loop_ge.html> -# loop_gt <http://www.csounds.com/manual/html/loop_gt.html> -# loop_le <http://www.csounds.com/manual/html/loop_le.html> -# loop_lt <http://www.csounds.com/manual/html/loop_lt.html> -# opcode <http://www.csounds.com/manual/html/opcode.html> -# return <http://www.csounds.com/manual/html/return.html> -# rigoto <http://www.csounds.com/manual/html/rigoto.html> -# tigoto <http://www.csounds.com/manual/html/tigoto.html> -# timout <http://www.csounds.com/manual/html/timout.html> -# which are treated as keywords; the scoreline opcodes -# scoreline <http://www.csounds.com/manual/html/scoreline.html> -# scoreline_i <http://www.csounds.com/manual/html/scoreline_i.html> -# which allow Csound Score highlighting; the pyrun opcodes -# <http://www.csounds.com/manual/html/pyrun.html> -# pylrun -# pylruni -# pylrunt -# pyrun -# pyruni -# pyrunt -# which allow Python highlighting; and the Lua opcodes -# lua_exec <http://www.csounds.com/manual/html/lua_exec.html> -# lua_opdef <http://www.csounds.com/manual/html/lua_opdef.html> -# which allow Lua highlighting. -OPCODES = set(( - 'ATSadd', - 'ATSaddnz', - 'ATSbufread', - 'ATScross', - 'ATSinfo', - 'ATSinterpread', - 'ATSpartialtap', - 'ATSread', - 'ATSreadnz', - 'ATSsinnoi', - 'FLbox', - 'FLbutBank', - 'FLbutton', - 'FLcloseButton', - 'FLcolor', - 'FLcolor2', - 'FLcount', - 'FLexecButton', - 'FLgetsnap', - 'FLgroup', - 'FLgroupEnd', - 'FLgroup_end', - 'FLhide', - 'FLhvsBox', - 'FLhvsBoxSetValue', - 'FLjoy', - 'FLkeyIn', - 'FLknob', - 'FLlabel', - 'FLloadsnap', - 'FLmouse', - 'FLpack', - 'FLpackEnd', - 'FLpack_end', - 'FLpanel', - 'FLpanelEnd', - 'FLpanel_end', - 'FLprintk', - 'FLprintk2', - 'FLroller', - 'FLrun', - 'FLsavesnap', - 'FLscroll', - 'FLscrollEnd', - 'FLscroll_end', - 'FLsetAlign', - 'FLsetBox', - 'FLsetColor', - 'FLsetColor2', - 'FLsetFont', - 'FLsetPosition', - 'FLsetSize', - 'FLsetSnapGroup', - 'FLsetText', - 'FLsetTextColor', - 'FLsetTextSize', - 'FLsetTextType', - 'FLsetVal', - 'FLsetVal_i', - 'FLsetVali', - 'FLsetsnap', - 'FLshow', - 'FLslidBnk', - 'FLslidBnk2', - 'FLslidBnk2Set', - 'FLslidBnk2Setk', - 'FLslidBnkGetHandle', - 'FLslidBnkSet', - 'FLslidBnkSetk', - 'FLslider', - 'FLtabs', - 'FLtabsEnd', - 'FLtabs_end', - 'FLtext', - 'FLupdate', - 'FLvalue', - 'FLvkeybd', - 'FLvslidBnk', - 'FLvslidBnk2', - 'FLxyin', - 'MixerClear', - 'MixerGetLevel', - 'MixerReceive', - 'MixerSend', - 'MixerSetLevel', - 'MixerSetLevel_i', - 'OSCinit', - 'OSClisten', - 'OSCsend', - 'a', - 'abs', - 'active', - 'adsr', - 'adsyn', - 'adsynt', - 'adsynt2', - 'aftouch', - 'alpass', - 'alwayson', - 'ampdb', - 'ampdbfs', - 'ampmidi', - 'ampmidid', - 'areson', - 'aresonk', - 'array', - 'atone', - 'atonek', - 'atonex', - 'babo', - 'balance', - 'bamboo', - 'barmodel', - 'bbcutm', - 'bbcuts', - 'betarand', - 'bexprnd', - 'bformdec', - 'bformdec1', - 'bformenc', - 'bformenc1', - 'binit', - 'biquad', - 'biquada', - 'birnd', - 'bqrez', - 'buchla', - 'butbp', - 'butbr', - 'buthp', - 'butlp', - 'butterbp', - 'butterbr', - 'butterhp', - 'butterlp', - 'button', - 'buzz', - 'c2r', - 'cabasa', - 'cauchy', - 'cauchyi', - 'ceil', - 'cell', - 'cent', - 'centroid', - 'ceps', - #'cggoto', - 'chanctrl', - 'changed', - 'chani', - 'chano', - 'chebyshevpoly', - 'checkbox', - 'chn_S', - 'chn_a', - 'chn_k', - 'chnclear', - 'chnexport', - 'chnget', - 'chnmix', - 'chnparams', - 'chnset', - 'chuap', - #'cigoto', - #'cingoto', - #'ckgoto', - 'clear', - 'clfilt', - 'clip', - 'clockoff', - 'clockon', - 'cmplxprod', - #'cngoto', - 'comb', - 'combinv', - 'compilecsd', - 'compileorc', - 'compilestr', - 'compress', - 'connect', - 'control', - 'convle', - 'convolve', - 'copy2ftab', - 'copy2ttab', - 'copya2ftab', - 'copyf2array', - 'cos', - 'cosh', - 'cosinv', - 'cosseg', - 'cossegb', - 'cossegr', - 'cps2pch', - 'cpsmidi', - 'cpsmidib', - 'cpsmidinn', - 'cpsoct', - 'cpspch', - 'cpstmid', - 'cpstun', - 'cpstuni', - 'cpsxpch', - 'cpuprc', - 'cross2', - 'crossfm', - 'crossfmi', - 'crossfmpm', - 'crossfmpmi', - 'crosspm', - 'crosspmi', - 'crunch', - 'ctlchn', - 'ctrl14', - 'ctrl21', - 'ctrl7', - 'ctrlinit', - 'cuserrnd', - 'dam', - 'date', - 'dates', - 'db', - 'dbamp', - 'dbfsamp', - 'dcblock', - 'dcblock2', - 'dconv', - 'delay', - 'delay1', - 'delayk', - 'delayr', - 'delayw', - 'deltap', - 'deltap3', - 'deltapi', - 'deltapn', - 'deltapx', - 'deltapxw', - 'denorm', - 'diff', - 'diskgrain', - 'diskin', - 'diskin2', - 'dispfft', - 'display', - 'distort', - 'distort1', - 'divz', - 'doppler', - 'downsamp', - 'dripwater', - 'dumpk', - 'dumpk2', - 'dumpk3', - 'dumpk4', - 'duserrnd', - 'dust', - 'dust2', - #'endin', - #'endop', - 'envlpx', - 'envlpxr', - 'ephasor', - 'eqfil', - 'evalstr', - 'event', - 'event_i', - 'exciter', - 'exitnow', - 'exp', - 'expcurve', - 'expon', - 'exprand', - 'exprandi', - 'expseg', - 'expsega', - 'expsegb', - 'expsegba', - 'expsegr', - 'fareylen', - 'fareyleni', - 'faustaudio', - 'faustcompile', - 'faustctl', - 'faustgen', - 'fft', - 'fftinv', - 'ficlose', - 'filebit', - 'filelen', - 'filenchnls', - 'filepeak', - 'filesr', - 'filevalid', - 'fillarray', - 'filter2', - 'fin', - 'fini', - 'fink', - 'fiopen', - 'flanger', - 'flashtxt', - 'flooper', - 'flooper2', - 'floor', - 'fluidAllOut', - 'fluidCCi', - 'fluidCCk', - 'fluidControl', - 'fluidEngine', - 'fluidLoad', - 'fluidNote', - 'fluidOut', - 'fluidProgramSelect', - 'fluidSetInterpMethod', - 'fmb3', - 'fmbell', - 'fmmetal', - 'fmpercfl', - 'fmrhode', - 'fmvoice', - 'fmwurlie', - 'fof', - 'fof2', - 'fofilter', - 'fog', - 'fold', - 'follow', - 'follow2', - 'foscil', - 'foscili', - 'fout', - 'fouti', - 'foutir', - 'foutk', - 'fprintks', - 'fprints', - 'frac', - 'fractalnoise', - 'freeverb', - 'ftchnls', - 'ftconv', - 'ftcps', - 'ftfree', - 'ftgen', - 'ftgenonce', - 'ftgentmp', - 'ftlen', - 'ftload', - 'ftloadk', - 'ftlptim', - 'ftmorf', - 'ftresize', - 'ftresizei', - 'ftsave', - 'ftsavek', - 'ftsr', - 'gain', - 'gainslider', - 'gauss', - 'gaussi', - 'gausstrig', - 'gbuzz', - 'genarray', - 'genarray_i', - 'gendy', - 'gendyc', - 'gendyx', - 'getcfg', - 'getcol', - 'getrow', - 'gogobel', - #'goto', - 'grain', - 'grain2', - 'grain3', - 'granule', - 'guiro', - 'harmon', - 'harmon2', - 'harmon3', - 'harmon4', - 'hdf5read', - 'hdf5write', - 'hilbert', - 'hrtfearly', - 'hrtfer', - 'hrtfmove', - 'hrtfmove2', - 'hrtfreverb', - 'hrtfstat', - 'hsboscil', - 'hvs1', - 'hvs2', - 'hvs3', - 'i', - 'iceps', - #'igoto', - 'ihold', - 'imagecreate', - 'imagefree', - 'imagegetpixel', - 'imageload', - 'imagesave', - 'imagesetpixel', - 'imagesize', - 'in', - 'in32', - 'inch', - 'inh', - 'init', - 'initc14', - 'initc21', - 'initc7', - 'inleta', - 'inletf', - 'inletk', - 'inletkid', - 'inletv', - 'ino', - 'inq', - 'inrg', - 'ins', - 'insglobal', - 'insremot', - #'instr', - 'int', - 'integ', - 'interp', - 'invalue', - 'inx', - 'inz', - 'jitter', - 'jitter2', - 'jspline', - 'k', - #'kgoto', - 'ktableseg', - 'lenarray', - 'lentab', - 'lfo', - 'limit', - 'line', - 'linen', - 'linenr', - 'lineto', - 'linrand', - 'linseg', - 'linsegb', - 'linsegr', - 'locsend', - 'locsig', - 'log', - 'log10', - 'log2', - 'logbtwo', - 'logcurve', - #'loop_ge', - #'loop_gt', - #'loop_le', - #'loop_lt', - 'loopseg', - 'loopsegp', - 'looptseg', - 'loopxseg', - 'lorenz', - 'loscil', - 'loscil3', - 'loscilx', - 'lowpass2', - 'lowres', - 'lowresx', - 'lpf18', - 'lpform', - 'lpfreson', - 'lphasor', - 'lpinterp', - 'lposcil', - 'lposcil3', - 'lposcila', - 'lposcilsa', - 'lposcilsa2', - 'lpread', - 'lpreson', - 'lpshold', - 'lpsholdp', - 'lpslot', - #'lua_exec', - 'lua_ikopcall', - #'lua_opdef', - 'mac', - 'maca', - 'madsr', - 'mags', - 'mandel', - 'mandol', - 'maparray', - 'maparray_i', - 'marimba', - 'massign', - 'max', - 'max_k', - 'maxabs', - 'maxabsaccum', - 'maxaccum', - 'maxalloc', - 'maxarray', - 'maxtab', - 'mclock', - 'mdelay', - 'median', - 'mediank', - 'metro', - 'midglobal', - 'midic14', - 'midic21', - 'midic7', - 'midichannelaftertouch', - 'midichn', - 'midicontrolchange', - 'midictrl', - 'mididefault', - 'midifilestatus', - 'midiin', - 'midinoteoff', - 'midinoteoncps', - 'midinoteonkey', - 'midinoteonoct', - 'midinoteonpch', - 'midion', - 'midion2', - 'midiout', - 'midipgm', - 'midipitchbend', - 'midipolyaftertouch', - 'midiprogramchange', - 'miditempo', - 'midremot', - 'min', - 'minabs', - 'minabsaccum', - 'minaccum', - 'minarray', - 'mincer', - 'mintab', - 'mirror', - 'mode', - 'modmatrix', - 'monitor', - 'moog', - 'moogladder', - 'moogvcf', - 'moogvcf2', - 'moscil', - 'mp3bitrate', - 'mp3in', - 'mp3len', - 'mp3nchnls', - 'mp3sr', - 'mpulse', - 'mrtmsg', - 'multitap', - 'mute', - 'mxadsr', - 'nestedap', - 'nlalp', - 'nlfilt', - 'nlfilt2', - 'noise', - 'noteoff', - 'noteon', - 'noteondur', - 'noteondur2', - 'notnum', - 'nreverb', - 'nrpn', - 'nsamp', - 'nstance', - 'nstrnum', - 'ntrpol', - 'octave', - 'octcps', - 'octmidi', - 'octmidib', - 'octmidinn', - 'octpch', - #'opcode', - 'oscbnk', - 'oscil', - 'oscil1', - 'oscil1i', - 'oscil3', - 'oscili', - 'oscilikt', - 'osciliktp', - 'oscilikts', - 'osciln', - 'oscils', - 'oscilx', - 'out', - 'out32', - 'outc', - 'outch', - 'outh', - 'outiat', - 'outic', - 'outic14', - 'outipat', - 'outipb', - 'outipc', - 'outkat', - 'outkc', - 'outkc14', - 'outkpat', - 'outkpb', - 'outkpc', - 'outleta', - 'outletf', - 'outletk', - 'outletkid', - 'outletv', - 'outo', - 'outq', - 'outq1', - 'outq2', - 'outq3', - 'outq4', - 'outrg', - 'outs', - 'outs1', - 'outs2', - 'outvalue', - 'outx', - 'outz', - 'p', - 'pan', - 'pan2', - 'pareq', - 'partials', - 'partikkel', - 'partikkelget', - 'partikkelset', - 'partikkelsync', - 'passign', - 'pcauchy', - 'pchbend', - 'pchmidi', - 'pchmidib', - 'pchmidinn', - 'pchoct', - 'pconvolve', - 'pcount', - 'pdclip', - 'pdhalf', - 'pdhalfy', - 'peak', - 'pgmassign', - 'pgmchn', - 'phaser1', - 'phaser2', - 'phasor', - 'phasorbnk', - 'phs', - 'pindex', - 'pinker', - 'pinkish', - 'pitch', - 'pitchac', - 'pitchamdf', - 'planet', - 'platerev', - 'plltrack', - 'pluck', - 'poisson', - 'pol2rect', - 'polyaft', - 'polynomial', - 'pop', - 'pop_f', - 'port', - 'portk', - 'poscil', - 'poscil3', - 'pow', - 'powershape', - 'powoftwo', - 'prealloc', - 'prepiano', - 'print', - 'print_type', - 'printf', - 'printf_i', - 'printk', - 'printk2', - 'printks', - 'printks2', - 'prints', - 'product', - 'pset', - 'ptable', - 'ptable3', - 'ptablei', - 'ptableiw', - 'ptablew', - 'ptrack', - 'push', - 'push_f', - 'puts', - 'pvadd', - 'pvbufread', - 'pvcross', - 'pvinterp', - 'pvoc', - 'pvread', - 'pvs2array', - 'pvs2tab', - 'pvsadsyn', - 'pvsanal', - 'pvsarp', - 'pvsbandp', - 'pvsbandr', - 'pvsbin', - 'pvsblur', - 'pvsbuffer', - 'pvsbufread', - 'pvsbufread2', - 'pvscale', - 'pvscent', - 'pvsceps', - 'pvscross', - 'pvsdemix', - 'pvsdiskin', - 'pvsdisp', - 'pvsenvftw', - 'pvsfilter', - 'pvsfread', - 'pvsfreeze', - 'pvsfromarray', - 'pvsftr', - 'pvsftw', - 'pvsfwrite', - 'pvsgain', - 'pvsgendy', - 'pvshift', - 'pvsifd', - 'pvsin', - 'pvsinfo', - 'pvsinit', - 'pvslock', - 'pvsmaska', - 'pvsmix', - 'pvsmooth', - 'pvsmorph', - 'pvsosc', - 'pvsout', - 'pvspitch', - 'pvstanal', - 'pvstencil', - 'pvsvoc', - 'pvswarp', - 'pvsynth', - 'pwd', - 'pyassign', - 'pyassigni', - 'pyassignt', - 'pycall', - 'pycall1', - 'pycall1i', - 'pycall1t', - 'pycall2', - 'pycall2i', - 'pycall2t', - 'pycall3', - 'pycall3i', - 'pycall3t', - 'pycall4', - 'pycall4i', - 'pycall4t', - 'pycall5', - 'pycall5i', - 'pycall5t', - 'pycall6', - 'pycall6i', - 'pycall6t', - 'pycall7', - 'pycall7i', - 'pycall7t', - 'pycall8', - 'pycall8i', - 'pycall8t', - 'pycalli', - 'pycalln', - 'pycallni', - 'pycallt', - 'pyeval', - 'pyevali', - 'pyevalt', - 'pyexec', - 'pyexeci', - 'pyexect', - 'pyinit', - 'pylassign', - 'pylassigni', - 'pylassignt', - 'pylcall', - 'pylcall1', - 'pylcall1i', - 'pylcall1t', - 'pylcall2', - 'pylcall2i', - 'pylcall2t', - 'pylcall3', - 'pylcall3i', - 'pylcall3t', - 'pylcall4', - 'pylcall4i', - 'pylcall4t', - 'pylcall5', - 'pylcall5i', - 'pylcall5t', - 'pylcall6', - 'pylcall6i', - 'pylcall6t', - 'pylcall7', - 'pylcall7i', - 'pylcall7t', - 'pylcall8', - 'pylcall8i', - 'pylcall8t', - 'pylcalli', - 'pylcalln', - 'pylcallni', - 'pylcallt', - 'pyleval', - 'pylevali', - 'pylevalt', - 'pylexec', - 'pylexeci', - 'pylexect', - #'pylrun', - #'pylruni', - #'pylrunt', - #'pyrun', - #'pyruni', - #'pyrunt', - 'qinf', - 'qnan', - 'r2c', - 'rand', - 'randh', - 'randi', - 'random', - 'randomh', - 'randomi', - 'rbjeq', - 'readclock', - 'readf', - 'readfi', - 'readk', - 'readk2', - 'readk3', - 'readk4', - 'readks', - 'readscore', - 'readscratch', - 'rect2pol', - 'reinit', - 'release', - 'remoteport', - 'remove', - 'repluck', - 'reson', - 'resonk', - 'resonr', - 'resonx', - 'resonxk', - 'resony', - 'resonz', - 'resyn', - #'return', - 'reverb', - 'reverb2', - 'reverbsc', - 'rewindscore', - 'rezzy', - 'rfft', - 'rifft', - #'rigoto', - 'rireturn', - 'rms', - 'rnd', - 'rnd31', - 'round', - 'rspline', - 'rtclock', - 's16b14', - 's32b14', - 'samphold', - 'sandpaper', - 'scale', - 'scalearray', - 'scalet', - 'scanhammer', - 'scans', - 'scantable', - 'scanu', - 'schedkwhen', - 'schedkwhennamed', - 'schedule', - 'schedwhen', - #'scoreline', - #'scoreline_i', - 'seed', - 'sekere', - 'semitone', - 'sense', - 'sensekey', - 'seqtime', - 'seqtime2', - 'serialBegin', - 'serialEnd', - 'serialFlush', - 'serialPrint', - 'serialRead', - 'serialWrite', - 'serialWrite_i', - 'setcol', - 'setctrl', - 'setksmps', - 'setrow', - 'setscorepos', - 'sfilist', - 'sfinstr', - 'sfinstr3', - 'sfinstr3m', - 'sfinstrm', - 'sfload', - 'sflooper', - 'sfpassign', - 'sfplay', - 'sfplay3', - 'sfplay3m', - 'sfplaym', - 'sfplist', - 'sfpreset', - 'shaker', - 'shiftin', - 'shiftout', - 'signalflowgraph', - 'signum', - 'sin', - 'sinh', - 'sininv', - 'sinsyn', - 'sleighbells', - 'slicearray', - 'slider16', - 'slider16f', - 'slider16table', - 'slider16tablef', - 'slider32', - 'slider32f', - 'slider32table', - 'slider32tablef', - 'slider64', - 'slider64f', - 'slider64table', - 'slider64tablef', - 'slider8', - 'slider8f', - 'slider8table', - 'slider8tablef', - 'sliderKawai', - 'sndload', - 'sndloop', - 'sndwarp', - 'sndwarpst', - 'sockrecv', - 'sockrecvs', - 'socksend', - 'socksends', - 'soundin', - 'soundout', - 'soundouts', - 'space', - 'spat3d', - 'spat3di', - 'spat3dt', - 'spdist', - 'specaddm', - 'specdiff', - 'specdisp', - 'specfilt', - 'spechist', - 'specptrk', - 'specscal', - 'specsum', - 'spectrum', - 'splitrig', - 'sprintf', - 'sprintfk', - 'spsend', - 'sqrt', - 'stack', - 'statevar', - 'stix', - 'strcat', - 'strcatk', - 'strchar', - 'strchark', - 'strcmp', - 'strcmpk', - 'strcpy', - 'strcpyk', - 'strecv', - 'streson', - 'strfromurl', - 'strget', - 'strindex', - 'strindexk', - 'strlen', - 'strlenk', - 'strlower', - 'strlowerk', - 'strrindex', - 'strrindexk', - 'strset', - 'strsub', - 'strsubk', - 'strtod', - 'strtodk', - 'strtol', - 'strtolk', - 'strupper', - 'strupperk', - 'stsend', - 'subinstr', - 'subinstrinit', - 'sum', - 'sumarray', - 'sumtab', - 'svfilter', - 'syncgrain', - 'syncloop', - 'syncphasor', - 'system', - 'system_i', - 'tab', - 'tab2pvs', - 'tab_i', - 'tabgen', - 'table', - 'table3', - 'table3kt', - 'tablecopy', - 'tablefilter', - 'tablefilteri', - 'tablegpw', - 'tablei', - 'tableicopy', - 'tableigpw', - 'tableikt', - 'tableimix', - 'tableiw', - 'tablekt', - 'tablemix', - 'tableng', - 'tablera', - 'tableseg', - 'tableshuffle', - 'tableshufflei', - 'tablew', - 'tablewa', - 'tablewkt', - 'tablexkt', - 'tablexseg', - 'tabmap', - 'tabmap_i', - 'tabmorph', - 'tabmorpha', - 'tabmorphak', - 'tabmorphi', - 'tabplay', - 'tabrec', - 'tabslice', - 'tabsum', - 'tabw', - 'tabw_i', - 'tambourine', - 'tan', - 'tanh', - 'taninv', - 'taninv2', - 'tb0', - 'tb0_init', - 'tb1', - 'tb10', - 'tb10_init', - 'tb11', - 'tb11_init', - 'tb12', - 'tb12_init', - 'tb13', - 'tb13_init', - 'tb14', - 'tb14_init', - 'tb15', - 'tb15_init', - 'tb1_init', - 'tb2', - 'tb2_init', - 'tb3', - 'tb3_init', - 'tb4', - 'tb4_init', - 'tb5', - 'tb5_init', - 'tb6', - 'tb6_init', - 'tb7', - 'tb7_init', - 'tb8', - 'tb8_init', - 'tb9', - 'tb9_init', - 'tbvcf', - 'tempest', - 'tempo', - 'temposcal', - 'tempoval', - #'tigoto', - 'timedseq', - 'timeinstk', - 'timeinsts', - 'timek', - 'times', - #'timout', - 'tival', - 'tlineto', - 'tone', - 'tonek', - 'tonex', - 'tradsyn', - 'trandom', - 'transeg', - 'transegb', - 'transegr', - 'trcross', - 'trfilter', - 'trhighest', - 'trigger', - 'trigseq', - 'trirand', - 'trlowest', - 'trmix', - 'trscale', - 'trshift', - 'trsplit', - 'turnoff', - 'turnoff2', - 'turnon', - 'unirand', - 'unwrap', - 'upsamp', - 'urd', - 'vactrol', - 'vadd', - 'vadd_i', - 'vaddv', - 'vaddv_i', - 'vaget', - 'valpass', - 'vaset', - 'vbap', - 'vbap16', - 'vbap4', - 'vbap4move', - 'vbap8', - 'vbap8move', - 'vbapg', - 'vbapgmove', - 'vbaplsinit', - 'vbapmove', - 'vbapz', - 'vbapzmove', - 'vcella', - 'vco', - 'vco2', - 'vco2ft', - 'vco2ift', - 'vco2init', - 'vcomb', - 'vcopy', - 'vcopy_i', - 'vdel_k', - 'vdelay', - 'vdelay3', - 'vdelayk', - 'vdelayx', - 'vdelayxq', - 'vdelayxs', - 'vdelayxw', - 'vdelayxwq', - 'vdelayxws', - 'vdivv', - 'vdivv_i', - 'vecdelay', - 'veloc', - 'vexp', - 'vexp_i', - 'vexpseg', - 'vexpv', - 'vexpv_i', - 'vibes', - 'vibr', - 'vibrato', - 'vincr', - 'vlimit', - 'vlinseg', - 'vlowres', - 'vmap', - 'vmirror', - 'vmult', - 'vmult_i', - 'vmultv', - 'vmultv_i', - 'voice', - 'vosim', - 'vphaseseg', - 'vport', - 'vpow', - 'vpow_i', - 'vpowv', - 'vpowv_i', - 'vpvoc', - 'vrandh', - 'vrandi', - 'vsubv', - 'vsubv_i', - 'vtaba', - 'vtabi', - 'vtabk', - 'vtable1k', - 'vtablea', - 'vtablei', - 'vtablek', - 'vtablewa', - 'vtablewi', - 'vtablewk', - 'vtabwa', - 'vtabwi', - 'vtabwk', - 'vwrap', - 'waveset', - 'weibull', - 'wgbow', - 'wgbowedbar', - 'wgbrass', - 'wgclar', - 'wgflute', - 'wgpluck', - 'wgpluck2', - 'wguide1', - 'wguide2', - 'wiiconnect', - 'wiidata', - 'wiirange', - 'wiisend', - 'window', - 'wrap', - 'writescratch', - 'wterrain', - 'xadsr', - 'xin', - 'xout', - 'xscanmap', - 'xscans', - 'xscansmap', - 'xscanu', - 'xtratim', - 'xyin', - 'zacl', - 'zakinit', - 'zamod', - 'zar', - 'zarg', - 'zaw', - 'zawm', - 'zfilter2', - 'zir', - 'ziw', - 'ziwm', - 'zkcl', - 'zkmod', - 'zkr', - 'zkw', - 'zkwm' -)) +# Opcodes in Csound 6.12.0 at commit 6ca322bd31f1ca907c008616b40a5f237ff449db using +# python -c " +# import re, subprocess +# output = subprocess.Popen(['csound', '--list-opcodes0'], stderr=subprocess.PIPE).communicate()[1] +# opcodes = output[re.search(r'^$', output, re.M).end():re.search(r'^\d+ opcodes$', output, re.M).start()].split() +# output = subprocess.Popen(['csound', '--list-opcodes2'], stderr=subprocess.PIPE).communicate()[1] +# all_opcodes = output[re.search(r'^$', output, re.M).end():re.search(r'^\d+ opcodes$', output, re.M).start()].split() +# deprecated_opcodes = [opcode for opcode in all_opcodes if opcode not in opcodes] +# print '''OPCODES = set(\''' +# {} +# \'''.split()) +# +# DEPRECATED_OPCODES = set(\''' +# {} +# \'''.split()) +# '''.format('\n'.join(opcodes), '\n'.join(deprecated_opcodes)) +# " +# except for +# cggoto csound.com/docs/manual/cggoto.html +# cigoto csound.com/docs/manual/cigoto.html +# cingoto (undocumented) +# ckgoto csound.com/docs/manual/ckgoto.html +# cngoto csound.com/docs/manual/cngoto.html +# cnkgoto (undocumented) +# endin csound.com/docs/manual/endin.html +# endop csound.com/docs/manual/endop.html +# goto csound.com/docs/manual/goto.html +# igoto csound.com/docs/manual/igoto.html +# instr csound.com/docs/manual/instr.html +# kgoto csound.com/docs/manual/kgoto.html +# loop_ge csound.com/docs/manual/loop_ge.html +# loop_gt csound.com/docs/manual/loop_gt.html +# loop_le csound.com/docs/manual/loop_le.html +# loop_lt csound.com/docs/manual/loop_lt.html +# opcode csound.com/docs/manual/opcode.html +# reinit csound.com/docs/manual/reinit.html +# return csound.com/docs/manual/return.html +# rireturn csound.com/docs/manual/rireturn.html +# rigoto csound.com/docs/manual/rigoto.html +# tigoto csound.com/docs/manual/tigoto.html +# timout csound.com/docs/manual/timout.html +# which are treated as keywords in csound.py. + +OPCODES = set(''' +ATSadd +ATSaddnz +ATSbufread +ATScross +ATSinfo +ATSinterpread +ATSpartialtap +ATSread +ATSreadnz +ATSsinnoi +FLbox +FLbutBank +FLbutton +FLcloseButton +FLcolor +FLcolor2 +FLcount +FLexecButton +FLgetsnap +FLgroup +FLgroupEnd +FLgroup_end +FLhide +FLhvsBox +FLhvsBoxSetValue +FLjoy +FLkeyIn +FLknob +FLlabel +FLloadsnap +FLmouse +FLpack +FLpackEnd +FLpack_end +FLpanel +FLpanelEnd +FLpanel_end +FLprintk +FLprintk2 +FLroller +FLrun +FLsavesnap +FLscroll +FLscrollEnd +FLscroll_end +FLsetAlign +FLsetBox +FLsetColor +FLsetColor2 +FLsetFont +FLsetPosition +FLsetSize +FLsetSnapGroup +FLsetText +FLsetTextColor +FLsetTextSize +FLsetTextType +FLsetVal +FLsetVal_i +FLsetVali +FLsetsnap +FLshow +FLslidBnk +FLslidBnk2 +FLslidBnk2Set +FLslidBnk2Setk +FLslidBnkGetHandle +FLslidBnkSet +FLslidBnkSetk +FLslider +FLtabs +FLtabsEnd +FLtabs_end +FLtext +FLupdate +FLvalue +FLvkeybd +FLvslidBnk +FLvslidBnk2 +FLxyin +JackoAudioIn +JackoAudioInConnect +JackoAudioOut +JackoAudioOutConnect +JackoFreewheel +JackoInfo +JackoInit +JackoMidiInConnect +JackoMidiOut +JackoMidiOutConnect +JackoNoteOut +JackoOn +JackoTransport +K35_hpf +K35_lpf +MixerClear +MixerGetLevel +MixerReceive +MixerSend +MixerSetLevel +MixerSetLevel_i +OSCbundle +OSCcount +OSCinit +OSCinitM +OSClisten +OSCraw +OSCsend +OSCsend_lo +S +STKBandedWG +STKBeeThree +STKBlowBotl +STKBlowHole +STKBowed +STKBrass +STKClarinet +STKDrummer +STKFlute +STKFMVoices +STKHevyMetl +STKMandolin +STKModalBar +STKMoog +STKPercFlut +STKPlucked +STKResonate +STKRhodey +STKSaxofony +STKShakers +STKSimple +STKSitar +STKStifKarp +STKTubeBell +STKVoicForm +STKWhistle +STKWurley +a +abs +active +adsr +adsyn +adsynt +adsynt2 +aftouch +alpass +alwayson +ampdb +ampdbfs +ampmidi +ampmidid +areson +aresonk +atone +atonek +atonex +babo +balance +balance2 +bamboo +barmodel +bbcutm +bbcuts +beadsynt +beosc +betarand +bexprnd +bformdec1 +bformenc1 +binit +biquad +biquada +birnd +bpf +bpfcos +bqrez +butbp +butbr +buthp +butlp +butterbp +butterbr +butterhp +butterlp +button +buzz +c2r +cabasa +cauchy +cauchyi +cbrt +ceil +cell +cent +centroid +ceps +cepsinv +chanctrl +changed +changed2 +chani +chano +chebyshevpoly +checkbox +chn_S +chn_a +chn_k +chnclear +chnexport +chnget +chngetks +chnmix +chnparams +chnset +chnsetks +chuap +clear +clfilt +clip +clockoff +clockon +cmp +cmplxprod +comb +combinv +compilecsd +compileorc +compilestr +compress +compress2 +connect +control +convle +convolve +copya2ftab +copyf2array +cos +cosh +cosinv +cosseg +cossegb +cossegr +cps2pch +cpsmidi +cpsmidib +cpsmidinn +cpsoct +cpspch +cpstmid +cpstun +cpstuni +cpsxpch +cpumeter +cpuprc +cross2 +crossfm +crossfmi +crossfmpm +crossfmpmi +crosspm +crosspmi +crunch +ctlchn +ctrl14 +ctrl21 +ctrl7 +ctrlinit +cuserrnd +dam +date +dates +db +dbamp +dbfsamp +dcblock +dcblock2 +dconv +dct +dctinv +deinterleave +delay +delay1 +delayk +delayr +delayw +deltap +deltap3 +deltapi +deltapn +deltapx +deltapxw +denorm +diff +diode_ladder +directory +diskgrain +diskin +diskin2 +dispfft +display +distort +distort1 +divz +doppler +dot +downsamp +dripwater +dssiactivate +dssiaudio +dssictls +dssiinit +dssilist +dumpk +dumpk2 +dumpk3 +dumpk4 +duserrnd +dust +dust2 +envlpx +envlpxr +ephasor +eqfil +evalstr +event +event_i +exciter +exitnow +exp +expcurve +expon +exprand +exprandi +expseg +expsega +expsegb +expsegba +expsegr +fareylen +fareyleni +faustaudio +faustcompile +faustctl +faustdsp +faustgen +faustplay +fft +fftinv +ficlose +filebit +filelen +filenchnls +filepeak +filescal +filesr +filevalid +fillarray +filter2 +fin +fini +fink +fiopen +flanger +flashtxt +flooper +flooper2 +floor +fmanal +fmax +fmb3 +fmbell +fmin +fmmetal +fmod +fmpercfl +fmrhode +fmvoice +fmwurlie +fof +fof2 +fofilter +fog +fold +follow +follow2 +foscil +foscili +fout +fouti +foutir +foutk +fprintks +fprints +frac +fractalnoise +framebuffer +freeverb +ftaudio +ftchnls +ftconv +ftcps +ftfree +ftgen +ftgenonce +ftgentmp +ftlen +ftload +ftloadk +ftlptim +ftmorf +ftom +ftprint +ftresize +ftresizei +ftsamplebank +ftsave +ftsavek +ftslice +ftsr +gain +gainslider +gauss +gaussi +gausstrig +gbuzz +genarray +genarray_i +gendy +gendyc +gendyx +getcfg +getcol +getftargs +getrow +getrowlin +getseed +gogobel +grain +grain2 +grain3 +granule +guiro +harmon +harmon2 +harmon3 +harmon4 +hdf5read +hdf5write +hilbert +hilbert2 +hrtfearly +hrtfmove +hrtfmove2 +hrtfreverb +hrtfstat +hsboscil +hvs1 +hvs2 +hvs3 +hypot +i +ihold +imagecreate +imagefree +imagegetpixel +imageload +imagesave +imagesetpixel +imagesize +in +in32 +inch +inh +init +initc14 +initc21 +initc7 +inleta +inletf +inletk +inletkid +inletv +ino +inq +inrg +ins +insglobal +insremot +int +integ +interleave +interp +invalue +inx +inz +jacktransport +jitter +jitter2 +joystick +jspline +k +la_i_add_mc +la_i_add_mr +la_i_add_vc +la_i_add_vr +la_i_assign_mc +la_i_assign_mr +la_i_assign_t +la_i_assign_vc +la_i_assign_vr +la_i_conjugate_mc +la_i_conjugate_mr +la_i_conjugate_vc +la_i_conjugate_vr +la_i_distance_vc +la_i_distance_vr +la_i_divide_mc +la_i_divide_mr +la_i_divide_vc +la_i_divide_vr +la_i_dot_mc +la_i_dot_mc_vc +la_i_dot_mr +la_i_dot_mr_vr +la_i_dot_vc +la_i_dot_vr +la_i_get_mc +la_i_get_mr +la_i_get_vc +la_i_get_vr +la_i_invert_mc +la_i_invert_mr +la_i_lower_solve_mc +la_i_lower_solve_mr +la_i_lu_det_mc +la_i_lu_det_mr +la_i_lu_factor_mc +la_i_lu_factor_mr +la_i_lu_solve_mc +la_i_lu_solve_mr +la_i_mc_create +la_i_mc_set +la_i_mr_create +la_i_mr_set +la_i_multiply_mc +la_i_multiply_mr +la_i_multiply_vc +la_i_multiply_vr +la_i_norm_euclid_mc +la_i_norm_euclid_mr +la_i_norm_euclid_vc +la_i_norm_euclid_vr +la_i_norm_inf_mc +la_i_norm_inf_mr +la_i_norm_inf_vc +la_i_norm_inf_vr +la_i_norm_max_mc +la_i_norm_max_mr +la_i_norm1_mc +la_i_norm1_mr +la_i_norm1_vc +la_i_norm1_vr +la_i_print_mc +la_i_print_mr +la_i_print_vc +la_i_print_vr +la_i_qr_eigen_mc +la_i_qr_eigen_mr +la_i_qr_factor_mc +la_i_qr_factor_mr +la_i_qr_sym_eigen_mc +la_i_qr_sym_eigen_mr +la_i_random_mc +la_i_random_mr +la_i_random_vc +la_i_random_vr +la_i_size_mc +la_i_size_mr +la_i_size_vc +la_i_size_vr +la_i_subtract_mc +la_i_subtract_mr +la_i_subtract_vc +la_i_subtract_vr +la_i_t_assign +la_i_trace_mc +la_i_trace_mr +la_i_transpose_mc +la_i_transpose_mr +la_i_upper_solve_mc +la_i_upper_solve_mr +la_i_vc_create +la_i_vc_set +la_i_vr_create +la_i_vr_set +la_k_a_assign +la_k_add_mc +la_k_add_mr +la_k_add_vc +la_k_add_vr +la_k_assign_a +la_k_assign_f +la_k_assign_mc +la_k_assign_mr +la_k_assign_t +la_k_assign_vc +la_k_assign_vr +la_k_conjugate_mc +la_k_conjugate_mr +la_k_conjugate_vc +la_k_conjugate_vr +la_k_current_f +la_k_current_vr +la_k_distance_vc +la_k_distance_vr +la_k_divide_mc +la_k_divide_mr +la_k_divide_vc +la_k_divide_vr +la_k_dot_mc +la_k_dot_mc_vc +la_k_dot_mr +la_k_dot_mr_vr +la_k_dot_vc +la_k_dot_vr +la_k_f_assign +la_k_get_mc +la_k_get_mr +la_k_get_vc +la_k_get_vr +la_k_invert_mc +la_k_invert_mr +la_k_lower_solve_mc +la_k_lower_solve_mr +la_k_lu_det_mc +la_k_lu_det_mr +la_k_lu_factor_mc +la_k_lu_factor_mr +la_k_lu_solve_mc +la_k_lu_solve_mr +la_k_mc_set +la_k_mr_set +la_k_multiply_mc +la_k_multiply_mr +la_k_multiply_vc +la_k_multiply_vr +la_k_norm_euclid_mc +la_k_norm_euclid_mr +la_k_norm_euclid_vc +la_k_norm_euclid_vr +la_k_norm_inf_mc +la_k_norm_inf_mr +la_k_norm_inf_vc +la_k_norm_inf_vr +la_k_norm_max_mc +la_k_norm_max_mr +la_k_norm1_mc +la_k_norm1_mr +la_k_norm1_vc +la_k_norm1_vr +la_k_qr_eigen_mc +la_k_qr_eigen_mr +la_k_qr_factor_mc +la_k_qr_factor_mr +la_k_qr_sym_eigen_mc +la_k_qr_sym_eigen_mr +la_k_random_mc +la_k_random_mr +la_k_random_vc +la_k_random_vr +la_k_subtract_mc +la_k_subtract_mr +la_k_subtract_vc +la_k_subtract_vr +la_k_t_assign +la_k_trace_mc +la_k_trace_mr +la_k_upper_solve_mc +la_k_upper_solve_mr +la_k_vc_set +la_k_vr_set +lenarray +lfo +limit +limit1 +lincos +line +linen +linenr +lineto +link_beat_force +link_beat_get +link_beat_request +link_create +link_enable +link_is_enabled +link_metro +link_peers +link_tempo_get +link_tempo_set +linlin +linrand +linseg +linsegb +linsegr +liveconv +locsend +locsig +log +log10 +log2 +logbtwo +logcurve +loopseg +loopsegp +looptseg +loopxseg +lorenz +loscil +loscil3 +loscil3phs +loscilphs +loscilx +lowpass2 +lowres +lowresx +lpf18 +lpform +lpfreson +lphasor +lpinterp +lposcil +lposcil3 +lposcila +lposcilsa +lposcilsa2 +lpread +lpreson +lpshold +lpsholdp +lpslot +lua_exec +lua_iaopcall +lua_iaopcall_off +lua_ikopcall +lua_ikopcall_off +lua_iopcall +lua_iopcall_off +lua_opdef +mac +maca +madsr +mags +mandel +mandol +maparray +maparray_i +marimba +massign +max +max_k +maxabs +maxabsaccum +maxaccum +maxalloc +maxarray +mclock +mdelay +median +mediank +metro +mfb +midglobal +midiarp +midic14 +midic21 +midic7 +midichannelaftertouch +midichn +midicontrolchange +midictrl +mididefault +midifilestatus +midiin +midinoteoff +midinoteoncps +midinoteonkey +midinoteonoct +midinoteonpch +midion +midion2 +midiout +midiout_i +midipgm +midipitchbend +midipolyaftertouch +midiprogramchange +miditempo +midremot +min +minabs +minabsaccum +minaccum +minarray +mincer +mirror +mode +modmatrix +monitor +moog +moogladder +moogladder2 +moogvcf +moogvcf2 +moscil +mp3bitrate +mp3in +mp3len +mp3nchnls +mp3scal +mp3sr +mpulse +mrtmsg +mtof +mton +multitap +mute +mvchpf +mvclpf1 +mvclpf2 +mvclpf3 +mvclpf4 +mxadsr +nchnls_hw +nestedap +nlalp +nlfilt +nlfilt2 +noise +noteoff +noteon +noteondur +noteondur2 +notnum +nreverb +nrpn +nsamp +nstance +nstrnum +ntom +ntrpol +nxtpow2 +octave +octcps +octmidi +octmidib +octmidinn +octpch +olabuffer +oscbnk +oscil +oscil1 +oscil1i +oscil3 +oscili +oscilikt +osciliktp +oscilikts +osciln +oscils +oscilx +out +out32 +outc +outch +outh +outiat +outic +outic14 +outipat +outipb +outipc +outkat +outkc +outkc14 +outkpat +outkpb +outkpc +outleta +outletf +outletk +outletkid +outletv +outo +outq +outq1 +outq2 +outq3 +outq4 +outrg +outs +outs1 +outs2 +outvalue +outx +outz +p +p5gconnect +p5gdata +pan +pan2 +pareq +part2txt +partials +partikkel +partikkelget +partikkelset +partikkelsync +passign +paulstretch +pcauchy +pchbend +pchmidi +pchmidib +pchmidinn +pchoct +pchtom +pconvolve +pcount +pdclip +pdhalf +pdhalfy +peak +pgmassign +pgmchn +phaser1 +phaser2 +phasor +phasorbnk +phs +pindex +pinker +pinkish +pitch +pitchac +pitchamdf +planet +platerev +plltrack +pluck +poisson +pol2rect +polyaft +polynomial +port +portk +poscil +poscil3 +pow +powershape +powoftwo +pows +prealloc +prepiano +print +print_type +printarray +printf +printf_i +printk +printk2 +printks +printks2 +prints +product +pset +ptable +ptable3 +ptablei +ptableiw +ptablew +ptrack +puts +pvadd +pvbufread +pvcross +pvinterp +pvoc +pvread +pvs2array +pvs2tab +pvsadsyn +pvsanal +pvsarp +pvsbandp +pvsbandr +pvsbin +pvsblur +pvsbuffer +pvsbufread +pvsbufread2 +pvscale +pvscent +pvsceps +pvscross +pvsdemix +pvsdiskin +pvsdisp +pvsenvftw +pvsfilter +pvsfread +pvsfreeze +pvsfromarray +pvsftr +pvsftw +pvsfwrite +pvsgain +pvshift +pvsifd +pvsin +pvsinfo +pvsinit +pvslock +pvsmaska +pvsmix +pvsmooth +pvsmorph +pvsosc +pvsout +pvspitch +pvstanal +pvstencil +pvstrace +pvsvoc +pvswarp +pvsynth +pwd +pyassign +pyassigni +pyassignt +pycall +pycall1 +pycall1i +pycall1t +pycall2 +pycall2i +pycall2t +pycall3 +pycall3i +pycall3t +pycall4 +pycall4i +pycall4t +pycall5 +pycall5i +pycall5t +pycall6 +pycall6i +pycall6t +pycall7 +pycall7i +pycall7t +pycall8 +pycall8i +pycall8t +pycalli +pycalln +pycallni +pycallt +pyeval +pyevali +pyevalt +pyexec +pyexeci +pyexect +pyinit +pylassign +pylassigni +pylassignt +pylcall +pylcall1 +pylcall1i +pylcall1t +pylcall2 +pylcall2i +pylcall2t +pylcall3 +pylcall3i +pylcall3t +pylcall4 +pylcall4i +pylcall4t +pylcall5 +pylcall5i +pylcall5t +pylcall6 +pylcall6i +pylcall6t +pylcall7 +pylcall7i +pylcall7t +pylcall8 +pylcall8i +pylcall8t +pylcalli +pylcalln +pylcallni +pylcallt +pyleval +pylevali +pylevalt +pylexec +pylexeci +pylexect +pylrun +pylruni +pylrunt +pyrun +pyruni +pyrunt +qinf +qnan +r2c +rand +randh +randi +random +randomh +randomi +rbjeq +readclock +readf +readfi +readk +readk2 +readk3 +readk4 +readks +readscore +readscratch +rect2pol +release +remoteport +remove +repluck +reshapearray +reson +resonk +resonr +resonx +resonxk +resony +resonz +resyn +reverb +reverb2 +reverbsc +rewindscore +rezzy +rfft +rifft +rms +rnd +rnd31 +round +rspline +rtclock +s16b14 +s32b14 +samphold +sandpaper +sc_lag +sc_lagud +sc_phasor +sc_trig +scale +scalearray +scanhammer +scans +scantable +scanu +schedkwhen +schedkwhennamed +schedule +schedwhen +scoreline +scoreline_i +seed +sekere +select +semitone +sense +sensekey +seqtime +seqtime2 +serialBegin +serialEnd +serialFlush +serialPrint +serialRead +serialWrite +serialWrite_i +setcol +setctrl +setksmps +setrow +setscorepos +sfilist +sfinstr +sfinstr3 +sfinstr3m +sfinstrm +sfload +sflooper +sfpassign +sfplay +sfplay3 +sfplay3m +sfplaym +sfplist +sfpreset +shaker +shiftin +shiftout +signum +sin +sinh +sininv +sinsyn +sleighbells +slicearray +slicearray_i +slider16 +slider16f +slider16table +slider16tablef +slider32 +slider32f +slider32table +slider32tablef +slider64 +slider64f +slider64table +slider64tablef +slider8 +slider8f +slider8table +slider8tablef +sliderKawai +sndloop +sndwarp +sndwarpst +sockrecv +sockrecvs +socksend +socksends +sorta +sortd +soundin +space +spat3d +spat3di +spat3dt +spdist +splitrig +sprintf +sprintfk +spsend +sqrt +squinewave +statevar +stix +strcat +strcatk +strchar +strchark +strcmp +strcmpk +strcpy +strcpyk +strecv +streson +strfromurl +strget +strindex +strindexk +strlen +strlenk +strlower +strlowerk +strrindex +strrindexk +strset +strsub +strsubk +strtod +strtodk +strtol +strtolk +strupper +strupperk +stsend +subinstr +subinstrinit +sum +sumarray +svfilter +syncgrain +syncloop +syncphasor +system +system_i +tab +tab2array +tab2pvs +tab_i +tabifd +table +table3 +table3kt +tablecopy +tablefilter +tablefilteri +tablegpw +tablei +tableicopy +tableigpw +tableikt +tableimix +tableiw +tablekt +tablemix +tableng +tablera +tableseg +tableshuffle +tableshufflei +tablew +tablewa +tablewkt +tablexkt +tablexseg +tabmorph +tabmorpha +tabmorphak +tabmorphi +tabplay +tabrec +tabrowlin +tabsum +tabw +tabw_i +tambourine +tan +tanh +taninv +taninv2 +tbvcf +tempest +tempo +temposcal +tempoval +timedseq +timeinstk +timeinsts +timek +times +tival +tlineto +tone +tonek +tonex +tradsyn +trandom +transeg +transegb +transegr +trcross +trfilter +trhighest +trigger +trigseq +trim +trim_i +trirand +trlowest +trmix +trscale +trshift +trsplit +turnoff +turnoff2 +turnon +tvconv +unirand +unwrap +upsamp +urandom +urd +vactrol +vadd +vadd_i +vaddv +vaddv_i +vaget +valpass +vaset +vbap +vbapg +vbapgmove +vbaplsinit +vbapmove +vbapz +vbapzmove +vcella +vco +vco2 +vco2ft +vco2ift +vco2init +vcomb +vcopy +vcopy_i +vdel_k +vdelay +vdelay3 +vdelayk +vdelayx +vdelayxq +vdelayxs +vdelayxw +vdelayxwq +vdelayxws +vdivv +vdivv_i +vecdelay +veloc +vexp +vexp_i +vexpseg +vexpv +vexpv_i +vibes +vibr +vibrato +vincr +vlimit +vlinseg +vlowres +vmap +vmirror +vmult +vmult_i +vmultv +vmultv_i +voice +vosim +vphaseseg +vport +vpow +vpow_i +vpowv +vpowv_i +vpvoc +vrandh +vrandi +vsubv +vsubv_i +vtaba +vtabi +vtabk +vtable1k +vtablea +vtablei +vtablek +vtablewa +vtablewi +vtablewk +vtabwa +vtabwi +vtabwk +vwrap +waveset +websocket +weibull +wgbow +wgbowedbar +wgbrass +wgclar +wgflute +wgpluck +wgpluck2 +wguide1 +wguide2 +wiiconnect +wiidata +wiirange +wiisend +window +wrap +writescratch +wterrain +xadsr +xin +xout +xscanmap +xscans +xscansmap +xscanu +xtratim +xyscale +zacl +zakinit +zamod +zar +zarg +zaw +zawm +zdf_1pole +zdf_1pole_mode +zdf_2pole +zdf_2pole_mode +zdf_ladder +zfilter2 +zir +ziw +ziwm +zkcl +zkmod +zkr +zkw +zkwm +'''.split()) + +DEPRECATED_OPCODES = set(''' +array +bformdec +bformenc +copy2ftab +copy2ttab +hrtfer +ktableseg +lentab +maxtab +mintab +pop +pop_f +push +push_f +scalet +sndload +soundout +soundouts +specaddm +specdiff +specdisp +specfilt +spechist +specptrk +specscal +specsum +spectrum +stack +sumtab +tabgen +tabmap +tabmap_i +tabslice +tb0 +tb0_init +tb1 +tb10 +tb10_init +tb11 +tb11_init +tb12 +tb12_init +tb13 +tb13_init +tb14 +tb14_init +tb15 +tb15_init +tb1_init +tb2 +tb2_init +tb3 +tb3_init +tb4 +tb4_init +tb5 +tb5_init +tb6 +tb6_init +tb7 +tb7_init +tb8 +tb8_init +tb9 +tb9_init +vbap16 +vbap4 +vbap4move +vbap8 +vbap8move +xyin +'''.split()) diff --git a/pygments/lexers/_lua_builtins.py b/pygments/lexers/_lua_builtins.py index c60bf5a2..0561725d 100644 --- a/pygments/lexers/_lua_builtins.py +++ b/pygments/lexers/_lua_builtins.py @@ -288,7 +288,7 @@ if __name__ == '__main__': # pragma: no cover print('>> %s' % full_function_name) m = get_function_module(full_function_name) modules.setdefault(m, []).append(full_function_name) - modules = {k: tuple(v) for k, v in modules.iteritems()} + modules = dict((k, tuple(v)) for k, v in modules.iteritems()) regenerate(__file__, modules) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index b48ee1d1..9ccc7615 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -153,6 +153,7 @@ LEXERS = { 'FancyLexer': ('pygments.lexers.ruby', 'Fancy', ('fancy', 'fy'), ('*.fy', '*.fancypack'), ('text/x-fancysrc',)), 'FantomLexer': ('pygments.lexers.fantom', 'Fantom', ('fan',), ('*.fan',), ('application/x-fantom',)), 'FelixLexer': ('pygments.lexers.felix', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)), + 'FennelLexer': ('pygments.lexers.lisp', 'Fennel', ('fennel', 'fnl'), ('*.fnl',), ()), 'FishShellLexer': ('pygments.lexers.shell', 'Fish', ('fish', 'fishshell'), ('*.fish', '*.load'), ('application/x-fish',)), 'FlatlineLexer': ('pygments.lexers.dsls', 'Flatline', ('flatline',), (), ('text/x-flatline',)), 'ForthLexer': ('pygments.lexers.forth', 'Forth', ('forth',), ('*.frt', '*.fs'), ('application/x-forth',)), @@ -174,6 +175,7 @@ LEXERS = { 'GosuTemplateLexer': ('pygments.lexers.jvm', 'Gosu Template', ('gst',), ('*.gst',), ('text/x-gosu-template',)), 'GroffLexer': ('pygments.lexers.markup', 'Groff', ('groff', 'nroff', 'man'), ('*.[1234567]', '*.man'), ('application/x-troff', 'text/troff')), 'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy', '*.gradle'), ('text/x-groovy',)), + 'HLSLShaderLexer': ('pygments.lexers.graphics', 'HLSL', ('hlsl',), ('*.hlsl', '*.hlsli'), ('text/x-hlsl',)), 'HamlLexer': ('pygments.lexers.html', 'Haml', ('haml',), ('*.haml',), ('text/x-haml',)), 'HandlebarsHtmlLexer': ('pygments.lexers.templates', 'HTML+Handlebars', ('html+handlebars',), ('*.handlebars', '*.hbs'), ('text/html+handlebars', 'text/x-handlebars-template')), 'HandlebarsLexer': ('pygments.lexers.templates', 'Handlebars', ('handlebars',), (), ()), @@ -367,7 +369,7 @@ LEXERS = { 'RtsLexer': ('pygments.lexers.trafficscript', 'TrafficScript', ('rts', 'trafficscript'), ('*.rts',), ()), 'RubyConsoleLexer': ('pygments.lexers.ruby', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)), 'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')), - 'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs', '*.rs.in'), ('text/rust',)), + 'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust', 'rs'), ('*.rs', '*.rs.in'), ('text/rust',)), 'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')), 'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')), 'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')), diff --git a/pygments/lexers/_php_builtins.py b/pygments/lexers/_php_builtins.py index fec3286a..bd4b7d99 100644 --- a/pygments/lexers/_php_builtins.py +++ b/pygments/lexers/_php_builtins.py @@ -4688,7 +4688,7 @@ if __name__ == '__main__': # pragma: no cover PHP_MANUAL_URL = 'http://us3.php.net/distributions/manual/php_manual_en.tar.gz' PHP_MANUAL_DIR = './php-chunked-xhtml/' PHP_REFERENCE_GLOB = 'ref.*' - PHP_FUNCTION_RE = '<a href="function\..*?\.html">(.*?)</a>' + PHP_FUNCTION_RE = r'<a href="function\..*?\.html">(.*?)</a>' PHP_MODULE_RE = '<title>(.*?) Functions</title>' def get_php_functions(): diff --git a/pygments/lexers/actionscript.py b/pygments/lexers/actionscript.py index 84607e68..fc3b90cd 100644 --- a/pygments/lexers/actionscript.py +++ b/pygments/lexers/actionscript.py @@ -125,7 +125,7 @@ class ActionScript3Lexer(RegexLexer): 'text/actionscript3'] identifier = r'[$a-zA-Z_]\w*' - typeidentifier = identifier + '(?:\.<\w+>)?' + typeidentifier = identifier + r'(?:\.<\w+>)?' flags = re.DOTALL | re.MULTILINE tokens = { @@ -232,7 +232,7 @@ class MxmlLexer(RegexLexer): (r'/?\s*>', Name.Tag, '#pop'), ], 'attr': [ - ('\s+', Text), + (r'\s+', Text), ('".*?"', String, '#pop'), ("'.*?'", String, '#pop'), (r'[^\s>]+', String, '#pop'), diff --git a/pygments/lexers/ampl.py b/pygments/lexers/ampl.py index d439cb19..638d025d 100644 --- a/pygments/lexers/ampl.py +++ b/pygments/lexers/ampl.py @@ -3,7 +3,7 @@ pygments.lexers.ampl ~~~~~~~~~~~~~~~~~~~~ - Lexers for the ampl language. <http://ampl.com/> + Lexers for the AMPL language. :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. @@ -18,7 +18,7 @@ __all__ = ['AmplLexer'] class AmplLexer(RegexLexer): """ - For AMPL source code. + For `AMPL <http://ampl.com/>`_ source code. .. versionadded:: 2.2 """ diff --git a/pygments/lexers/apl.py b/pygments/lexers/apl.py index b3414cc0..4bb88ae3 100644 --- a/pygments/lexers/apl.py +++ b/pygments/lexers/apl.py @@ -71,14 +71,14 @@ class APLLexer(RegexLexer): # # Numbers # ======= - (u'¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞)' - u'([Jj]¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞))?', + (u'¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞)' + u'([Jj]¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞))?', Number), # # Operators # ========== - (u'[\.\\\/⌿⍀¨⍣⍨⍠⍤∘]', Name.Attribute), # closest token type - (u'[+\-×÷⌈⌊∣|⍳?*⍟○!⌹<≤=>≥≠≡≢∊⍷∪∩~∨∧⍱⍲⍴,⍪⌽⊖⍉↑↓⊂⊃⌷⍋⍒⊤⊥⍕⍎⊣⊢⍁⍂≈⌸⍯↗]', + (u'[\\.\\\\\\/⌿⍀¨⍣⍨⍠⍤∘]', Name.Attribute), # closest token type + (u'[+\\-×÷⌈⌊∣|⍳?*⍟○!⌹<≤=>≥≠≡≢∊⍷∪∩~∨∧⍱⍲⍴,⍪⌽⊖⍉↑↓⊂⊃⌷⍋⍒⊤⊥⍕⍎⊣⊢⍁⍂≈⌸⍯↗]', Operator), # # Constant diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py index 5b8cab80..0a40e641 100644 --- a/pygments/lexers/asm.py +++ b/pygments/lexers/asm.py @@ -35,7 +35,7 @@ class GasLexer(RegexLexer): #: optional Comment or Whitespace string = r'"(\\"|[^"])*"' char = r'[\w$.@-]' - identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)' + identifier = r'(?:[a-zA-Z$_]' + char + r'*|\.' + char + '+)' number = r'(?:0[xX][a-zA-Z0-9]+|\d+)' tokens = { @@ -53,6 +53,7 @@ class GasLexer(RegexLexer): ('@' + identifier, Name.Attribute), (number, Number.Integer), (r'[\r\n]+', Text, '#pop'), + (r'[;#].*?\n', Comment, '#pop'), include('punctuation'), include('whitespace') @@ -76,6 +77,7 @@ class GasLexer(RegexLexer): ('$'+number, Number.Integer), (r"$'(.|\\')'", String.Char), (r'[\r\n]+', Text, '#pop'), + (r'[;#].*?\n', Comment, '#pop'), include('punctuation'), include('whitespace') @@ -256,7 +258,7 @@ class HsailLexer(RegexLexer): (r'0[xX][a-fA-F0-9]+', Number.Hex), (ieeefloat, Number.Float), (float, Number.Float), - ('\d+', Number.Integer), + (r'\d+', Number.Integer), (r'[=<>{}\[\]()*.,:;!]|x\b', Punctuation) ], @@ -350,7 +352,7 @@ class LlvmLexer(RegexLexer): include('whitespace'), # Before keywords, because keywords are valid label names :(... - (identifier + '\s*:', Name.Label), + (identifier + r'\s*:', Name.Label), include('keyword'), diff --git a/pygments/lexers/bibtex.py b/pygments/lexers/bibtex.py index a6159f81..7244ef2f 100644 --- a/pygments/lexers/bibtex.py +++ b/pygments/lexers/bibtex.py @@ -101,12 +101,12 @@ class BibTeXLexer(ExtendedRegexLexer): 'quoted-string': [ (r'\{', String, 'braced-string'), ('"', String, '#pop'), - ('[^\{\"]+', String), + (r'[^\{\"]+', String), ], 'braced-string': [ (r'\{', String, '#push'), (r'\}', String, '#pop'), - ('[^\{\}]+', String), + (r'[^\{\}]+', String), ], 'whitespace': [ (r'\s+', Text), @@ -154,7 +154,7 @@ class BSTLexer(RegexLexer): default('#pop'), ], 'whitespace': [ - ('\s+', Text), + (r'\s+', Text), ('%.*?$', Comment.SingleLine), ], } diff --git a/pygments/lexers/c_cpp.py b/pygments/lexers/c_cpp.py index 691f5ab4..38f425db 100644 --- a/pygments/lexers/c_cpp.py +++ b/pygments/lexers/c_cpp.py @@ -36,7 +36,7 @@ class CFamilyLexer(RegexLexer): tokens = { 'whitespace': [ # preprocessor directives: without whitespace - ('^#if\s+0', Comment.Preproc, 'if0'), + (r'^#if\s+0', Comment.Preproc, 'if0'), ('^#', Comment.Preproc, 'macro'), # or with whitespace ('^(' + _ws1 + r')(#if\s+0)', @@ -84,7 +84,7 @@ class CFamilyLexer(RegexLexer): prefix=r'__', suffix=r'\b'), Keyword.Reserved), (r'(true|false|NULL)\b', Name.Builtin), (r'([a-zA-Z_]\w*)(\s*)(:)(?!:)', bygroups(Name.Label, Text, Punctuation)), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'root': [ include('whitespace'), @@ -190,9 +190,9 @@ class CLexer(CFamilyLexer): priority = 0.1 def analyse_text(text): - if re.search('^\s*#include [<"]', text, re.MULTILINE): + if re.search(r'^\s*#include [<"]', text, re.MULTILINE): return 0.1 - if re.search('^\s*#ifn?def ', text, re.MULTILINE): + if re.search(r'^\s*#ifn?def ', text, re.MULTILINE): return 0.1 diff --git a/pygments/lexers/c_like.py b/pygments/lexers/c_like.py index 38827219..fd147a8a 100644 --- a/pygments/lexers/c_like.py +++ b/pygments/lexers/c_like.py @@ -245,7 +245,7 @@ class ValaLexer(RegexLexer): 'ulong', 'unichar', 'ushort'), suffix=r'\b'), Keyword.Type), (r'(true|false|null)\b', Name.Builtin), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'root': [ include('whitespace'), @@ -344,7 +344,7 @@ class SwigLexer(CppLexer): # SWIG directives (r'(%[a-z_][a-z0-9_]*)', Name.Function), # Special variables - ('\$\**\&?\w+', Name), + (r'\$\**\&?\w+', Name), # Stringification / additional preprocessor directives (r'##*[a-zA-Z_]\w*', Comment.Preproc), inherit, diff --git a/pygments/lexers/capnproto.py b/pygments/lexers/capnproto.py index 203523a1..2615dcaf 100644 --- a/pygments/lexers/capnproto.py +++ b/pygments/lexers/capnproto.py @@ -44,34 +44,34 @@ class CapnProtoLexer(RegexLexer): ], 'type': [ (r'[^][=;,(){}$]+', Name.Class), - (r'[[(]', Name.Class, 'parentype'), + (r'[\[(]', Name.Class, 'parentype'), default('#pop'), ], 'parentype': [ (r'[^][;()]+', Name.Class), - (r'[[(]', Name.Class, '#push'), + (r'[\[(]', Name.Class, '#push'), (r'[])]', Name.Class, '#pop'), default('#pop'), ], 'expression': [ (r'[^][;,(){}$]+', Literal), - (r'[[(]', Literal, 'parenexp'), + (r'[\[(]', Literal, 'parenexp'), default('#pop'), ], 'parenexp': [ (r'[^][;()]+', Literal), - (r'[[(]', Literal, '#push'), + (r'[\[(]', Literal, '#push'), (r'[])]', Literal, '#pop'), default('#pop'), ], 'annotation': [ (r'[^][;,(){}=:]+', Name.Attribute), - (r'[[(]', Name.Attribute, 'annexp'), + (r'[\[(]', Name.Attribute, 'annexp'), default('#pop'), ], 'annexp': [ (r'[^][;()]+', Name.Attribute), - (r'[[(]', Name.Attribute, '#push'), + (r'[\[(]', Name.Attribute, '#push'), (r'[])]', Name.Attribute, '#pop'), default('#pop'), ], diff --git a/pygments/lexers/chapel.py b/pygments/lexers/chapel.py index 55bf0e1e..16ce720b 100644 --- a/pygments/lexers/chapel.py +++ b/pygments/lexers/chapel.py @@ -42,16 +42,26 @@ class ChapelLexer(RegexLexer): (r'(bool|complex|imag|int|opaque|range|real|string|uint)\b', Keyword.Type), (words(( - 'align', 'as', 'atomic', 'begin', 'break', 'by', 'cobegin', - 'coforall', 'continue', 'delete', 'dmapped', 'do', 'domain', - 'else', 'enum', 'except', 'export', 'extern', 'for', 'forall', - 'if', 'index', 'inline', 'iter', 'label', 'lambda', 'let', - 'local', 'new', 'noinit', 'on', 'only', 'otherwise', 'pragma', - 'private', 'public', 'reduce', 'require', 'return', 'scan', - 'select', 'serial', 'single', 'sparse', 'subdomain', 'sync', - 'then', 'use', 'when', 'where', 'while', 'with', 'yield', + 'align', 'as', 'atomic', + 'begin', 'borrowed', 'break', 'by', + 'catch', 'cobegin', 'coforall', 'continue', + 'delete', 'dmapped', 'do', 'domain', + 'else', 'enum', 'except', 'export', 'extern', + 'for', 'forall', + 'if', 'index', 'inline', + 'label', 'lambda', 'let', 'local', + 'new', 'noinit', + 'on', 'only', 'otherwise', 'override', 'owned', + 'pragma', 'private', 'prototype', 'public', + 'reduce', 'require', 'return', + 'scan', 'select', 'serial', 'shared', 'single', 'sparse', 'subdomain', 'sync', + 'then', 'throw', 'throws', 'try', + 'unmanaged', 'use', + 'when', 'where', 'while', 'with', + 'yield', 'zip'), suffix=r'\b'), Keyword), + (r'(iter)((?:\s)+)', bygroups(Keyword, Text), 'procname'), (r'(proc)((?:\s)+)', bygroups(Keyword, Text), 'procname'), (r'(class|module|record|union)(\s+)', bygroups(Keyword, Text), 'classname'), @@ -96,7 +106,7 @@ class ChapelLexer(RegexLexer): (r'[a-zA-Z_][\w$]*', Name.Class, '#pop'), ], 'procname': [ - (r'([a-zA-Z_][\w$]+|\~[a-zA-Z_][\w$]+|[+*/!~%<>=&^|\-]{1,2})', + (r'([a-zA-Z_][.\w$]*|\~[a-zA-Z_][.\w$]*|[+*/!~%<>=&^|\-]{1,2})', Name.Function, '#pop'), ], } diff --git a/pygments/lexers/clean.py b/pygments/lexers/clean.py index ba2569f6..dc973bea 100644 --- a/pygments/lexers/clean.py +++ b/pygments/lexers/clean.py @@ -5,14 +5,13 @@ Lexer for the Clean language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -from pygments.lexer import ExtendedRegexLexer, LexerContext, \ - bygroups, words, include, default -from pygments.token import Comment, Keyword, Literal, Name, Number, Operator, \ - Punctuation, String, Text, Whitespace +from pygments.lexer import ExtendedRegexLexer, words, include, bygroups +from pygments.token import Comment, Error, Keyword, Literal, Name, Number, \ + Operator, Punctuation, String, Whitespace __all__ = ['CleanLexer'] @@ -28,261 +27,152 @@ class CleanLexer(ExtendedRegexLexer): aliases = ['clean'] filenames = ['*.icl', '*.dcl'] - def get_tokens_unprocessed(self, text=None, context=None): - ctx = LexerContext(text, 0) - ctx.indent = 0 - return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context=ctx) + keywords = ( + 'case', 'ccall', 'class', 'code', 'code inline', 'derive', 'export', + 'foreign', 'generic', 'if', 'in', 'infix', 'infixl', 'infixr', + 'instance', 'let', 'of', 'otherwise', 'special', 'stdcall', 'where', + 'with') - def check_class_not_import(lexer, match, ctx): - if match.group(0) == 'import': - yield match.start(), Keyword.Namespace, match.group(0) - ctx.stack = ctx.stack[:-1] + ['fromimportfunc'] - else: - yield match.start(), Name.Class, match.group(0) - ctx.pos = match.end() + modulewords = ('implementation', 'definition', 'system') - def check_instance_class(lexer, match, ctx): - if match.group(0) == 'instance' or match.group(0) == 'class': - yield match.start(), Keyword, match.group(0) - else: - yield match.start(), Name.Function, match.group(0) - ctx.stack = ctx.stack + ['fromimportfunctype'] - ctx.pos = match.end() - - @staticmethod - def indent_len(text): - # Tabs are four spaces: - # https://svn.cs.ru.nl/repos/clean-platform/trunk/doc/STANDARDS.txt - text = text.replace('\n', '') - return len(text.replace('\t', ' ')), len(text) - - def store_indent(lexer, match, ctx): - ctx.indent, _ = CleanLexer.indent_len(match.group(0)) - ctx.pos = match.end() - yield match.start(), Text, match.group(0) - - def check_indent1(lexer, match, ctx): - indent, reallen = CleanLexer.indent_len(match.group(0)) - if indent > ctx.indent: - yield match.start(), Whitespace, match.group(0) - ctx.pos = match.start() + reallen + 1 - else: - ctx.indent = 0 - ctx.pos = match.start() - ctx.stack = ctx.stack[:-1] - yield match.start(), Whitespace, match.group(0)[1:] - - def check_indent2(lexer, match, ctx): - indent, reallen = CleanLexer.indent_len(match.group(0)) - if indent > ctx.indent: - yield match.start(), Whitespace, match.group(0) - ctx.pos = match.start() + reallen + 1 - else: - ctx.indent = 0 - ctx.pos = match.start() - ctx.stack = ctx.stack[:-2] - - def check_indent3(lexer, match, ctx): - indent, reallen = CleanLexer.indent_len(match.group(0)) - if indent > ctx.indent: - yield match.start(), Whitespace, match.group(0) - ctx.pos = match.start() + reallen + 1 - else: - ctx.indent = 0 - ctx.pos = match.start() - ctx.stack = ctx.stack[:-3] - yield match.start(), Whitespace, match.group(0)[1:] - if match.group(0) == '\n\n': - ctx.pos = ctx.pos + 1 - - def skip(lexer, match, ctx): - ctx.stack = ctx.stack[:-1] - ctx.pos = match.end() - yield match.start(), Comment, match.group(0) - - keywords = ('class', 'instance', 'where', 'with', 'let', 'let!', - 'in', 'case', 'of', 'infix', 'infixr', 'infixl', 'generic', - 'derive', 'otherwise', 'code', 'inline') + lowerId = r'[a-z`][\w\d`]*' + upperId = r'[A-Z`][\w\d`]*' + funnyId = r'[~@#\$%\^?!+\-*<>\\/|&=:]+' + scoreUpperId = r'_' + upperId + scoreLowerId = r'_' + lowerId + moduleId = r'[a-zA-Z_][a-zA-Z0-9_.`]+' + classId = '|'.join([lowerId, upperId, funnyId]) tokens = { - 'common': [ - (r';', Punctuation, '#pop'), - (r'//', Comment, 'singlecomment'), - ], 'root': [ - # Comments + include('comments'), + include('keywords'), + include('module'), + include('import'), + include('whitespace'), + include('literals'), + include('operators'), + include('delimiters'), + include('names'), + ], + 'whitespace': [ + (r'\s+', Whitespace), + ], + 'comments': [ (r'//.*\n', Comment.Single), - (r'(?s)/\*\*.*?\*/', Comment.Special), - (r'(?s)/\*.*?\*/', Comment.Multi), - - # Modules, imports, etc. - (r'\b((?:implementation|definition|system)\s+)?(module)(\s+)([\w`.]+)', - bygroups(Keyword.Namespace, Keyword.Namespace, Text, Name.Class)), - (r'(?<=\n)import(?=\s)', Keyword.Namespace, 'import'), - (r'(?<=\n)from(?=\s)', Keyword.Namespace, 'fromimport'), - - # Keywords - # We cannot use (?s)^|(?<=\s) as prefix, so need to repeat this - (words(keywords, prefix=r'(?<=\s)', suffix=r'(?=\s)'), Keyword), - (words(keywords, prefix=r'^', suffix=r'(?=\s)'), Keyword), - - # Function definitions - (r'(?=\{\|)', Whitespace, 'genericfunction'), - (r'(?<=\n)([ \t]*)([\w`$()=\-<>~*\^|+&%]+)((?:\s+\w)*)(\s*)(::)', - bygroups(store_indent, Name.Function, Keyword.Type, Whitespace, - Punctuation), - 'functiondefargs'), - - # Type definitions - (r'(?<=\n)([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'), - (r'^([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'), - - # Literals - (r'\'\\?.(?<!\\)\'', String.Char), - (r'\'\\\d+\'', String.Char), - (r'\'\\\\\'', String.Char), # (special case for '\\') - (r'[+\-~]?\s*\d+\.\d+(E[+\-~]?\d+)?\b', Number.Float), - (r'[+\-~]?\s*0[0-7]\b', Number.Oct), - (r'[+\-~]?\s*0x[0-9a-fA-F]\b', Number.Hex), - (r'[+\-~]?\s*\d+\b', Number.Integer), - (r'"', String.Double, 'doubleqstring'), - (words(('True', 'False'), prefix=r'(?<=\s)', suffix=r'(?=\s)'), - Literal), - - # Qualified names - (r'(\')([\w.]+)(\'\.)', - bygroups(Punctuation, Name.Namespace, Punctuation)), - - # Everything else is some name - (r'([\w`$%/?@]+\.?)*[\w`$%/?@]+', Name), - - # Punctuation - (r'[{}()\[\],:;.#]', Punctuation), - (r'[+\-=!<>|&~*\^/]', Operator), - (r'\\\\', Operator), - - # Lambda expressions - (r'\\.*?(->|\.|=)', Name.Function), - - # Whitespace - (r'\s', Whitespace), - - include('common'), + (r'/\*', Comment.Multi, 'comments.in'), + (r'/\*\*', Comment.Special, 'comments.in'), ], - 'fromimport': [ - include('common'), - (r'([\w`.]+)', check_class_not_import), - (r'\n', Whitespace, '#pop'), - (r'\s', Whitespace), + 'comments.in': [ + (r'\*\/', Comment.Multi, '#pop'), + (r'/\*', Comment.Multi, '#push'), + (r'[^*/]+', Comment.Multi), + (r'\*(?!/)', Comment.Multi), + (r'/', Comment.Multi), ], - 'fromimportfunc': [ - include('common'), - (r'(::)(\s+)([^,\s]+)', bygroups(Punctuation, Text, Keyword.Type)), - (r'([\w`$()=\-<>~*\^|+&%/]+)', check_instance_class), - (r',', Punctuation), - (r'\n', Whitespace, '#pop'), - (r'\s', Whitespace), + 'keywords': [ + (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword), ], - 'fromimportfunctype': [ - include('common'), - (r'[{(\[]', Punctuation, 'combtype'), - (r',', Punctuation, '#pop'), - (r'[:;.#]', Punctuation), - (r'\n', Whitespace, '#pop:2'), - (r'[^\S\n]+', Whitespace), - (r'\S+', Keyword.Type), + 'module': [ + (words(modulewords, prefix=r'\b', suffix=r'\b'), Keyword.Namespace), + (r'\bmodule\b', Keyword.Namespace, 'module.name'), ], - 'combtype': [ - include('common'), - (r'[})\]]', Punctuation, '#pop'), - (r'[{(\[]', Punctuation, '#pop'), - (r'[,:;.#]', Punctuation), - (r'\s+', Whitespace), - (r'\S+', Keyword.Type), + 'module.name': [ + include('whitespace'), + (moduleId, Name.Class, '#pop'), ], 'import': [ - include('common'), - (words(('from', 'import', 'as', 'qualified'), - prefix='(?<=\s)', suffix='(?=\s)'), Keyword.Namespace), - (r'[\w`.]+', Name.Class), - (r'\n', Whitespace, '#pop'), - (r',', Punctuation), - (r'[^\S\n]+', Whitespace), - ], - 'singlecomment': [ - (r'(.)(?=\n)', skip), - (r'.+(?!\n)', Comment), - ], - 'doubleqstring': [ - (r'[^\\"]+', String.Double), - (r'"', String.Double, '#pop'), - (r'\\.', String.Double), - ], - 'typedef': [ - include('common'), - (r'[\w`]+', Keyword.Type), - (r'[:=|(),\[\]{}!*]', Punctuation), - (r'->', Punctuation), - (r'\n(?=[^\s|])', Whitespace, '#pop'), - (r'\s', Whitespace), - (r'.', Keyword.Type), + (r'\b(import)\b(\s*)', bygroups(Keyword, Whitespace), 'import.module'), + (r'\b(from)\b(\s*)\b(' + moduleId + r')\b(\s*)\b(import)\b', + bygroups(Keyword, Whitespace, Name.Class, Whitespace, Keyword), + 'import.what'), + ], + 'import.module': [ + (r'\b(qualified)\b(\s*)', bygroups(Keyword, Whitespace)), + (r'(\s*)\b(as)\b', bygroups(Whitespace, Keyword), ('#pop', 'import.module.as')), + (moduleId, Name.Class), + (r'(\s*)(,)(\s*)', bygroups(Whitespace, Punctuation, Whitespace)), + (r'\s*', Whitespace, '#pop'), + ], + 'import.module.as': [ + include('whitespace'), + (lowerId, Name.Class, '#pop'), + (upperId, Name.Class, '#pop'), + ], + 'import.what': [ + (r'\b(class)\b(\s+)(' + classId + r')', + bygroups(Keyword, Whitespace, Name.Class), 'import.what.class'), + (r'\b(instance)(\s+)(' + classId + r')(\s+)', + bygroups(Keyword, Whitespace, Name.Class, Whitespace), 'import.what.instance'), + (r'(::)(\s*)\b(' + upperId + r')\b', + bygroups(Punctuation, Whitespace, Name.Class), 'import.what.type'), + (r'\b(generic)\b(\s+)\b(' + lowerId + '|' + upperId + r')\b', + bygroups(Keyword, Whitespace, Name)), + include('names'), + (r'(,)(\s+)', bygroups(Punctuation, Whitespace)), + (r'$', Whitespace, '#pop'), + include('whitespace'), + ], + 'import.what.class': [ + (r',', Punctuation, '#pop'), + (r'\(', Punctuation, 'import.what.class.members'), + (r'$', Whitespace, '#pop:2'), + include('whitespace'), ], - 'genericfunction': [ - include('common'), - (r'\{\|', Punctuation), - (r'\|\}', Punctuation, '#pop'), + 'import.what.class.members': [ (r',', Punctuation), - (r'->', Punctuation), - (r'(\s+of\s+)(\{)', bygroups(Keyword, Punctuation), 'genericftypes'), - (r'\s', Whitespace), - (r'[\w`\[\]{}!]+', Keyword.Type), - (r'[*()]', Punctuation), + (r'\.\.', Punctuation), + (r'\)', Punctuation, '#pop'), + include('names'), + ], + 'import.what.instance': [ + (r'[,)]', Punctuation, '#pop'), + (r'\(', Punctuation, 'import.what.instance'), + (r'$', Whitespace, '#pop:2'), + include('whitespace'), + include('names'), + ], + 'import.what.type': [ + (r',', Punctuation, '#pop'), + (r'[({]', Punctuation, 'import.what.type.consesandfields'), + (r'$', Whitespace, '#pop:2'), + include('whitespace'), ], - 'genericftypes': [ - include('common'), - (r'[\w`]+', Keyword.Type), + 'import.what.type.consesandfields': [ (r',', Punctuation), - (r'\s', Whitespace), - (r'\}', Punctuation, '#pop'), - ], - 'functiondefargs': [ - include('common'), - (r'\n(\s*)', check_indent1), - (r'[!{}()\[\],:;.#]', Punctuation), - (r'->', Punctuation, 'functiondefres'), - (r'^(?=\S)', Whitespace, '#pop'), - (r'\S', Keyword.Type), - (r'\s', Whitespace), - ], - 'functiondefres': [ - include('common'), - (r'\n(\s*)', check_indent2), - (r'^(?=\S)', Whitespace, '#pop:2'), - (r'[!{}()\[\],:;.#]', Punctuation), - (r'\|', Punctuation, 'functiondefclasses'), - (r'\S', Keyword.Type), - (r'\s', Whitespace), - ], - 'functiondefclasses': [ - include('common'), - (r'\n(\s*)', check_indent3), - (r'^(?=\S)', Whitespace, '#pop:3'), - (r'[,&]', Punctuation), - (r'\[', Punctuation, 'functiondefuniquneq'), - (r'[\w`$()=\-<>~*\^|+&%/{}\[\]@]', Name.Function, 'functionname'), - (r'\s+', Whitespace), - ], - 'functiondefuniquneq': [ - include('common'), - (r'[a-z]+', Keyword.Type), - (r'\s+', Whitespace), - (r'<=|,', Punctuation), - (r'\]', Punctuation, '#pop') - ], - 'functionname': [ - include('common'), - (r'[\w`$()=\-<>~*\^|+&%/]+', Name.Function), - (r'(?=\{\|)', Punctuation, 'genericfunction'), - default('#pop'), + (r'\.\.', Punctuation), + (r'[)}]', Punctuation, '#pop'), + include('names'), + ], + 'literals': [ + (r'\'([^\'\\]|\\(x[\da-fA-F]+|\d+|.))\'', Literal.Char), + (r'[+~-]?0[0-7]+\b', Number.Oct), + (r'[+~-]?\d+\.\d+(E[+-]?\d+)?', Number.Float), + (r'[+~-]?\d+\b', Number.Integer), + (r'[+~-]?0x[\da-fA-F]+\b', Number.Hex), + (r'True|False', Literal), + (r'"', String.Double, 'literals.stringd'), + ], + 'literals.stringd': [ + (r'[^\\"\n]+', String.Double), + (r'"', String.Double, '#pop'), + (r'\\.', String.Double), + (r'[$\n]', Error, '#pop'), + ], + 'operators': [ + (r'[-~@#\$%\^?!+*<>\\/|&=:\.]+', Operator), + (r'\b_+\b', Operator), + ], + 'delimiters': [ + (r'[,;(){}\[\]]', Punctuation), + (r'(\')([\w`.]+)(\')', + bygroups(Punctuation, Name.Class, Punctuation)), + ], + 'names': [ + (lowerId, Name), + (scoreLowerId, Name), + (funnyId, Name.Function), + (upperId, Name.Class), + (scoreUpperId, Name.Class), ] } diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py index c39b1a52..c35e866d 100644 --- a/pygments/lexers/configs.py +++ b/pygments/lexers/configs.py @@ -15,6 +15,7 @@ from pygments.lexer import RegexLexer, default, words, bygroups, include, using from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Whitespace, Literal from pygments.lexers.shell import BashLexer +from pygments.lexers.data import JsonLexer __all__ = ['IniLexer', 'RegeditLexer', 'PropertiesLexer', 'KconfigLexer', 'Cfengine3Lexer', 'ApacheConfLexer', 'SquidConfLexer', @@ -539,20 +540,25 @@ class DockerLexer(RegexLexer): filenames = ['Dockerfile', '*.docker'] mimetypes = ['text/x-dockerfile-config'] - _keywords = (r'(?:FROM|MAINTAINER|CMD|EXPOSE|ENV|ADD|ENTRYPOINT|' - r'VOLUME|WORKDIR)') - + _keywords = (r'(?:FROM|MAINTAINER|EXPOSE|WORKDIR|USER|STOPSIGNAL)') + _bash_keywords = (r'(?:RUN|CMD|ENTRYPOINT|ENV|ARG|LABEL|ADD|COPY)') + _lb = r'(?:\s*\\?\s*)' # dockerfile line break regex flags = re.IGNORECASE | re.MULTILINE tokens = { 'root': [ - (r'^(ONBUILD)(\s+)(%s)\b' % (_keywords,), - bygroups(Name.Keyword, Whitespace, Keyword)), - (r'^(%s)\b(.*)' % (_keywords,), bygroups(Keyword, String)), (r'#.*', Comment), - (r'RUN', Keyword), # Rest of line falls through + (r'(ONBUILD)(%s)' % (_lb,), bygroups(Keyword, using(BashLexer))), + (r'(HEALTHCHECK)((%s--\w+=\w+%s)*)' % (_lb, _lb), + bygroups(Keyword, using(BashLexer))), + (r'(VOLUME|ENTRYPOINT|CMD|SHELL)(%s)(\[.*?\])' % (_lb,), + bygroups(Keyword, using(BashLexer), using(JsonLexer))), + (r'(LABEL|ENV|ARG)((%s\w+=\w+%s)*)' % (_lb, _lb), + bygroups(Keyword, using(BashLexer))), + (r'(%s|VOLUME)\b(.*)' % (_keywords), bygroups(Keyword, String)), + (r'(%s)' % (_bash_keywords,), Keyword), (r'(.*\\\n)*.+', using(BashLexer)), - ], + ] } @@ -586,7 +592,7 @@ class TerraformLexer(RegexLexer): prefix=r'\b', suffix=r'\b'), Keyword.Reserved, 'function'), (words(('ingress', 'egress', 'listener', 'default', 'connection', 'alias'), prefix=r'\b', suffix=r'\b'), Keyword.Declaration), - ('\$\{', String.Interpol, 'var_builtin'), + (r'\$\{', String.Interpol, 'var_builtin'), ], 'function': [ (r'(\s+)(".*")(\s+)', bygroups(Text, String, Text)), diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 858aa348..c30b87cc 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -3,7 +3,7 @@ pygments.lexers.csound ~~~~~~~~~~~~~~~~~~~~~~ - Lexers for CSound languages. + Lexers for Csound languages. :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. @@ -12,9 +12,9 @@ import re from pygments.lexer import RegexLexer, bygroups, default, include, using, words -from pygments.token import Comment, Keyword, Name, Number, Operator, Punctuation, \ - String, Text -from pygments.lexers._csound_builtins import OPCODES +from pygments.token import Comment, Error, Keyword, Name, Number, Operator, Punctuation, \ + String, Text, Whitespace +from pygments.lexers._csound_builtins import OPCODES, DEPRECATED_OPCODES from pygments.lexers.html import HtmlLexer from pygments.lexers.python import PythonLexer from pygments.lexers.scripting import LuaLexer @@ -25,74 +25,104 @@ newline = (r'((?:(?:;|//).*)*)(\n)', bygroups(Comment.Single, Text)) class CsoundLexer(RegexLexer): - # Subclasses must define a 'single-line string' state. tokens = { 'whitespace': [ (r'[ \t]+', Text), - (r'\\\n', Text), - (r'/[*](.|\n)*?[*]/', Comment.Multiline) + (r'/[*](?:.|\n)*?[*]/', Comment.Multiline), + (r'(?:;|//).*$', Comment.Single), + (r'(\\)(\n)', bygroups(Whitespace, Text)) ], - 'macro call': [ - (r'(\$\w+\.?)(\()', bygroups(Comment.Preproc, Punctuation), - 'function macro call'), - (r'\$\w+(\.|\b)', Comment.Preproc) - ], - 'function macro call': [ - (r"((?:\\['\)]|[^'\)])+)(')", bygroups(Comment.Preproc, Punctuation)), - (r"([^'\)]+)(\))", bygroups(Comment.Preproc, Punctuation), '#pop') + 'preprocessor directives': [ + (r'#(?:e(?:nd(?:if)?|lse)\b|##)|@@?[ \t]*\d+', Comment.Preproc), + (r'#include', Comment.Preproc, 'include directive'), + (r'#[ \t]*define', Comment.Preproc, 'define directive'), + (r'#(?:ifn?def|undef)\b', Comment.Preproc, 'macro directive') ], - 'whitespace or macro call': [ + 'include directive': [ include('whitespace'), - include('macro call') + (r'([^ \t]).*?\1', String, '#pop') ], - 'preprocessor directives': [ - (r'#(e(nd(if)?|lse)|ifn?def|undef)\b|##', Comment.Preproc), - (r'#include\b', Comment.Preproc, 'include'), - (r'#[ \t]*define\b', Comment.Preproc, 'macro name'), - (r'@+[ \t]*\d*', Comment.Preproc) + 'define directive': [ + (r'\n', Text), + include('whitespace'), + (r'([A-Z_a-z]\w*)(\()', bygroups(Comment.Preproc, Punctuation), + ('#pop', 'macro parameter name list')), + (r'[A-Z_a-z]\w*', Comment.Preproc, ('#pop', 'before macro body')) ], - - 'include': [ + 'macro parameter name list': [ include('whitespace'), - (r'"', String, 'single-line string') + (r'[A-Z_a-z]\w*', Comment.Preproc), + (r"['#]", Punctuation), + (r'\)', Punctuation, ('#pop', 'before macro body')) ], - - 'macro name': [ + 'before macro body': [ + (r'\n', Text), include('whitespace'), - (r'(\w+)(\()', bygroups(Comment.Preproc, Text), - 'function macro argument list'), - (r'\w+', Comment.Preproc, 'object macro definition after name') + (r'#', Punctuation, ('#pop', 'macro body')) + ], + 'macro body': [ + (r'(?:\\(?!#)|[^#\\]|\n)+', Comment.Preproc), + (r'\\#', Comment.Preproc), + (r'(?<!\\)#', Punctuation, '#pop') ], - 'object macro definition after name': [ + + 'macro directive': [ include('whitespace'), - (r'#', Punctuation, 'object macro replacement text') + (r'[A-Z_a-z]\w*', Comment.Preproc, '#pop') + ], + + 'macro uses': [ + (r'(\$[A-Z_a-z]\w*\.?)(\()', bygroups(Comment.Preproc, Punctuation), + 'macro parameter value list'), + (r'\$[A-Z_a-z]\w*(?:\.|\b)', Comment.Preproc) ], - 'object macro replacement text': [ - (r'(\\#|[^#])+', Comment.Preproc), - (r'#', Punctuation, '#pop:3') + 'macro parameter value list': [ + (r'(?:[^\'#"{()]|\{(?!\{))+', Comment.Preproc), + (r"['#]", Punctuation), + (r'"', String, 'macro parameter value quoted string'), + (r'\{\{', String, 'macro parameter value braced string'), + (r'\(', Comment.Preproc, 'macro parameter value parenthetical'), + (r'\)', Punctuation, '#pop') ], - 'function macro argument list': [ - (r"(\w+)(['#])", bygroups(Comment.Preproc, Punctuation)), - (r'(\w+)(\))', bygroups(Comment.Preproc, Punctuation), - 'function macro definition after name') + 'macro parameter value quoted string': [ + (r"\\[#'()]", Comment.Preproc), + (r"[#'()]", Error), + include('quoted string') ], - 'function macro definition after name': [ - (r'[ \t]+', Text), - (r'#', Punctuation, 'function macro replacement text') + 'macro parameter value braced string': [ + (r"\\[#'()]", Comment.Preproc), + (r"[#'()]", Error), + include('braced string') + ], + 'macro parameter value parenthetical': [ + (r'(?:[^\\()]|\\\))+', Comment.Preproc), + (r'\(', Comment.Preproc, '#push'), + (r'\)', Comment.Preproc, '#pop') ], - 'function macro replacement text': [ - (r'(\\#|[^#])+', Comment.Preproc), - (r'#', Punctuation, '#pop:4') + + 'whitespace and macro uses': [ + include('whitespace'), + include('macro uses') + ], + + 'numbers': [ + (r'\d+[Ee][+-]?\d+|(\d+\.\d*|\d*\.\d+)([Ee][+-]?\d+)?', Number.Float), + (r'(0[Xx])([0-9A-Fa-f]+)', bygroups(Keyword.Type, Number.Hex)), + (r'\d+', Number.Integer) + ], + + 'braced string': [ + # Do nothing. This must be defined in subclasses. ] } class CsoundScoreLexer(CsoundLexer): """ - For `Csound <http://csound.github.io>`_ scores. + For `Csound <https://csound.github.io>`_ scores. .. versionadded:: 2.1 """ @@ -102,47 +132,77 @@ class CsoundScoreLexer(CsoundLexer): filenames = ['*.sco'] tokens = { - 'partial statement': [ + 'root': [ + (r'\n', Text), + include('whitespace and macro uses'), include('preprocessor directives'), - (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float), - (r'0[xX][a-fA-F0-9]+', Number.Hex), - (r'\d+', Number.Integer), - (r'"', String, 'single-line string'), - (r'[+\-*/%^!=<>|&#~.]', Operator), - (r'[]()[]', Punctuation), - (r'\w+', Comment.Preproc) - ], - 'statement': [ - include('whitespace or macro call'), - newline + ('#pop',), - include('partial statement') + (r'[abCdefiqstvxy]', Keyword), + # There is also a w statement that is generated internally and should not be + # used; see https://github.com/csound/csound/issues/750. + + (r'z', Keyword.Constant), + # z is a constant equal to 800,000,000,000. 800 billion seconds is about + # 25,367.8 years. See also + # https://csound.github.io/docs/manual/ScoreTop.html and + # https://github.com/csound/csound/search?q=stof+path%3AEngine+filename%3Asread.c. + + (r'([nNpP][pP])(\d+)', bygroups(Keyword, Number.Integer)), + + (r'[mn]', Keyword, 'mark statement'), + + include('numbers'), + (r'[!+\-*/^%&|<>#~.]', Operator), + (r'[()\[\]]', Punctuation), + (r'"', String, 'quoted string'), + (r'\{', Comment.Preproc, 'loop after left brace'), ], - 'root': [ - newline, - include('whitespace or macro call'), - (r'[{}]', Punctuation, 'statement'), - (r'[abefimq-tv-z]|[nN][pP]?', Keyword, 'statement') + 'mark statement': [ + include('whitespace and macro uses'), + (r'[A-Z_a-z]\w*', Name.Label), + (r'\n', Text, '#pop') ], - 'single-line string': [ + 'quoted string': [ (r'"', String, '#pop'), - (r'[^\\"]+', String) + (r'[^"$]+', String), + include('macro uses'), + (r'[$]', String) + ], + + 'loop after left brace': [ + include('whitespace and macro uses'), + (r'\d+', Number.Integer, ('#pop', 'loop after repeat count')), + ], + 'loop after repeat count': [ + include('whitespace and macro uses'), + (r'[A-Z_a-z]\w*', Comment.Preproc, ('#pop', 'loop')) + ], + 'loop': [ + (r'\}', Comment.Preproc, '#pop'), + include('root') + ], + + # Braced strings are not allowed in Csound scores, but this is needed + # because the superclass includes it. + 'braced string': [ + (r'\}\}', String, '#pop'), + (r'[^}]|\}(?!\})', String) ] } class CsoundOrchestraLexer(CsoundLexer): """ - For `Csound <http://csound.github.io>`_ orchestras. + For `Csound <https://csound.github.io>`_ orchestras. .. versionadded:: 2.1 """ name = 'Csound Orchestra' aliases = ['csound', 'csound-orc'] - filenames = ['*.orc'] + filenames = ['*.orc', '*.udo'] user_defined_opcodes = set() @@ -152,159 +212,185 @@ class CsoundOrchestraLexer(CsoundLexer): yield match.start(), Name.Function, opcode def name_callback(lexer, match): - name = match.group(0) - if re.match('p\d+$', name) or name in OPCODES: + name = match.group(1) + if name in OPCODES or name in DEPRECATED_OPCODES: yield match.start(), Name.Builtin, name + if match.group(2): + yield match.start(2), Punctuation, match.group(2) + yield match.start(3), Keyword.Type, match.group(3) elif name in lexer.user_defined_opcodes: yield match.start(), Name.Function, name else: - nameMatch = re.search(r'^(g?[aikSw])(\w+)', name) + nameMatch = re.search(r'^(g?[afikSw])(\w+)', name) if nameMatch: yield nameMatch.start(1), Keyword.Type, nameMatch.group(1) yield nameMatch.start(2), Name, nameMatch.group(2) else: yield match.start(), Name, name + if match.group(2): + yield match.start(2), Punctuation, match.group(2) + yield match.start(3), Name, match.group(3) tokens = { - 'label': [ - (r'\b(\w+)(:)', bygroups(Name.Label, Punctuation)) - ], + 'root': [ + (r'\n', Text), + + (r'^([ \t]*)(\w+)(:)(?:[ \t]+|$)', bygroups(Text, Name.Label, Punctuation)), - 'partial expression': [ + include('whitespace and macro uses'), include('preprocessor directives'), - (r'\b(0dbfs|k(r|smps)|nchnls(_i)?|sr)\b', Name.Variable.Global), - (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float), - (r'0[xX][a-fA-F0-9]+', Number.Hex), - (r'\d+', Number.Integer), - (r'"', String, 'single-line string'), - (r'\{\{', String, 'multi-line string'), - (r'[+\-*/%^!=&|<>#~¬]', Operator), - (r'[](),?:[]', Punctuation), + + (r'\binstr\b', Keyword.Declaration, 'instrument numbers and identifiers'), + (r'\bopcode\b', Keyword.Declaration, 'after opcode keyword'), + (r'\b(?:end(?:in|op))\b', Keyword.Declaration), + + include('partial statements') + ], + + 'partial statements': [ + (r'\b(?:0dbfs|A4|k(?:r|smps)|nchnls(?:_i)?|sr)\b', Name.Variable.Global), + + include('numbers'), + + (r'\+=|-=|\*=|/=|<<|>>|<=|>=|==|!=|&&|\|\||[~¬]|[=!+\-*/^%&|<>#?:]', Operator), + (r'[(),\[\]]', Punctuation), + + (r'"', String, 'quoted string'), + (r'\{\{', String, 'braced string'), + (words(( - # Keywords 'do', 'else', 'elseif', 'endif', 'enduntil', 'fi', 'if', 'ithen', 'kthen', 'od', 'then', 'until', 'while', - # Opcodes that act as control structures - 'return', 'timout' ), prefix=r'\b', suffix=r'\b'), Keyword), - (words(('goto', 'igoto', 'kgoto', 'rigoto', 'tigoto'), - prefix=r'\b', suffix=r'\b'), Keyword, 'goto label'), - (words(('cggoto', 'cigoto', 'cingoto', 'ckgoto', 'cngoto'), - prefix=r'\b', suffix=r'\b'), Keyword, - ('goto label', 'goto expression')), - (words(('loop_ge', 'loop_gt', 'loop_le', 'loop_lt'), - prefix=r'\b', suffix=r'\b'), Keyword, - ('goto label', 'goto expression', 'goto expression', 'goto expression')), - (r'\bscoreline(_i)?\b', Name.Builtin, 'scoreline opcode'), - (r'\bpyl?run[it]?\b', Name.Builtin, 'python opcode'), - (r'\blua_(exec|opdef)\b', Name.Builtin, 'lua opcode'), - (r'\b[a-zA-Z_]\w*\b', name_callback) - ], - - 'expression': [ - include('whitespace or macro call'), - newline + ('#pop',), - include('partial expression') - ], + (words(('return', 'rireturn'), prefix=r'\b', suffix=r'\b'), Keyword.Pseudo), - 'root': [ - newline, - include('whitespace or macro call'), - (r'\binstr\b', Keyword, ('instrument block', 'instrument name list')), - (r'\bopcode\b', Keyword, ('opcode block', 'opcode parameter list', - 'opcode types', 'opcode types', 'opcode name')), - include('label'), - default('expression') - ], - - 'instrument name list': [ - include('whitespace or macro call'), - (r'\d+|\+?[a-zA-Z_]\w*', Name.Function), - (r',', Punctuation), - newline + ('#pop',) + (r'\b[ik]?goto\b', Keyword, 'goto label'), + (r'\b(r(?:einit|igoto)|tigoto)(\(|\b)', bygroups(Keyword.Pseudo, Punctuation), + 'goto label'), + (r'\b(c(?:g|in?|k|nk?)goto)(\(|\b)', bygroups(Keyword.Pseudo, Punctuation), + ('goto label', 'goto argument')), + (r'\b(timout)(\(|\b)', bygroups(Keyword.Pseudo, Punctuation), + ('goto label', 'goto argument', 'goto argument')), + (r'\b(loop_[gl][et])(\(|\b)', bygroups(Keyword.Pseudo, Punctuation), + ('goto label', 'goto argument', 'goto argument', 'goto argument')), + + (r'\bprintk?s\b', Name.Builtin, 'prints opcode'), + (r'\b(?:readscore|scoreline(?:_i)?)\b', Name.Builtin, 'Csound score opcode'), + (r'\bpyl?run[it]?\b', Name.Builtin, 'Python opcode'), + (r'\blua_(?:exec|opdef)\b', Name.Builtin, 'Lua opcode'), + (r'\bp\d+\b', Name.Variable.Instance), + (r'\b([A-Z_a-z]\w*)(?:(:)([A-Za-z]))?\b', name_callback) ], - 'instrument block': [ - newline, - include('whitespace or macro call'), - (r'\bendin\b', Keyword, '#pop'), - include('label'), - default('expression') + + 'instrument numbers and identifiers': [ + include('whitespace and macro uses'), + (r'\d+|[A-Z_a-z]\w*', Name.Function), + (r'[+,]', Punctuation), + (r'\n', Text, '#pop') ], - 'opcode name': [ - include('whitespace or macro call'), - (r'[a-zA-Z_]\w*', opcode_name_callback, '#pop') + 'after opcode keyword': [ + include('whitespace and macro uses'), + (r'[A-Z_a-z]\w*', opcode_name_callback, ('#pop', 'opcode type signatures')), + (r'\n', Text, '#pop') ], - 'opcode types': [ - include('whitespace or macro call'), - (r'0|[]afijkKoOpPStV[]+', Keyword.Type, '#pop'), - (r',', Punctuation) + 'opcode type signatures': [ + include('whitespace and macro uses'), + + # https://github.com/csound/csound/search?q=XIDENT+path%3AEngine+filename%3Acsound_orc.lex + (r'0|[afijkKoOpPStV\[\]]+', Keyword.Type), + + (r',', Punctuation), + (r'\n', Text, '#pop') ], - 'opcode parameter list': [ - include('whitespace or macro call'), - newline + ('#pop',) + + 'quoted string': [ + (r'"', String, '#pop'), + (r'[^\\"$%)]+', String), + include('macro uses'), + include('escape sequences'), + include('format specifiers'), + (r'[\\$%)]', String) ], - 'opcode block': [ - newline, - include('whitespace or macro call'), - (r'\bendop\b', Keyword, '#pop'), - include('label'), - default('expression') + 'braced string': [ + (r'\}\}', String, '#pop'), + (r'(?:[^\\%)}]|\}(?!\}))+', String), + include('escape sequences'), + include('format specifiers'), + (r'[\\%)]', String) + ], + 'escape sequences': [ + # https://github.com/csound/csound/search?q=unquote_string+path%3AEngine+filename%3Acsound_orc_compile.c + (r'\\(?:[\\abnrt"]|[0-7]{1,3})', String.Escape) + ], + # Format specifiers are highlighted in all strings, even though only + # fprintks https://csound.github.io/docs/manual/fprintks.html + # fprints https://csound.github.io/docs/manual/fprints.html + # printf/printf_i https://csound.github.io/docs/manual/printf.html + # printks https://csound.github.io/docs/manual/printks.html + # prints https://csound.github.io/docs/manual/prints.html + # sprintf https://csound.github.io/docs/manual/sprintf.html + # sprintfk https://csound.github.io/docs/manual/sprintfk.html + # work with strings that contain format specifiers. In addition, these + # opcodes’ handling of format specifiers is inconsistent: + # - fprintks, fprints, printks, and prints do accept %a and %A + # specifiers, but can’t accept %s specifiers. + # - printf, printf_i, sprintf, and sprintfk don’t accept %a and %A + # specifiers, but can accept %s specifiers. + # See https://github.com/csound/csound/issues/747 for more information. + 'format specifiers': [ + (r'%[#0\- +]*\d*(?:\.\d+)?[diuoxXfFeEgGaAcs]', String.Interpol), + (r'%%', String.Escape) + ], + + 'goto argument': [ + include('whitespace and macro uses'), + (r',', Punctuation, '#pop'), + include('partial statements') ], - 'goto label': [ - include('whitespace or macro call'), + include('whitespace and macro uses'), (r'\w+', Name.Label, '#pop'), default('#pop') ], - 'goto expression': [ - include('whitespace or macro call'), - (r',', Punctuation, '#pop'), - include('partial expression') - ], - 'single-line string': [ - include('macro call'), - (r'"', String, '#pop'), - # From https://github.com/csound/csound/blob/develop/Opcodes/fout.c#L1405 - (r'%\d*(\.\d+)?[cdhilouxX]', String.Interpol), - (r'%[!%nNrRtT]|[~^]|\\([\\aAbBnNrRtT"]|[0-7]{1,3})', String.Escape), - (r'[^\\"~$%\^\n]+', String), - (r'[\\"~$%\^\n]', String) + 'prints opcode': [ + include('whitespace and macro uses'), + (r'"', String, 'prints quoted string'), + default('#pop') ], - 'multi-line string': [ - (r'\}\}', String, '#pop'), - (r'[^}]+|\}(?!\})', String) + 'prints quoted string': [ + (r'\\\\[aAbBnNrRtT]', String.Escape), + (r'%[!nNrRtT]|[~^]{1,2}', String.Escape), + include('quoted string') ], - 'scoreline opcode': [ - include('whitespace or macro call'), - (r'\{\{', String, 'scoreline'), - default('#pop') + 'Csound score opcode': [ + include('whitespace and macro uses'), + (r'\{\{', String, 'Csound score'), + (r'\n', Text, '#pop') ], - 'scoreline': [ + 'Csound score': [ (r'\}\}', String, '#pop'), (r'([^}]+)|\}(?!\})', using(CsoundScoreLexer)) ], - 'python opcode': [ - include('whitespace or macro call'), - (r'\{\{', String, 'python'), - default('#pop') + 'Python opcode': [ + include('whitespace and macro uses'), + (r'\{\{', String, 'Python'), + (r'\n', Text, '#pop') ], - 'python': [ + 'Python': [ (r'\}\}', String, '#pop'), (r'([^}]+)|\}(?!\})', using(PythonLexer)) ], - 'lua opcode': [ - include('whitespace or macro call'), - (r'"', String, 'single-line string'), - (r'\{\{', String, 'lua'), - (r',', Punctuation), - default('#pop') + 'Lua opcode': [ + include('whitespace and macro uses'), + (r'\{\{', String, 'Lua'), + (r'\n', Text, '#pop') ], - 'lua': [ + 'Lua': [ (r'\}\}', String, '#pop'), (r'([^}]+)|\}(?!\})', using(LuaLexer)) ] @@ -313,7 +399,7 @@ class CsoundOrchestraLexer(CsoundLexer): class CsoundDocumentLexer(RegexLexer): """ - For `Csound <http://csound.github.io>`_ documents. + For `Csound <https://csound.github.io>`_ documents. .. versionadded:: 2.1 """ @@ -331,15 +417,18 @@ class CsoundDocumentLexer(RegexLexer): # be XML files. tokens = { 'root': [ - newline, (r'/[*](.|\n)*?[*]/', Comment.Multiline), - (r'[^<&;/]+', Text), + (r'(?:;|//).*$', Comment.Single), + (r'[^/;<]+|/(?!/)', Text), + (r'<\s*CsInstruments', Name.Tag, ('orchestra', 'tag')), (r'<\s*CsScore', Name.Tag, ('score', 'tag')), - (r'<\s*[hH][tT][mM][lL]', Name.Tag, ('HTML', 'tag')), + (r'<\s*[Hh][Tt][Mm][Ll]', Name.Tag, ('HTML', 'tag')), + (r'<\s*[\w:.-]+', Name.Tag, 'tag'), (r'<\s*/\s*[\w:.-]+\s*>', Name.Tag) ], + 'orchestra': [ (r'<\s*/\s*CsInstruments\s*>', Name.Tag, '#pop'), (r'(.|\n)+?(?=<\s*/\s*CsInstruments\s*>)', using(CsoundOrchestraLexer)) @@ -349,9 +438,10 @@ class CsoundDocumentLexer(RegexLexer): (r'(.|\n)+?(?=<\s*/\s*CsScore\s*>)', using(CsoundScoreLexer)) ], 'HTML': [ - (r'<\s*/\s*[hH][tT][mM][lL]\s*>', Name.Tag, '#pop'), - (r'(.|\n)+?(?=<\s*/\s*[hH][tT][mM][lL]\s*>)', using(HtmlLexer)) + (r'<\s*/\s*[Hh][Tt][Mm][Ll]\s*>', Name.Tag, '#pop'), + (r'(.|\n)+?(?=<\s*/\s*[Hh][Tt][Mm][Ll]\s*>)', using(HtmlLexer)) ], + 'tag': [ (r'\s+', Text), (r'[\w.:-]+\s*=', Name.Attribute, 'attr'), diff --git a/pygments/lexers/css.py b/pygments/lexers/css.py index 29d83707..ce97730e 100644 --- a/pygments/lexers/css.py +++ b/pygments/lexers/css.py @@ -125,7 +125,7 @@ _css_properties = ( 'wrap-flow', 'wrap-inside', 'wrap-through', 'writing-mode', 'z-index', ) -# List of keyword values obtained from: +# List of keyword values obtained from: # http://cssvalues.com/ _keyword_values = ( 'absolute', 'alias', 'all', 'all-petite-caps', 'all-scroll', @@ -263,7 +263,7 @@ _time_units = ( 's', 'ms', ) _all_units = _angle_units + _frequency_units + _length_units + \ - _resolution_units + _time_units + _resolution_units + _time_units class CssLexer(RegexLexer): @@ -322,16 +322,18 @@ class CssLexer(RegexLexer): include('urls'), (r'('+r'|'.join(_functional_notation_keyword_values)+r')(\()', bygroups(Name.Builtin, Punctuation), 'function-start'), - (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'), + (r'([a-zA-Z_][\w-]+)(\()', + bygroups(Name.Function, Punctuation), 'function-start'), (words(_keyword_values, suffix=r'\b'), Keyword.Constant), (words(_other_keyword_values, suffix=r'\b'), Keyword.Constant), (words(_color_keywords, suffix=r'\b'), Keyword.Constant), - (words(_css_properties, suffix=r'\b'), Keyword), # for transition-property etc. + # for transition-property etc. + (words(_css_properties, suffix=r'\b'), Keyword), (r'\!important', Comment.Preproc), (r'/\*(?:.|\n)*?\*/', Comment), include('numeric-values'), - + (r'[~^*!%&<>|+=@:./?-]+', Operator), (r'[\[\](),]+', Punctuation), (r'"(\\\\|\\"|[^"])*"', String.Double), @@ -351,7 +353,8 @@ class CssLexer(RegexLexer): # function-start may be entered recursively (r'(' + r'|'.join(_functional_notation_keyword_values) + r')(\()', bygroups(Name.Builtin, Punctuation), 'function-start'), - (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'), + (r'([a-zA-Z_][\w-]+)(\()', + bygroups(Name.Function, Punctuation), 'function-start'), (r'/\*(?:.|\n)*?\*/', Comment), include('numeric-values'), @@ -373,8 +376,8 @@ class CssLexer(RegexLexer): 'numeric-values': [ (r'\#[a-zA-Z0-9]{1,6}', Number.Hex), (r'[+\-]?[0-9]*[.][0-9]+', Number.Float, 'numeric-end'), - (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'), - ], + (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'), + ], 'numeric-end': [ (words(_all_units, suffix=r'\b'), Keyword.Type), (r'%', Keyword.Type), @@ -466,9 +469,9 @@ common_sass_tokens = { ], 'string-single': [ - (r"(\\.|#(?=[^\n{])|[^\n'#])+", String.Double), + (r"(\\.|#(?=[^\n{])|[^\n'#])+", String.Single), (r'#\{', String.Interpol, 'interpolation'), - (r"'", String.Double, '#pop'), + (r"'", String.Single, '#pop'), ], 'string-url': [ diff --git a/pygments/lexers/data.py b/pygments/lexers/data.py index 296366c2..a67d084e 100644 --- a/pygments/lexers/data.py +++ b/pygments/lexers/data.py @@ -205,7 +205,7 @@ class YamlLexer(ExtendedRegexLexer): bygroups(Text, Number), 'ignored-line'), ], - # the %YAG directive + # the %TAG directive 'tag-directive': [ # a tag handle and the corresponding prefix (r'([ ]+)(!|![\w-]*!)' @@ -218,7 +218,7 @@ class YamlLexer(ExtendedRegexLexer): 'indentation': [ # trailing whitespaces are ignored (r'[ ]*$', something(Text), '#pop:2'), - # whitespaces preceeding block collection indicators + # whitespaces preceding block collection indicators (r'[ ]+(?=[?:-](?:[ ]|$))', save_indent(Text)), # block collection indicators (r'[?:-](?=[ ]|$)', set_indent(Punctuation.Indicator)), @@ -232,6 +232,9 @@ class YamlLexer(ExtendedRegexLexer): (r'[ ]*(?=#|$)', something(Text), '#pop'), # whitespaces separating tokens (r'[ ]+', Text), + # key with colon + (r'([^,:?\[\]{}\n]+)(:)(?=[ ]|$)', + bygroups(Name.Tag, set_indent(Punctuation, implicit=True))), # tags, anchors and aliases, include('descriptors'), # block collections and scalars @@ -250,7 +253,7 @@ class YamlLexer(ExtendedRegexLexer): (r'!<[\w#;/?:@&=+$,.!~*\'()\[\]%-]+>', Keyword.Type), # a tag in the form '!', '!suffix' or '!handle!suffix' (r'!(?:[\w-]+!)?' - r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]+', Keyword.Type), + r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]*', Keyword.Type), # an anchor (r'&[\w-]+', Name.Label), # an alias @@ -308,6 +311,9 @@ class YamlLexer(ExtendedRegexLexer): # a flow mapping indicated by '{' and '}' 'flow-mapping': [ + # key with colon + (r'([^,:?\[\]{}\n]+)(:)(?=[ ]|$)', + bygroups(Name.Tag, Punctuation)), # include flow collection rules include('flow-collection'), # the closing indicator diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py index 4e2bc8ab..27ae77c5 100644 --- a/pygments/lexers/dotnet.py +++ b/pygments/lexers/dotnet.py @@ -58,7 +58,7 @@ class CSharpLexer(RegexLexer): # http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf levels = { - 'none': '@?[_a-zA-Z]\w*', + 'none': r'@?[_a-zA-Z]\w*', 'basic': ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' + '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc', 'Cf', 'Mn', 'Mc') + ']*'), @@ -171,7 +171,7 @@ class NemerleLexer(RegexLexer): # http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf levels = { - 'none': '@?[_a-zA-Z]\w*', + 'none': r'@?[_a-zA-Z]\w*', 'basic': ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' + '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc', 'Cf', 'Mn', 'Mc') + ']*'), @@ -352,13 +352,13 @@ class BooLexer(RegexLexer): ('[*/]', Comment.Multiline) ], 'funcname': [ - ('[a-zA-Z_]\w*', Name.Function, '#pop') + (r'[a-zA-Z_]\w*', Name.Function, '#pop') ], 'classname': [ - ('[a-zA-Z_]\w*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'namespace': [ - ('[a-zA-Z_][\w.]*', Name.Namespace, '#pop') + (r'[a-zA-Z_][\w.]*', Name.Namespace, '#pop') ] } @@ -413,7 +413,7 @@ class VbNetLexer(RegexLexer): 'Static', 'Step', 'Stop', 'SyncLock', 'Then', 'Throw', 'To', 'True', 'Try', 'TryCast', 'Wend', 'Using', 'When', 'While', 'Widening', 'With', 'WithEvents', 'WriteOnly'), - prefix='(?<!\.)', suffix=r'\b'), Keyword), + prefix=r'(?<!\.)', suffix=r'\b'), Keyword), (r'(?<!\.)End\b', Keyword, 'end'), (r'(?<!\.)(Dim|Const)\b', Keyword, 'dim'), (r'(?<!\.)(Function|Sub|Property)(\s+)', @@ -574,10 +574,10 @@ class FSharpLexer(RegexLexer): 'virtual', 'volatile', ] keyopts = [ - '!=', '#', '&&', '&', '\(', '\)', '\*', '\+', ',', '-\.', - '->', '-', '\.\.', '\.', '::', ':=', ':>', ':', ';;', ';', '<-', - '<\]', '<', '>\]', '>', '\?\?', '\?', '\[<', '\[\|', '\[', '\]', - '_', '`', '\{', '\|\]', '\|', '\}', '~', '<@@', '<@', '=', '@>', '@@>', + '!=', '#', '&&', '&', r'\(', r'\)', r'\*', r'\+', ',', r'-\.', + '->', '-', r'\.\.', r'\.', '::', ':=', ':>', ':', ';;', ';', '<-', + r'<\]', '<', r'>\]', '>', r'\?\?', r'\?', r'\[<', r'\[\|', r'\[', r'\]', + '_', '`', r'\{', r'\|\]', r'\|', r'\}', '~', '<@@', '<@', '=', '@>', '@@>', ] operators = r'[!$%&*+\./:<=>?@^|~-]' diff --git a/pygments/lexers/dsls.py b/pygments/lexers/dsls.py index a1426bd6..4451b480 100644 --- a/pygments/lexers/dsls.py +++ b/pygments/lexers/dsls.py @@ -66,7 +66,7 @@ class ProtoBufLexer(RegexLexer): (r'[+-=]', Operator), (r'([a-zA-Z_][\w.]*)([ \t]*)(=)', bygroups(Name.Attribute, Text, Operator)), - ('[a-zA-Z_][\w.]*', Name), + (r'[a-zA-Z_][\w.]*', Name), ], 'package': [ (r'[a-zA-Z_]\w*', Name.Namespace, '#pop'), @@ -300,7 +300,7 @@ class PuppetLexer(RegexLexer): ], 'names': [ - ('[a-zA-Z_]\w*', Name.Attribute), + (r'[a-zA-Z_]\w*', Name.Attribute), (r'(\$\S+)(\[)(\S+)(\])', bygroups(Name.Variable, Punctuation, String, Punctuation)), (r'\$\S+', Name.Variable), @@ -688,7 +688,7 @@ class CrmshLexer(RegexLexer): (r'([\w#$-]+)(?:(:)(%s))?(?![\w#$-])' % rsc_role_action, bygroups(Name, Punctuation, Operator.Word)), # punctuation - (r'(\\(?=\n)|[[\](){}/:@])', Punctuation), + (r'(\\(?=\n)|[\[\](){}/:@])', Punctuation), (r'\s+|\n', Whitespace), ], } diff --git a/pygments/lexers/dylan.py b/pygments/lexers/dylan.py index f61bb60d..30318f38 100644 --- a/pygments/lexers/dylan.py +++ b/pygments/lexers/dylan.py @@ -179,10 +179,10 @@ class DylanLexer(RegexLexer): (valid_name + ':', Keyword), # class names - (r'<' + valid_name + '>', Name.Class), + ('<' + valid_name + '>', Name.Class), # define variable forms. - (r'\*' + valid_name + '\*', Name.Variable.Global), + (r'\*' + valid_name + r'\*', Name.Variable.Global), # define constant forms. (r'\$' + valid_name, Name.Constant), @@ -260,7 +260,7 @@ class DylanConsoleLexer(Lexer): mimetypes = ['text/x-dylan-console'] _line_re = re.compile('.*?\n') - _prompt_re = re.compile('\?| ') + _prompt_re = re.compile(r'\?| ') def get_tokens_unprocessed(self, text): dylexer = DylanLexer(**self.options) diff --git a/pygments/lexers/elm.py b/pygments/lexers/elm.py index 0fa36367..22a10bd9 100644 --- a/pygments/lexers/elm.py +++ b/pygments/lexers/elm.py @@ -27,7 +27,7 @@ class ElmLexer(RegexLexer): filenames = ['*.elm'] mimetypes = ['text/x-elm'] - validName = r'[a-z_][a-zA-Z_\']*' + validName = r'[a-z_][a-zA-Z0-9_\']*' specialName = r'^main ' diff --git a/pygments/lexers/erlang.py b/pygments/lexers/erlang.py index 9e7f85c1..0d0d0798 100644 --- a/pygments/lexers/erlang.py +++ b/pygments/lexers/erlang.py @@ -344,7 +344,7 @@ class ElixirLexer(RegexLexer): op1_re = "|".join(re.escape(s) for s in OPERATORS1) ops_re = r'(?:%s|%s|%s)' % (op3_re, op2_re, op1_re) punctuation_re = "|".join(re.escape(s) for s in PUNCTUATION) - alnum = '\w' + alnum = r'\w' name_re = r'(?:\.\.\.|[a-z_]%s*[!?]?)' % alnum modname_re = r'[A-Z]%(alnum)s*(?:\.[A-Z]%(alnum)s*)*' % {'alnum': alnum} complex_name_re = r'(?:%s|%s|%s)' % (name_re, modname_re, ops_re) @@ -495,7 +495,7 @@ class ElixirConsoleLexer(Lexer): aliases = ['iex'] mimetypes = ['text/x-elixir-shellsession'] - _prompt_re = re.compile('(iex|\.{3})(\(\d+\))?> ') + _prompt_re = re.compile(r'(iex|\.{3})(\(\d+\))?> ') def get_tokens_unprocessed(self, text): exlexer = ElixirLexer(**self.options) diff --git a/pygments/lexers/fortran.py b/pygments/lexers/fortran.py index 1a611c9d..5165bac0 100644 --- a/pygments/lexers/fortran.py +++ b/pygments/lexers/fortran.py @@ -158,6 +158,7 @@ class FortranLexer(RegexLexer): (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer), (r'[+-]?\d*\.\d+([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), (r'[+-]?\d+\.\d*([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), + (r'[+-]?\d+(\.\d*)?[ed][-+]?\d+(_[a-z]\w+)?', Number.Float), ], } diff --git a/pygments/lexers/grammar_notation.py b/pygments/lexers/grammar_notation.py index 076249d3..bc715ffa 100644 --- a/pygments/lexers/grammar_notation.py +++ b/pygments/lexers/grammar_notation.py @@ -158,7 +158,7 @@ class JsgfLexer(RegexLexer): (r'//.*', Comment.Single), ], 'non-comments': [ - ('\A#JSGF[^;]*', Comment.Preproc), + (r'\A#JSGF[^;]*', Comment.Preproc), (r'\s+', Text), (r';', Punctuation), (r'[=|()\[\]*+]', Operator), diff --git a/pygments/lexers/graphics.py b/pygments/lexers/graphics.py index c8af9f99..30ab2cbc 100644 --- a/pygments/lexers/graphics.py +++ b/pygments/lexers/graphics.py @@ -15,7 +15,7 @@ from pygments.token import Text, Comment, Operator, Keyword, Name, \ Number, Punctuation, String __all__ = ['GLShaderLexer', 'PostScriptLexer', 'AsymptoteLexer', 'GnuplotLexer', - 'PovrayLexer'] + 'PovrayLexer', 'HLSLShaderLexer'] class GLShaderLexer(RegexLexer): @@ -46,28 +46,102 @@ class GLShaderLexer(RegexLexer): (r'0[0-7]*', Number.Oct), (r'[1-9][0-9]*', Number.Integer), (words(( - 'attribute', 'const', 'uniform', 'varying', 'centroid', 'break', - 'continue', 'do', 'for', 'while', 'if', 'else', 'in', 'out', - 'inout', 'float', 'int', 'void', 'bool', 'true', 'false', - 'invariant', 'discard', 'return', 'mat2', 'mat3' 'mat4', - 'mat2x2', 'mat3x2', 'mat4x2', 'mat2x3', 'mat3x3', 'mat4x3', - 'mat2x4', 'mat3x4', 'mat4x4', 'vec2', 'vec3', 'vec4', - 'ivec2', 'ivec3', 'ivec4', 'bvec2', 'bvec3', 'bvec4', - 'sampler1D', 'sampler2D', 'sampler3D' 'samplerCube', - 'sampler1DShadow', 'sampler2DShadow', 'struct'), + # Storage qualifiers + 'attribute', 'const', 'uniform', 'varying', + 'buffer', 'shared', 'in', 'out', + # Layout qualifiers + 'layout', + # Interpolation qualifiers + 'flat', 'smooth', 'noperspective', + # Auxiliary qualifiers + 'centroid', 'sample', 'patch', + # Parameter qualifiers. Some double as Storage qualifiers + 'inout', + # Precision qualifiers + 'lowp', 'mediump', 'highp', 'precision', + # Invariance qualifiers + 'invariant', + # Precise qualifiers + 'precise', + # Memory qualifiers + 'coherent', 'volatile', 'restrict', 'readonly', 'writeonly', + # Statements + 'break', 'continue', 'do', 'for', 'while', 'switch', + 'case', 'default', 'if', 'else', 'subroutine', + 'discard', 'return', 'struct'), prefix=r'\b', suffix=r'\b'), Keyword), (words(( - 'asm', 'class', 'union', 'enum', 'typedef', 'template', 'this', - 'packed', 'goto', 'switch', 'default', 'inline', 'noinline', - 'volatile', 'public', 'static', 'extern', 'external', 'interface', - 'long', 'short', 'double', 'half', 'fixed', 'unsigned', 'lowp', - 'mediump', 'highp', 'precision', 'input', 'output', - 'hvec2', 'hvec3', 'hvec4', 'dvec2', 'dvec3', 'dvec4', - 'fvec2', 'fvec3', 'fvec4', 'sampler2DRect', 'sampler3DRect', - 'sampler2DRectShadow', 'sizeof', 'cast', 'namespace', 'using'), + # Boolean values + 'true', 'false'), prefix=r'\b', suffix=r'\b'), - Keyword), # future use + Keyword.Constant), + (words(( + # Miscellaneous types + 'void', 'atomic_uint', + # Floating-point scalars and vectors + 'float', 'vec2', 'vec3', 'vec4', + 'double', 'dvec2', 'dvec3', 'dvec4', + # Integer scalars and vectors + 'int', 'ivec2', 'ivec3', 'ivec4', + 'uint', 'uvec2', 'uvec3', 'uvec4', + # Boolean scalars and vectors + 'bool', 'bvec2', 'bvec3', 'bvec4', + # Matrices + 'mat2', 'mat3', 'mat4', 'dmat2', 'dmat3', 'dmat4', + 'mat2x2', 'mat2x3', 'mat2x4', 'dmat2x2', 'dmat2x3', 'dmat2x4', + 'mat3x2', 'mat3x3', 'mat3x4', 'dmat3x2', 'dmat3x3', + 'dmat3x4', 'mat4x2', 'mat4x3', 'mat4x4', 'dmat4x2', 'dmat4x3', 'dmat4x4', + # Floating-point samplers + 'sampler1D', 'sampler2D', 'sampler3D', 'samplerCube', + 'sampler1DArray', 'sampler2DArray', 'samplerCubeArray', + 'sampler2DRect', 'samplerBuffer', + 'sampler2DMS', 'sampler2DMSArray', + # Shadow samplers + 'sampler1DShadow', 'sampler2DShadow', 'samplerCubeShadow', + 'sampler1DArrayShadow', 'sampler2DArrayShadow', + 'samplerCubeArrayShadow', 'sampler2DRectShadow', + # Signed integer samplers + 'isampler1D', 'isampler2D', 'isampler3D', 'isamplerCube', + 'isampler1DArray', 'isampler2DArray', 'isamplerCubeArray', + 'isampler2DRect', 'isamplerBuffer', + 'isampler2DMS', 'isampler2DMSArray', + # Unsigned integer samplers + 'usampler1D', 'usampler2D', 'usampler3D', 'usamplerCube', + 'usampler1DArray', 'usampler2DArray', 'usamplerCubeArray', + 'usampler2DRect', 'usamplerBuffer', + 'usampler2DMS', 'usampler2DMSArray', + # Floating-point image types + 'image1D', 'image2D', 'image3D', 'imageCube', + 'image1DArray', 'image2DArray', 'imageCubeArray', + 'image2DRect', 'imageBuffer', + 'image2DMS', 'image2DMSArray', + # Signed integer image types + 'iimage1D', 'iimage2D', 'iimage3D', 'iimageCube', + 'iimage1DArray', 'iimage2DArray', 'iimageCubeArray', + 'iimage2DRect', 'iimageBuffer', + 'iimage2DMS', 'iimage2DMSArray', + # Unsigned integer image types + 'uimage1D', 'uimage2D', 'uimage3D', 'uimageCube', + 'uimage1DArray', 'uimage2DArray', 'uimageCubeArray', + 'uimage2DRect', 'uimageBuffer', + 'uimage2DMS', 'uimage2DMSArray'), + prefix=r'\b', suffix=r'\b'), + Keyword.Type), + (words(( + # Reserved for future use. + 'common', 'partition', 'active', 'asm', 'class', + 'union', 'enum', 'typedef', 'template', 'this', + 'resource', 'goto', 'inline', 'noinline', 'public', + 'static', 'extern', 'external', 'interface', 'long', + 'short', 'half', 'fixed', 'unsigned', 'superp', 'input', + 'output', 'hvec2', 'hvec3', 'hvec4', 'fvec2', 'fvec3', + 'fvec4', 'sampler3DRect', 'filter', 'sizeof', 'cast', + 'namespace', 'using'), + prefix=r'\b', suffix=r'\b'), + Keyword.Reserved), + # All names beginning with "gl_" are reserved. + (r'gl_\w*', Name.Builtin), (r'[a-zA-Z_]\w*', Name), (r'\.', Punctuation), (r'\s+', Text), @@ -75,6 +149,160 @@ class GLShaderLexer(RegexLexer): } +class HLSLShaderLexer(RegexLexer): + """ + HLSL (Microsoft Direct3D Shader) lexer. + + .. versionadded:: 2.3 + """ + name = 'HLSL' + aliases = ['hlsl'] + filenames = ['*.hlsl', '*.hlsli'] + mimetypes = ['text/x-hlsl'] + + tokens = { + 'root': [ + (r'^#.*', Comment.Preproc), + (r'//.*', Comment.Single), + (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), + (r'\+|-|~|!=?|\*|/|%|<<|>>|<=?|>=?|==?|&&?|\^|\|\|?', + Operator), + (r'[?:]', Operator), # quick hack for ternary + (r'\bdefined\b', Operator), + (r'[;{}(),.\[\]]', Punctuation), + # FIXME when e is present, no decimal point needed + (r'[+-]?\d*\.\d+([eE][-+]?\d+)?f?', Number.Float), + (r'[+-]?\d+\.\d*([eE][-+]?\d+)?f?', Number.Float), + (r'0[xX][0-9a-fA-F]*', Number.Hex), + (r'0[0-7]*', Number.Oct), + (r'[1-9][0-9]*', Number.Integer), + (r'"', String, 'string'), + (words(( + 'asm','asm_fragment','break','case','cbuffer','centroid','class', + 'column_major','compile','compile_fragment','const','continue', + 'default','discard','do','else','export','extern','for','fxgroup', + 'globallycoherent','groupshared','if','in','inline','inout', + 'interface','line','lineadj','linear','namespace','nointerpolation', + 'noperspective','NULL','out','packoffset','pass','pixelfragment', + 'point','precise','return','register','row_major','sample', + 'sampler','shared','stateblock','stateblock_state','static', + 'struct','switch','tbuffer','technique','technique10', + 'technique11','texture','typedef','triangle','triangleadj', + 'uniform','vertexfragment','volatile','while'), + prefix=r'\b', suffix=r'\b'), + Keyword), + (words(('true','false'), prefix=r'\b', suffix=r'\b'), + Keyword.Constant), + (words(( + 'auto','catch','char','const_cast','delete','dynamic_cast','enum', + 'explicit','friend','goto','long','mutable','new','operator', + 'private','protected','public','reinterpret_cast','short','signed', + 'sizeof','static_cast','template','this','throw','try','typename', + 'union','unsigned','using','virtual'), + prefix=r'\b', suffix=r'\b'), + Keyword.Reserved), + (words(( + 'dword','matrix','snorm','string','unorm','unsigned','void','vector', + 'BlendState','Buffer','ByteAddressBuffer','ComputeShader', + 'DepthStencilState','DepthStencilView','DomainShader', + 'GeometryShader','HullShader','InputPatch','LineStream', + 'OutputPatch','PixelShader','PointStream','RasterizerState', + 'RenderTargetView','RasterizerOrderedBuffer', + 'RasterizerOrderedByteAddressBuffer', + 'RasterizerOrderedStructuredBuffer','RasterizerOrderedTexture1D', + 'RasterizerOrderedTexture1DArray','RasterizerOrderedTexture2D', + 'RasterizerOrderedTexture2DArray','RasterizerOrderedTexture3D', + 'RWBuffer','RWByteAddressBuffer','RWStructuredBuffer', + 'RWTexture1D','RWTexture1DArray','RWTexture2D','RWTexture2DArray', + 'RWTexture3D','SamplerState','SamplerComparisonState', + 'StructuredBuffer','Texture1D','Texture1DArray','Texture2D', + 'Texture2DArray','Texture2DMS','Texture2DMSArray','Texture3D', + 'TextureCube','TextureCubeArray','TriangleStream','VertexShader'), + prefix=r'\b', suffix=r'\b'), + Keyword.Type), + (words(( + 'bool','double','float','int','half','min16float','min10float', + 'min16int','min12int','min16uint','uint'), + prefix=r'\b', suffix=r'([1-4](x[1-4])?)?\b'), + Keyword.Type), # vector and matrix types + (words(( + 'abort','abs','acos','all','AllMemoryBarrier', + 'AllMemoryBarrierWithGroupSync','any','AppendStructuredBuffer', + 'asdouble','asfloat','asin','asint','asuint','asuint','atan', + 'atan2','ceil','CheckAccessFullyMapped','clamp','clip', + 'CompileShader','ConsumeStructuredBuffer','cos','cosh','countbits', + 'cross','D3DCOLORtoUBYTE4','ddx','ddx_coarse','ddx_fine','ddy', + 'ddy_coarse','ddy_fine','degrees','determinant', + 'DeviceMemoryBarrier','DeviceMemoryBarrierWithGroupSync','distance', + 'dot','dst','errorf','EvaluateAttributeAtCentroid', + 'EvaluateAttributeAtSample','EvaluateAttributeSnapped','exp', + 'exp2','f16tof32','f32tof16','faceforward','firstbithigh', + 'firstbitlow','floor','fma','fmod','frac','frexp','fwidth', + 'GetRenderTargetSampleCount','GetRenderTargetSamplePosition', + 'GlobalOrderedCountIncrement','GroupMemoryBarrier', + 'GroupMemoryBarrierWithGroupSync','InterlockedAdd','InterlockedAnd', + 'InterlockedCompareExchange','InterlockedCompareStore', + 'InterlockedExchange','InterlockedMax','InterlockedMin', + 'InterlockedOr','InterlockedXor','isfinite','isinf','isnan', + 'ldexp','length','lerp','lit','log','log10','log2','mad','max', + 'min','modf','msad4','mul','noise','normalize','pow','printf', + 'Process2DQuadTessFactorsAvg','Process2DQuadTessFactorsMax', + 'Process2DQuadTessFactorsMin','ProcessIsolineTessFactors', + 'ProcessQuadTessFactorsAvg','ProcessQuadTessFactorsMax', + 'ProcessQuadTessFactorsMin','ProcessTriTessFactorsAvg', + 'ProcessTriTessFactorsMax','ProcessTriTessFactorsMin', + 'QuadReadLaneAt','QuadSwapX','QuadSwapY','radians','rcp', + 'reflect','refract','reversebits','round','rsqrt','saturate', + 'sign','sin','sincos','sinh','smoothstep','sqrt','step','tan', + 'tanh','tex1D','tex1D','tex1Dbias','tex1Dgrad','tex1Dlod', + 'tex1Dproj','tex2D','tex2D','tex2Dbias','tex2Dgrad','tex2Dlod', + 'tex2Dproj','tex3D','tex3D','tex3Dbias','tex3Dgrad','tex3Dlod', + 'tex3Dproj','texCUBE','texCUBE','texCUBEbias','texCUBEgrad', + 'texCUBElod','texCUBEproj','transpose','trunc','WaveAllBitAnd', + 'WaveAllMax','WaveAllMin','WaveAllBitOr','WaveAllBitXor', + 'WaveAllEqual','WaveAllProduct','WaveAllSum','WaveAllTrue', + 'WaveAnyTrue','WaveBallot','WaveGetLaneCount','WaveGetLaneIndex', + 'WaveGetOrderedIndex','WaveIsHelperLane','WaveOnce', + 'WavePrefixProduct','WavePrefixSum','WaveReadFirstLane', + 'WaveReadLaneAt'), + prefix=r'\b', suffix=r'\b'), + Name.Builtin), # built-in functions + (words(( + 'SV_ClipDistance','SV_ClipDistance0','SV_ClipDistance1', + 'SV_Culldistance','SV_CullDistance0','SV_CullDistance1', + 'SV_Coverage','SV_Depth','SV_DepthGreaterEqual', + 'SV_DepthLessEqual','SV_DispatchThreadID','SV_DomainLocation', + 'SV_GroupID','SV_GroupIndex','SV_GroupThreadID','SV_GSInstanceID', + 'SV_InnerCoverage','SV_InsideTessFactor','SV_InstanceID', + 'SV_IsFrontFace','SV_OutputControlPointID','SV_Position', + 'SV_PrimitiveID','SV_RenderTargetArrayIndex','SV_SampleIndex', + 'SV_StencilRef','SV_TessFactor','SV_VertexID', + 'SV_ViewportArrayIndex'), + prefix=r'\b', suffix=r'\b'), + Name.Decorator), # system-value semantics + (r'\bSV_Target[0-7]?\b', Name.Decorator), + (words(( + 'allow_uav_condition','branch','call','domain','earlydepthstencil', + 'fastopt','flatten','forcecase','instance','loop','maxtessfactor', + 'numthreads','outputcontrolpoints','outputtopology','partitioning', + 'patchconstantfunc','unroll'), + prefix=r'\b', suffix=r'\b'), + Name.Decorator), # attributes + (r'[a-zA-Z_]\w*', Name), + (r'\\$', Comment.Preproc), # backslash at end of line -- usually macro continuation + (r'\s+', Text), + ], + 'string': [ + (r'"', String, '#pop'), + (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|' + r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape), + (r'[^\\"\n]+', String), # all other characters + (r'\\\n', String), # line continuation + (r'\\', String), # stray backslash + ], + } + + class PostScriptLexer(RegexLexer): """ Lexer for PostScript files. @@ -233,8 +461,8 @@ class AsymptoteLexer(RegexLexer): r'bounds|coord|frame|guide|horner|int|linefit|marginT|pair|pen|' r'picture|position|real|revolution|slice|splitface|ticksgridT|' r'tickvalues|tree|triple|vertex|void)\b', Keyword.Type), - ('[a-zA-Z_]\w*:(?!:)', Name.Label), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*:(?!:)', Name.Label), + (r'[a-zA-Z_]\w*', Name), ], 'root': [ include('whitespace'), @@ -334,9 +562,9 @@ class GnuplotLexer(RegexLexer): (_shortened_many('pwd$', 're$read', 'res$et', 'scr$eendump', 'she$ll', 'test$'), Keyword, 'noargs'), - ('([a-zA-Z_]\w*)(\s*)(=)', + (r'([a-zA-Z_]\w*)(\s*)(=)', bygroups(Name.Variable, Text, Operator), 'genericargs'), - ('([a-zA-Z_]\w*)(\s*\(.*?\)\s*)(=)', + (r'([a-zA-Z_]\w*)(\s*\(.*?\)\s*)(=)', bygroups(Name.Function, Text, Operator), 'genericargs'), (r'@[a-zA-Z_]\w*', Name.Constant), # macros (r';', Keyword), @@ -382,7 +610,7 @@ class GnuplotLexer(RegexLexer): (r'(\d+\.\d*|\.\d+)', Number.Float), (r'-?\d+', Number.Integer), ('[,.~!%^&*+=|?:<>/-]', Operator), - ('[{}()\[\]]', Punctuation), + (r'[{}()\[\]]', Punctuation), (r'(eq|ne)\b', Operator.Word), (r'([a-zA-Z_]\w*)(\s*)(\()', bygroups(Name.Function, Text, Punctuation)), diff --git a/pygments/lexers/haskell.py b/pygments/lexers/haskell.py index b4624124..88d4a4df 100644 --- a/pygments/lexers/haskell.py +++ b/pygments/lexers/haskell.py @@ -680,10 +680,10 @@ class KokaLexer(RegexLexer): symbols = r'[$%&*+@!/\\^~=.:\-?|<>]+' # symbol boundary: an operator keyword should not be followed by any of these - sboundary = '(?!'+symbols+')' + sboundary = '(?!' + symbols + ')' # name boundary: a keyword should not be followed by any of these - boundary = '(?![\w/])' + boundary = r'(?![\w/])' # koka token abstractions tokenType = Name.Attribute diff --git a/pygments/lexers/haxe.py b/pygments/lexers/haxe.py index 6f5c3599..364ad344 100644 --- a/pygments/lexers/haxe.py +++ b/pygments/lexers/haxe.py @@ -43,7 +43,7 @@ class HaxeLexer(ExtendedRegexLexer): typeid = r'_*[A-Z]\w*' # combined ident and dollar and idtype - ident = r'(?:_*[a-z]\w*|_+[0-9]\w*|' + typeid + '|_+|\$\w+)' + ident = r'(?:_*[a-z]\w*|_+[0-9]\w*|' + typeid + r'|_+|\$\w+)' binop = (r'(?:%=|&=|\|=|\^=|\+=|\-=|\*=|/=|<<=|>\s*>\s*=|>\s*>\s*>\s*=|==|' r'!=|<=|>\s*=|&&|\|\||<<|>>>|>\s*>|\.\.\.|<|>|%|&|\||\^|\+|\*|' @@ -182,7 +182,7 @@ class HaxeLexer(ExtendedRegexLexer): (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float), (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float), (r'[0-9]+\.[0-9]+', Number.Float), - (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float), + (r'[0-9]+\.(?!' + ident + r'|\.\.)', Number.Float), # Int (r'0x[0-9a-fA-F]+', Number.Hex), @@ -219,7 +219,7 @@ class HaxeLexer(ExtendedRegexLexer): (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), (r'[0-9]+\.[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), - (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, ('#pop', 'preproc-expr-chain')), + (r'[0-9]+\.(?!' + ident + r'|\.\.)', Number.Float, ('#pop', 'preproc-expr-chain')), # Int (r'0x[0-9a-fA-F]+', Number.Hex, ('#pop', 'preproc-expr-chain')), @@ -456,7 +456,7 @@ class HaxeLexer(ExtendedRegexLexer): (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'expr-chain')), (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'expr-chain')), (r'[0-9]+\.[0-9]+', Number.Float, ('#pop', 'expr-chain')), - (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, ('#pop', 'expr-chain')), + (r'[0-9]+\.(?!' + ident + r'|\.\.)', Number.Float, ('#pop', 'expr-chain')), # Int (r'0x[0-9a-fA-F]+', Number.Hex, ('#pop', 'expr-chain')), @@ -711,7 +711,7 @@ class HaxeLexer(ExtendedRegexLexer): (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float, '#pop'), (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float, '#pop'), (r'[0-9]+\.[0-9]+', Number.Float, '#pop'), - (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, '#pop'), + (r'[0-9]+\.(?!' + ident + r'|\.\.)', Number.Float, '#pop'), # Int (r'0x[0-9a-fA-F]+', Number.Hex, '#pop'), diff --git a/pygments/lexers/html.py b/pygments/lexers/html.py index 73f020fa..091379ce 100644 --- a/pygments/lexers/html.py +++ b/pygments/lexers/html.py @@ -220,7 +220,7 @@ class XmlLexer(RegexLexer): (r'/?\s*>', Name.Tag, '#pop'), ], 'attr': [ - ('\s+', Text), + (r'\s+', Text), ('".*?"', String, '#pop'), ("'.*?'", String, '#pop'), (r'[^\s>]+', String, '#pop'), @@ -313,7 +313,7 @@ class HamlLexer(ExtendedRegexLexer): include('css'), (r'%[\w:-]+', Name.Tag, 'tag'), (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'), - (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)', + (r'(/)(\[' + _dot + r'*?\])(' + _dot + r'*\n)', bygroups(Comment, Comment.Special, Comment), '#pop'), (r'/' + _dot + r'*\n', _starts_block(Comment, 'html-comment-block'), @@ -330,8 +330,8 @@ class HamlLexer(ExtendedRegexLexer): 'tag': [ include('css'), - (r'\{(,\n|' + _dot + ')*?\}', using(RubyLexer)), - (r'\[' + _dot + '*?\]', using(RubyLexer)), + (r'\{(,\n|' + _dot + r')*?\}', using(RubyLexer)), + (r'\[' + _dot + r'*?\]', using(RubyLexer)), (r'\(', Text, 'html-attributes'), (r'/[ \t]*\n', Punctuation, '#pop:2'), (r'[<>]{1,2}(?=[ \t=])', Punctuation), @@ -340,7 +340,7 @@ class HamlLexer(ExtendedRegexLexer): 'plain': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Text), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(RubyLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -373,7 +373,7 @@ class HamlLexer(ExtendedRegexLexer): 'filter-block': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Name.Decorator), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(RubyLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -422,7 +422,7 @@ class ScamlLexer(ExtendedRegexLexer): include('css'), (r'%[\w:-]+', Name.Tag, 'tag'), (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'), - (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)', + (r'(/)(\[' + _dot + r'*?\])(' + _dot + r'*\n)', bygroups(Comment, Comment.Special, Comment), '#pop'), (r'/' + _dot + r'*\n', _starts_block(Comment, 'html-comment-block'), @@ -442,8 +442,8 @@ class ScamlLexer(ExtendedRegexLexer): 'tag': [ include('css'), - (r'\{(,\n|' + _dot + ')*?\}', using(ScalaLexer)), - (r'\[' + _dot + '*?\]', using(ScalaLexer)), + (r'\{(,\n|' + _dot + r')*?\}', using(ScalaLexer)), + (r'\[' + _dot + r'*?\]', using(ScalaLexer)), (r'\(', Text, 'html-attributes'), (r'/[ \t]*\n', Punctuation, '#pop:2'), (r'[<>]{1,2}(?=[ \t=])', Punctuation), @@ -452,7 +452,7 @@ class ScamlLexer(ExtendedRegexLexer): 'plain': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Text), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(ScalaLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -485,7 +485,7 @@ class ScamlLexer(ExtendedRegexLexer): 'filter-block': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Name.Decorator), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(ScalaLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -530,7 +530,7 @@ class PugLexer(ExtendedRegexLexer): 'content': [ include('css'), (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'), - (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)', + (r'(/)(\[' + _dot + r'*?\])(' + _dot + r'*\n)', bygroups(Comment, Comment.Special, Comment), '#pop'), (r'/' + _dot + r'*\n', _starts_block(Comment, 'html-comment-block'), @@ -551,8 +551,8 @@ class PugLexer(ExtendedRegexLexer): 'tag': [ include('css'), - (r'\{(,\n|' + _dot + ')*?\}', using(ScalaLexer)), - (r'\[' + _dot + '*?\]', using(ScalaLexer)), + (r'\{(,\n|' + _dot + r')*?\}', using(ScalaLexer)), + (r'\[' + _dot + r'*?\]', using(ScalaLexer)), (r'\(', Text, 'html-attributes'), (r'/[ \t]*\n', Punctuation, '#pop:2'), (r'[<>]{1,2}(?=[ \t=])', Punctuation), @@ -561,7 +561,7 @@ class PugLexer(ExtendedRegexLexer): 'plain': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Text), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(ScalaLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -594,7 +594,7 @@ class PugLexer(ExtendedRegexLexer): 'filter-block': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Name.Decorator), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(ScalaLexer), String.Interpol)), (r'\n', Text, 'root'), ], diff --git a/pygments/lexers/idl.py b/pygments/lexers/idl.py index 2fc39318..87cafe6a 100644 --- a/pygments/lexers/idl.py +++ b/pygments/lexers/idl.py @@ -53,7 +53,7 @@ class IDLLexer(RegexLexer): 'broyden', 'butterworth', 'bytarr', 'byte', 'byteorder', 'bytscl', 'caldat', 'calendar', 'call_external', 'call_function', 'call_method', 'call_procedure', 'canny', - 'catch', 'cd', 'cdf_\w*', 'ceil', 'chebyshev', + 'catch', 'cd', r'cdf_\w*', 'ceil', 'chebyshev', 'check_math', 'chisqr_cvf', 'chisqr_pdf', 'choldc', 'cholsol', 'cindgen', 'cir_3pnt', 'close', 'cluster', 'cluster_tree', 'clust_wts', @@ -87,7 +87,7 @@ class IDLLexer(RegexLexer): 'dlm_load', 'dlm_register', 'doc_library', 'double', 'draw_roi', 'edge_dog', 'efont', 'eigenql', 'eigenvec', 'ellipse', 'elmhes', 'emboss', 'empty', 'enable_sysrtn', - 'eof', 'eos_\w*', 'erase', 'erf', 'erfc', 'erfcx', + 'eof', r'eos_\w*', 'erase', 'erf', 'erfc', 'erfcx', 'erode', 'errorplot', 'errplot', 'estimator_filter', 'execute', 'exit', 'exp', 'expand', 'expand_path', 'expint', 'extrac', 'extract_slice', 'factorial', 'fft', 'filepath', @@ -104,11 +104,11 @@ class IDLLexer(RegexLexer): 'gauss_cvf', 'gauss_pdf', 'gauss_smooth', 'getenv', 'getwindows', 'get_drive_list', 'get_dxf_objects', 'get_kbrd', 'get_login_info', 'get_lun', 'get_screen_size', - 'greg2jul', 'grib_\w*', 'grid3', 'griddata', + 'greg2jul', r'grib_\w*', 'grid3', 'griddata', 'grid_input', 'grid_tps', 'gs_iter', - 'h5[adfgirst]_\w*', 'h5_browser', 'h5_close', + r'h5[adfgirst]_\w*', 'h5_browser', 'h5_close', 'h5_create', 'h5_get_libversion', 'h5_open', 'h5_parse', - 'hanning', 'hash', 'hdf_\w*', 'heap_free', + 'hanning', 'hash', r'hdf_\w*', 'heap_free', 'heap_gc', 'heap_nosave', 'heap_refcount', 'heap_save', 'help', 'hilbert', 'histogram', 'hist_2d', 'hist_equal', 'hls', 'hough', 'hqr', 'hsv', 'h_eq_ct', 'h_eq_int', @@ -156,7 +156,7 @@ class IDLLexer(RegexLexer): 'modifyct', 'moment', 'morph_close', 'morph_distance', 'morph_gradient', 'morph_hitormiss', 'morph_open', 'morph_thin', 'morph_tophat', 'multi', 'm_correlate', - 'ncdf_\w*', 'newton', 'noise_hurl', 'noise_pick', + r'ncdf_\w*', 'newton', 'noise_hurl', 'noise_pick', 'noise_scatter', 'noise_slur', 'norm', 'n_elements', 'n_params', 'n_tags', 'objarr', 'obj_class', 'obj_destroy', 'obj_hasmethod', 'obj_isa', 'obj_new', 'obj_valid', diff --git a/pygments/lexers/inferno.py b/pygments/lexers/inferno.py index 5fc5a0ba..0d68856d 100644 --- a/pygments/lexers/inferno.py +++ b/pygments/lexers/inferno.py @@ -64,7 +64,7 @@ class LimboLexer(RegexLexer): (r'(byte|int|big|real|string|array|chan|list|adt' r'|fn|ref|of|module|self|type)\b', Keyword.Type), (r'(con|iota|nil)\b', Keyword.Constant), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'statement' : [ include('whitespace'), diff --git a/pygments/lexers/int_fiction.py b/pygments/lexers/int_fiction.py index f280a56d..57ace259 100644 --- a/pygments/lexers/int_fiction.py +++ b/pygments/lexers/int_fiction.py @@ -911,7 +911,7 @@ class Tads3Lexer(RegexLexer): 'block?/root': [ (r'\{', Punctuation, ('#pop', 'block')), include('whitespace'), - (r'(?=[[\'"<(:])', Text, # It might be a VerbRule macro. + (r'(?=[\[\'"<(:])', Text, # It might be a VerbRule macro. ('#pop', 'object-body/no-braces', 'grammar', 'grammar-rules')), # It might be a macro like DefineAction. default(('#pop', 'object-body/no-braces')) diff --git a/pygments/lexers/iolang.py b/pygments/lexers/iolang.py index bbc17faf..26f44e27 100644 --- a/pygments/lexers/iolang.py +++ b/pygments/lexers/iolang.py @@ -49,7 +49,7 @@ class IoLexer(RegexLexer): # names (r'(Object|list|List|Map|args|Sequence|Coroutine|File)\b', Name.Builtin), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), # numbers (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), (r'\d+', Number.Integer) diff --git a/pygments/lexers/j.py b/pygments/lexers/j.py index 434964fe..46037820 100644 --- a/pygments/lexers/j.py +++ b/pygments/lexers/j.py @@ -52,13 +52,13 @@ class JLexer(RegexLexer): Name.Function, 'explicitDefinition'), # Flow Control - (words(('for_', 'goto_', 'label_'), suffix=validName+'\.'), Name.Label), + (words(('for_', 'goto_', 'label_'), suffix=validName+r'\.'), Name.Label), (words(( 'assert', 'break', 'case', 'catch', 'catchd', 'catcht', 'continue', 'do', 'else', 'elseif', 'end', 'fcase', 'for', 'if', 'return', 'select', 'throw', 'try', 'while', 'whilst', - ), suffix='\.'), Name.Label), + ), suffix=r'\.'), Name.Label), # Variable Names (validName, Name.Variable), diff --git a/pygments/lexers/javascript.py b/pygments/lexers/javascript.py index 862535c9..840ce9ed 100644 --- a/pygments/lexers/javascript.py +++ b/pygments/lexers/javascript.py @@ -535,8 +535,8 @@ class TypeScriptLexer(RegexLexer): } def analyse_text(text): - if re.search('^(import.+(from\s+)?["\']|' - '(export\s*)?(interface|class|function)\s+)', + if re.search(r'^(import.+(from\s+)?["\']|' + r'(export\s*)?(interface|class|function)\s+)', text, re.MULTILINE): return 1.0 @@ -1015,7 +1015,7 @@ class ObjectiveJLexer(RegexLexer): } def analyse_text(text): - if re.search('^\s*@import\s+[<"]', text, re.MULTILINE): + if re.search(r'^\s*@import\s+[<"]', text, re.MULTILINE): # special directive found in most Objective-J files return True return False @@ -1500,8 +1500,10 @@ class JuttleLexer(RegexLexer): (r'^(?=\s|/)', Text, 'slashstartsregex'), include('commentsandwhitespace'), (r':\d{2}:\d{2}:\d{2}(\.\d*)?:', String.Moment), - (r':(now|beginning|end|forever|yesterday|today|tomorrow|(\d+(\.\d*)?|\.\d+)(ms|[smhdwMy])?):', String.Moment), - (r':\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d*)?)?(Z|[+-]\d{2}:\d{2}|[+-]\d{4})?:', String.Moment), + (r':(now|beginning|end|forever|yesterday|today|tomorrow|' + r'(\d+(\.\d*)?|\.\d+)(ms|[smhdwMy])?):', String.Moment), + (r':\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d*)?)?' + r'(Z|[+-]\d{2}:\d{2}|[+-]\d{4})?:', String.Moment), (r':((\d+(\.\d*)?|\.\d+)[ ]+)?(millisecond|second|minute|hour|day|week|month|year)[s]?' r'(([ ]+and[ ]+(\d+[ ]+)?(millisecond|second|minute|hour|day|week|month|year)[s]?)' r'|[ ]+(ago|from[ ]+now))*:', String.Moment), diff --git a/pygments/lexers/julia.py b/pygments/lexers/julia.py index 67453aba..69d14751 100644 --- a/pygments/lexers/julia.py +++ b/pygments/lexers/julia.py @@ -146,7 +146,7 @@ class JuliaLexer(RegexLexer): (words([ # prec-assignment u'=', u':=', u'+=', u'-=', u'*=', u'/=', u'//=', u'.//=', u'.*=', u'./=', - u'\=', u'.\=', u'^=', u'.^=', u'÷=', u'.÷=', u'%=', u'.%=', u'|=', u'&=', + u'\\=', u'.\\=', u'^=', u'.^=', u'÷=', u'.÷=', u'%=', u'.%=', u'|=', u'&=', u'$=', u'=>', u'<<=', u'>>=', u'>>>=', u'~', u'.+=', u'.-=', # prec-conditional u'?', @@ -181,7 +181,7 @@ class JuliaLexer(RegexLexer): # prec-dot u'.', # unary op - u'+', u'-', u'!', u'~', u'√', u'∛', u'∜' + u'+', u'-', u'!', u'√', u'∛', u'∜' ]), Operator), # chars diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index f4392839..6fd33e9d 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -257,7 +257,7 @@ class ScalaLexer(RegexLexer): u'\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\uff21-\uff3a]') idrest = u'%s(?:%s|[0-9])*(?:(?<=_)%s)?' % (letter, letter, op) - letter_letter_digit = u'%s(?:%s|\d)*' % (letter, letter) + letter_letter_digit = u'%s(?:%s|\\d)*' % (letter, letter) tokens = { 'root': [ @@ -689,7 +689,7 @@ class IokeLexer(RegexLexer): # functions (u'(generateMatchMethod|aliasMethod|\u03bb|\u028E|fnx|fn|method|' u'dmacro|dlecro|syntax|macro|dlecrox|lecrox|lecro|syntax)' - u'(?![\w!:?])', Name.Function), + u'(?![\\w!:?])', Name.Function), # Numbers (r'-?0[xX][0-9a-fA-F]+', Number.Hex), @@ -801,7 +801,7 @@ class ClojureLexer(RegexLexer): # TODO / should divide keywords/symbols into namespace/rest # but that's hard, so just pretend / is part of the name - valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+' + valid_name = r'(?!#)[\w!$%*+<=>?/.#|-]+' tokens = { 'root': [ @@ -1258,7 +1258,7 @@ class GoloLexer(RegexLexer): (r'-?\d[\d_]*L', Number.Integer.Long), (r'-?\d[\d_]*', Number.Integer), - ('`?[a-zA-Z_][\w$]*', Name), + (r'`?[a-zA-Z_][\w$]*', Name), (r'@[a-zA-Z_][\w$.]*', Name.Decorator), (r'"""', String, combined('stringescape', 'triplestring')), diff --git a/pygments/lexers/lisp.py b/pygments/lexers/lisp.py index e258c347..ba3fd403 100644 --- a/pygments/lexers/lisp.py +++ b/pygments/lexers/lisp.py @@ -19,7 +19,7 @@ from pygments.lexers.python import PythonLexer __all__ = ['SchemeLexer', 'CommonLispLexer', 'HyLexer', 'RacketLexer', 'NewLispLexer', 'EmacsLispLexer', 'ShenLexer', 'CPSALexer', - 'XtlangLexer'] + 'XtlangLexer', 'FennelLexer'] class SchemeLexer(RegexLexer): @@ -139,7 +139,7 @@ class SchemeLexer(RegexLexer): (r"(?<=#\()" + valid_name, Name.Variable), # highlight the builtins - ("(?<=\()(%s)" % '|'.join(re.escape(entry) + ' ' for entry in builtins), + (r"(?<=\()(%s)" % '|'.join(re.escape(entry) + ' ' for entry in builtins), Name.Builtin), # the remaining functions @@ -321,7 +321,7 @@ class CommonLispLexer(RegexLexer): (r'#\d+#', Operator), # read-time comment - (r'#+nil' + terminated + '\s*\(', Comment.Preproc, 'commented-form'), + (r'#+nil' + terminated + r'\s*\(', Comment.Preproc, 'commented-form'), # read-time conditional (r'#[+-]', Operator), @@ -333,7 +333,7 @@ class CommonLispLexer(RegexLexer): (r'(t|nil)' + terminated, Name.Constant), # functions and variables - (r'\*' + symbol + '\*', Name.Variable.Global), + (r'\*' + symbol + r'\*', Name.Variable.Global), (symbol, Name.Variable), # parentheses @@ -382,7 +382,7 @@ class HyLexer(RegexLexer): # valid names for identifiers # well, names can only not consist fully of numbers # but this should be good enough for now - valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+' + valid_name = r'(?!#)[\w!$%*+<=>?/.#-:]+' def _multi_escape(entries): return words(entries, suffix=' ') @@ -1249,7 +1249,7 @@ class RacketLexer(RegexLexer): _opening_parenthesis = r'[([{]' _closing_parenthesis = r'[)\]}]' _delimiters = r'()[\]{}",\'`;\s' - _symbol = r'(?u)(?:\|[^|]*\||\\[\w\W]|[^|\\%s]+)+' % _delimiters + _symbol = r'(?:\|[^|]*\||\\[\w\W]|[^|\\%s]+)+' % _delimiters _exact_decimal_prefix = r'(?:#e)?(?:#d)?(?:#e)?' _exponent = r'(?:[defls][-+]?\d+)' _inexact_simple_no_hashes = r'(?:\d+(?:/\d+|\.\d*)?|\.\d+)' @@ -1301,16 +1301,16 @@ class RacketLexer(RegexLexer): (_inexact_simple, _delimiters), Number.Float, '#pop'), # #b - (r'(?i)(#[ei])?#b%s' % _symbol, Number.Bin, '#pop'), + (r'(?iu)(#[ei])?#b%s' % _symbol, Number.Bin, '#pop'), # #o - (r'(?i)(#[ei])?#o%s' % _symbol, Number.Oct, '#pop'), + (r'(?iu)(#[ei])?#o%s' % _symbol, Number.Oct, '#pop'), # #x - (r'(?i)(#[ei])?#x%s' % _symbol, Number.Hex, '#pop'), + (r'(?iu)(#[ei])?#x%s' % _symbol, Number.Hex, '#pop'), # #i is always inexact, i.e. float - (r'(?i)(#d)?#i%s' % _symbol, Number.Float, '#pop'), + (r'(?iu)(#d)?#i%s' % _symbol, Number.Float, '#pop'), # Strings and characters (r'#?"', String.Double, ('#pop', 'string')), @@ -1323,7 +1323,7 @@ class RacketLexer(RegexLexer): (r'#(true|false|[tTfF])', Name.Constant, '#pop'), # Keyword argument names (e.g. #:keyword) - (r'#:%s' % _symbol, Keyword.Declaration, '#pop'), + (r'(?u)#:%s' % _symbol, Keyword.Declaration, '#pop'), # Reader extensions (r'(#lang |#!)(\S+)', @@ -2154,7 +2154,7 @@ class EmacsLispLexer(RegexLexer): (r'(t|nil)' + terminated, Name.Constant), # functions and variables - (r'\*' + symbol + '\*', Name.Variable.Global), + (r'\*' + symbol + r'\*', Name.Variable.Global), (symbol, Name.Variable), # parentheses @@ -2327,13 +2327,13 @@ class ShenLexer(RegexLexer): token = Name.Function if token == Literal else token yield index, token, value - raise StopIteration + return def _process_signature(self, tokens): for index, token, value in tokens: if token == Literal and value == '}': yield index, Punctuation, value - raise StopIteration + return elif token in (Literal, Name.Function): token = Name.Variable if value.istitle() else Keyword.Type yield index, token, value @@ -2619,3 +2619,75 @@ class XtlangLexer(RegexLexer): include('scheme') ], } + + +class FennelLexer(RegexLexer): + """A lexer for the `Fennel programming language <https://fennel-lang.org>`_. + + Fennel compiles to Lua, so all the Lua builtins are recognized as well + as the special forms that are particular to the Fennel compiler. + + .. versionadded:: 2.3 + """ + name = 'Fennel' + aliases = ['fennel', 'fnl'] + filenames = ['*.fnl'] + + # these two lists are taken from fennel-mode.el: + # https://gitlab.com/technomancy/fennel-mode + # this list is current as of Fennel version 0.1.0. + special_forms = ( + u'require-macros', u'eval-compiler', + u'do', u'values', u'if', u'when', u'each', u'for', u'fn', u'lambda', + u'λ', u'set', u'global', u'var', u'local', u'let', u'tset', u'doto', + u'set-forcibly!', u'defn', u'partial', u'while', u'or', u'and', u'true', + u'false', u'nil', u'.', u'+', u'..', u'^', u'-', u'*', u'%', u'/', u'>', + u'<', u'>=', u'<=', u'=', u'~=', u'#', u'...', u':', u'->', u'->>', + ) + + # Might be nicer to use the list from _lua_builtins.py but it's unclear how? + builtins = ( + u'_G', u'_VERSION', u'arg', u'assert', u'bit32', u'collectgarbage', + u'coroutine', u'debug', u'dofile', u'error', u'getfenv', + u'getmetatable', u'io', u'ipairs', u'load', u'loadfile', u'loadstring', + u'math', u'next', u'os', u'package', u'pairs', u'pcall', u'print', + u'rawequal', u'rawget', u'rawlen', u'rawset', u'require', u'select', + u'setfenv', u'setmetatable', u'string', u'table', u'tonumber', + u'tostring', u'type', u'unpack', u'xpcall' + ) + + # based on the scheme definition, but disallowing leading digits and commas + valid_name = r'[a-zA-Z_!$%&*+/:<=>?@^~|-][\w!$%&*+/:<=>?@^~|\.-]*' + + tokens = { + 'root': [ + # the only comment form is a semicolon; goes to the end of the line + (r';.*$', Comment.Single), + + (r'[,\s]+', Text), + (r'-?\d+\.\d+', Number.Float), + (r'-?\d+', Number.Integer), + + (r'"(\\\\|\\"|[^"])*"', String), + (r"'(\\\\|\\'|[^'])*'", String), + + # these are technically strings, but it's worth visually + # distinguishing them because their intent is different + # from regular strings. + (r':' + valid_name, String.Symbol), + + # special forms are keywords + (words(special_forms, suffix=' '), Keyword), + # lua standard library are builtins + (words(builtins, suffix=' '), Name.Builtin), + # special-case the vararg symbol + (r'\.\.\.', Name.Variable), + # regular identifiers + (valid_name, Name.Variable), + + # all your normal paired delimiters for your programming enjoyment + (r'(\(|\))', Punctuation), + (r'(\[|\])', Punctuation), + (r'(\{|\})', Punctuation), + ] + } diff --git a/pygments/lexers/markup.py b/pygments/lexers/markup.py index 92dc9e7a..e6265f40 100644 --- a/pygments/lexers/markup.py +++ b/pygments/lexers/markup.py @@ -536,10 +536,9 @@ class MarkdownLexer(RegexLexer): # no lexer for this language. handle it like it was a code block if lexer is None: yield match.start(4), String, code - return - - for item in do_insertions([], lexer.get_tokens_unprocessed(code)): - yield item + else: + for item in do_insertions([], lexer.get_tokens_unprocessed(code)): + yield item yield match.start(5), String , match.group(5) diff --git a/pygments/lexers/matlab.py b/pygments/lexers/matlab.py index 56a0f6d6..1c77b60c 100644 --- a/pygments/lexers/matlab.py +++ b/pygments/lexers/matlab.py @@ -134,9 +134,9 @@ class MatlabLexer(RegexLexer): } def analyse_text(text): - if re.match('^\s*%', text, re.M): # comment + if re.match(r'^\s*%', text, re.M): # comment return 0.2 - elif re.match('^!\w+', text, re.M): # system cmd + elif re.match(r'^!\w+', text, re.M): # system cmd return 0.2 diff --git a/pygments/lexers/ml.py b/pygments/lexers/ml.py index f80d5bfa..0bff9816 100644 --- a/pygments/lexers/ml.py +++ b/pygments/lexers/ml.py @@ -43,7 +43,7 @@ class SMLLexer(RegexLexer): symbolicid_reserved = set(( # Core - ':', '\|', '=', '=>', '->', '#', + ':', r'\|', '=', '=>', '->', '#', # Modules ':>', )) diff --git a/pygments/lexers/objective.py b/pygments/lexers/objective.py index 7807255e..179928e9 100644 --- a/pygments/lexers/objective.py +++ b/pygments/lexers/objective.py @@ -87,26 +87,26 @@ def objective(baselexer): ], 'oc_classname': [ # interface definition that inherits - ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)(\{)', + (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)(\{)', bygroups(Name.Class, Text, Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', + (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', bygroups(Name.Class, Text, Name.Class), '#pop'), # interface definition for a category - ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)(\{)', + (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)(\{)', bygroups(Name.Class, Text, Name.Label, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))', + (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))', bygroups(Name.Class, Text, Name.Label), '#pop'), # simple interface / implementation - ('([a-zA-Z$_][\w$]*)(\s*)(\{)', + (r'([a-zA-Z$_][\w$]*)(\s*)(\{)', bygroups(Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop') + (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop') ], 'oc_forward_classname': [ - ('([a-zA-Z$_][\w$]*)(\s*,\s*)', + (r'([a-zA-Z$_][\w$]*)(\s*,\s*)', bygroups(Name.Class, Text), 'oc_forward_classname'), - ('([a-zA-Z$_][\w$]*)(\s*;?)', + (r'([a-zA-Z$_][\w$]*)(\s*;?)', bygroups(Name.Class, Text), '#pop') ], 'oc_ivars': [ @@ -244,17 +244,17 @@ class LogosLexer(ObjectiveCppLexer): inherit, ], 'logos_init_directive': [ - ('\s+', Text), + (r'\s+', Text), (',', Punctuation, ('logos_init_directive', '#pop')), - ('([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)', + (r'([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)', bygroups(Name.Class, Text, Punctuation, Text, Text)), - ('([a-zA-Z$_][\w$]*)', Name.Class), - ('\)', Punctuation, '#pop'), + (r'([a-zA-Z$_][\w$]*)', Name.Class), + (r'\)', Punctuation, '#pop'), ], 'logos_classname': [ - ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', + (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', bygroups(Name.Class, Text, Name.Class), '#pop'), - ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop') + (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop') ], 'root': [ (r'(%subclass)(\s+)', bygroups(Keyword, Text), diff --git a/pygments/lexers/parsers.py b/pygments/lexers/parsers.py index 1f3c9b4d..43eb6c1f 100644 --- a/pygments/lexers/parsers.py +++ b/pygments/lexers/parsers.py @@ -364,13 +364,13 @@ class AntlrLexer(RegexLexer): # tokensSpec (r'tokens\b', Keyword, 'tokens'), # attrScope - (r'(scope)(\s*)(' + _id + ')(\s*)(\{)', + (r'(scope)(\s*)(' + _id + r')(\s*)(\{)', bygroups(Keyword, Whitespace, Name.Variable, Whitespace, Punctuation), 'action'), # exception (r'(catch|finally)\b', Keyword, 'exception'), # action - (r'(@' + _id + ')(\s*)(::)?(\s*)(' + _id + ')(\s*)(\{)', + (r'(@' + _id + r')(\s*)(::)?(\s*)(' + _id + r')(\s*)(\{)', bygroups(Name.Label, Whitespace, Punctuation, Whitespace, Name.Label, Whitespace, Punctuation), 'action'), # rule @@ -405,10 +405,10 @@ class AntlrLexer(RegexLexer): # L173 ANTLRv3.g from ANTLR book (r'(scope)(\s+)(\{)', bygroups(Keyword, Whitespace, Punctuation), 'action'), - (r'(scope)(\s+)(' + _id + ')(\s*)(;)', + (r'(scope)(\s+)(' + _id + r')(\s*)(;)', bygroups(Keyword, Whitespace, Name.Label, Whitespace, Punctuation)), # ruleAction - (r'(@' + _id + ')(\s*)(\{)', + (r'(@' + _id + r')(\s*)(\{)', bygroups(Name.Label, Whitespace, Punctuation), 'action'), # finished prelims, go to rule alts! (r':', Punctuation, '#pop') @@ -442,7 +442,7 @@ class AntlrLexer(RegexLexer): include('comments'), (r'\{', Punctuation), (r'(' + _TOKEN_REF + r')(\s*)(=)?(\s*)(' + _STRING_LITERAL - + ')?(\s*)(;)', + + r')?(\s*)(;)', bygroups(Name.Label, Whitespace, Punctuation, Whitespace, String, Whitespace, Punctuation)), (r'\}', Punctuation, '#pop'), @@ -452,7 +452,7 @@ class AntlrLexer(RegexLexer): include('comments'), (r'\{', Punctuation), (r'(' + _id + r')(\s*)(=)(\s*)(' + - '|'.join((_id, _STRING_LITERAL, _INT, '\*')) + ')(\s*)(;)', + '|'.join((_id, _STRING_LITERAL, _INT, r'\*')) + r')(\s*)(;)', bygroups(Name.Variable, Whitespace, Punctuation, Whitespace, Text, Whitespace, Punctuation)), (r'\}', Punctuation, '#pop'), diff --git a/pygments/lexers/pascal.py b/pygments/lexers/pascal.py index 9aa1ac8f..467a0b2c 100644 --- a/pygments/lexers/pascal.py +++ b/pygments/lexers/pascal.py @@ -593,8 +593,8 @@ class AdaLexer(RegexLexer): ], 'end': [ ('(if|case|record|loop|select)', Keyword.Reserved), - ('"[^"]+"|[\w.]+', Name.Function), - ('\s+', Text), + (r'"[^"]+"|[\w.]+', Name.Function), + (r'\s+', Text), (';', Punctuation, '#pop'), ], 'type_def': [ @@ -628,11 +628,11 @@ class AdaLexer(RegexLexer): ], 'package': [ ('body', Keyword.Declaration), - ('is\s+new|renames', Keyword.Reserved), + (r'is\s+new|renames', Keyword.Reserved), ('is', Keyword.Reserved, '#pop'), (';', Punctuation, '#pop'), - ('\(', Punctuation, 'package_instantiation'), - ('([\w.]+)', Name.Class), + (r'\(', Punctuation, 'package_instantiation'), + (r'([\w.]+)', Name.Class), include('root'), ], 'package_instantiation': [ diff --git a/pygments/lexers/pawn.py b/pygments/lexers/pawn.py index f462a883..0ef28175 100644 --- a/pygments/lexers/pawn.py +++ b/pygments/lexers/pawn.py @@ -36,7 +36,7 @@ class SourcePawnLexer(RegexLexer): tokens = { 'root': [ # preprocessor directives: without whitespace - ('^#if\s+0', Comment.Preproc, 'if0'), + (r'^#if\s+0', Comment.Preproc, 'if0'), ('^#', Comment.Preproc, 'macro'), # or with whitespace ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'), @@ -62,7 +62,7 @@ class SourcePawnLexer(RegexLexer): r'public|return|sizeof|static|decl|struct|switch)\b', Keyword), (r'(bool|Float)\b', Keyword.Type), (r'(true|false)\b', Keyword.Constant), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'string': [ (r'"', String, '#pop'), @@ -148,7 +148,7 @@ class PawnLexer(RegexLexer): tokens = { 'root': [ # preprocessor directives: without whitespace - ('^#if\s+0', Comment.Preproc, 'if0'), + (r'^#if\s+0', Comment.Preproc, 'if0'), ('^#', Comment.Preproc, 'macro'), # or with whitespace ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'), @@ -174,7 +174,7 @@ class PawnLexer(RegexLexer): r'public|return|sizeof|tagof|state|goto)\b', Keyword), (r'(bool|Float)\b', Keyword.Type), (r'(true|false)\b', Keyword.Constant), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'string': [ (r'"', String, '#pop'), diff --git a/pygments/lexers/perl.py b/pygments/lexers/perl.py index db5a9361..27e3cc79 100644 --- a/pygments/lexers/perl.py +++ b/pygments/lexers/perl.py @@ -208,7 +208,7 @@ class PerlLexer(RegexLexer): def analyse_text(text): if shebang_matches(text, r'perl'): return True - if re.search('(?:my|our)\s+[$@%(]', text): + if re.search(r'(?:my|our)\s+[$@%(]', text): return 0.9 @@ -226,7 +226,7 @@ class Perl6Lexer(ExtendedRegexLexer): mimetypes = ['text/x-perl6', 'application/x-perl6'] flags = re.MULTILINE | re.DOTALL | re.UNICODE - PERL6_IDENTIFIER_RANGE = "['\w:-]" + PERL6_IDENTIFIER_RANGE = r"['\w:-]" PERL6_KEYWORDS = ( 'BEGIN', 'CATCH', 'CHECK', 'CONTROL', 'END', 'ENTER', 'FIRST', 'INIT', @@ -495,7 +495,7 @@ class Perl6Lexer(ExtendedRegexLexer): (r'^=.*?\n\s*?\n', Comment.Multiline), (r'(regex|token|rule)(\s*' + PERL6_IDENTIFIER_RANGE + '+:sym)', bygroups(Keyword, Name), 'token-sym-brackets'), - (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + ')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?', + (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + r')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?', bygroups(Keyword, Name), 'pre-token'), # deal with a special case in the Perl 6 grammar (role q { ... }) (r'(role)(\s+)(q)(\s*)', bygroups(Keyword, Text, Name, Text)), @@ -591,21 +591,21 @@ class Perl6Lexer(ExtendedRegexLexer): rating = False # check for my/our/has declarations - if re.search("(?:my|our|has)\s+(?:" + Perl6Lexer.PERL6_IDENTIFIER_RANGE + - "+\s+)?[$@%&(]", text): + if re.search(r"(?:my|our|has)\s+(?:" + Perl6Lexer.PERL6_IDENTIFIER_RANGE + + r"+\s+)?[$@%&(]", text): rating = 0.8 saw_perl_decl = True for line in lines: line = re.sub('#.*', '', line) - if re.match('^\s*$', line): + if re.match(r'^\s*$', line): continue # match v6; use v6; use v6.0; use v6.0.0; - if re.match('^\s*(?:use\s+)?v6(?:\.\d(?:\.\d)?)?;', line): + if re.match(r'^\s*(?:use\s+)?v6(?:\.\d(?:\.\d)?)?;', line): return True # match class, module, role, enum, grammar declarations - class_decl = re.match('^\s*(?:(?P<scope>my|our)\s+)?(?:module|class|role|enum|grammar)', line) + class_decl = re.match(r'^\s*(?:(?P<scope>my|our)\s+)?(?:module|class|role|enum|grammar)', line) if class_decl: if saw_perl_decl or class_decl.group('scope') is not None: return True diff --git a/pygments/lexers/php.py b/pygments/lexers/php.py index f618b5fd..f959fb1f 100644 --- a/pygments/lexers/php.py +++ b/pygments/lexers/php.py @@ -173,7 +173,7 @@ class PhpLexer(RegexLexer): r'finally)\b', Keyword), (r'(true|false|null)\b', Keyword.Constant), include('magicconstants'), - (r'\$\{\$+' + _ident_inner + '\}', Name.Variable), + (r'\$\{\$+' + _ident_inner + r'\}', Name.Variable), (r'\$+' + _ident_inner, Name.Variable), (_ident_inner, Name.Other), (r'(\d+\.\d*|\d*\.\d+)(e[+-]?[0-9]+)?', Number.Float), @@ -214,7 +214,7 @@ class PhpLexer(RegexLexer): (r'"', String.Double, '#pop'), (r'[^{$"\\]+', String.Double), (r'\\([nrt"$\\]|[0-7]{1,3}|x[0-9a-f]{1,2})', String.Escape), - (r'\$' + _ident_inner + '(\[\S+?\]|->' + _ident_inner + ')?', + (r'\$' + _ident_inner + r'(\[\S+?\]|->' + _ident_inner + ')?', String.Interpol), (r'(\{\$\{)(.*?)(\}\})', bygroups(String.Interpol, using(this, _startinline=True), diff --git a/pygments/lexers/prolog.py b/pygments/lexers/prolog.py index 90f9529c..58e762b0 100644 --- a/pygments/lexers/prolog.py +++ b/pygments/lexers/prolog.py @@ -57,15 +57,15 @@ class PrologLexer(RegexLexer): (r'_', Keyword), # The don't-care variable (r'([a-z]+)(:)', bygroups(Name.Namespace, Punctuation)), (u'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' + u'[\\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' u'(\\s*)(:-|-->)', bygroups(Name.Function, Text, Operator)), # function defn (u'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' + u'[\\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' u'(\\s*)(\\()', bygroups(Name.Function, Text, Punctuation)), (u'[a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*', + u'[\\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*', String.Atom), # atom, characters # This one includes ! (u'[#&*+\\-./:<=>?@\\\\^~\u00a1-\u00bf\u2010-\u303f]+', @@ -300,7 +300,7 @@ class LogtalkLexer(RegexLexer): return 1.0 elif ':- category(' in text: return 1.0 - elif re.search('^:-\s[a-z]', text, re.M): + elif re.search(r'^:-\s[a-z]', text, re.M): return 0.9 else: return 0.0 diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py index 390eafe8..c87282ca 100644 --- a/pygments/lexers/python.py +++ b/pygments/lexers/python.py @@ -124,10 +124,10 @@ class PythonLexer(RegexLexer): 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError', - 'MemoryError', 'NameError', 'NotImplemented', 'NotImplementedError', + 'MemoryError', 'ModuleNotFoundError', 'NameError', 'NotImplemented', 'NotImplementedError', 'OSError', 'OverflowError', 'OverflowWarning', 'PendingDeprecationWarning', - 'ReferenceError', 'RuntimeError', 'RuntimeWarning', 'StandardError', - 'StopIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', + 'RecursionError', 'ReferenceError', 'RuntimeError', 'RuntimeWarning', 'StandardError', + 'StopIteration', 'StopAsyncIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning', @@ -180,15 +180,15 @@ class PythonLexer(RegexLexer): ], 'name': [ (r'@[\w.]+', Name.Decorator), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'funcname': [ include('magicfuncs'), - ('[a-zA-Z_]\w*', Name.Function, '#pop'), + (r'[a-zA-Z_]\w*', Name.Function, '#pop'), default('#pop'), ], 'classname': [ - ('[a-zA-Z_]\w*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ (r'(?:[ \t]|\\\n)+', Text), @@ -262,13 +262,13 @@ class Python3Lexer(RegexLexer): return [ # the old style '%s' % (...) string formatting (still valid in Py3) (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' - '[hlL]?[E-GXc-giorsux%]', String.Interpol), + '[hlL]?[E-GXc-giorsaux%]', String.Interpol), # the new style '{}'.format(...) string formatting (r'\{' - '((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name - '(\![sra])?' # conversion - '(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?' - '\}', String.Interpol), + r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name + r'(\![sra])?' # conversion + r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?' + r'\}', String.Interpol), # backslashes, quotes and formatting signs must be parsed one at a time (r'[^\\\'"%{\n]+', ttype), @@ -361,12 +361,12 @@ class Python3Lexer(RegexLexer): Name.Variable.Magic), ] tokens['numbers'] = [ - (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), - (r'\d+[eE][+-]?[0-9]+j?', Number.Float), - (r'0[oO][0-7]+', Number.Oct), - (r'0[bB][01]+', Number.Bin), - (r'0[xX][a-fA-F0-9]+', Number.Hex), - (r'\d+', Number.Integer) + (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)([eE][+-]?\d(?:_?\d)*)?', Number.Float), + (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float), + (r'0[oO](?:_?[0-7])+', Number.Oct), + (r'0[bB](?:_?[01])+', Number.Bin), + (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex), + (r'\d(?:_?\d)*', Number.Integer) ] tokens['backtick'] = [] tokens['name'] = [ @@ -396,6 +396,7 @@ class Python3Lexer(RegexLexer): tokens['strings-single'] = innerstring_rules(String.Single) tokens['strings-double'] = innerstring_rules(String.Double) + def analyse_text(text): return shebang_matches(text, r'pythonw?3(\.\d)?') @@ -671,10 +672,10 @@ class CythonLexer(RegexLexer): ], 'name': [ (r'@\w+', Name.Decorator), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'funcname': [ - ('[a-zA-Z_]\w*', Name.Function, '#pop') + (r'[a-zA-Z_]\w*', Name.Function, '#pop') ], 'cdef': [ (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved), @@ -691,7 +692,7 @@ class CythonLexer(RegexLexer): (r'.', Text), ], 'classname': [ - ('[a-zA-Z_]\w*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)), diff --git a/pygments/lexers/qvt.py b/pygments/lexers/qvt.py index f496d600..af091a65 100644 --- a/pygments/lexers/qvt.py +++ b/pygments/lexers/qvt.py @@ -126,7 +126,7 @@ class QVToLexer(RegexLexer): (r'[^\\\'"\n]+', String), # quotes, percents and backslashes must be parsed one at a time (r'[\'"\\]', String), - ], + ], 'stringescape': [ (r'\\([\\btnfr"\']|u[0-3][0-7]{2}|u[0-7]{1,2})', String.Escape) ], @@ -134,15 +134,15 @@ class QVToLexer(RegexLexer): (r'"', String, '#pop'), (r'\\\\|\\"', String.Escape), include('strings') - ], + ], 'sqs': [ # single-quoted string (r"'", String, '#pop'), (r"\\\\|\\'", String.Escape), include('strings') - ], + ], 'name': [ - ('[a-zA-Z_]\w*', Name), - ], + (r'[a-zA-Z_]\w*', Name), + ], # numbers: excerpt taken from the python lexer 'numbers': [ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), diff --git a/pygments/lexers/r.py b/pygments/lexers/r.py index dce61969..0829ae6e 100644 --- a/pygments/lexers/r.py +++ b/pygments/lexers/r.py @@ -11,7 +11,7 @@ import re -from pygments.lexer import Lexer, RegexLexer, include, words, do_insertions +from pygments.lexer import Lexer, RegexLexer, include, do_insertions, bygroups from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Generic @@ -80,286 +80,25 @@ class SLexer(RegexLexer): mimetypes = ['text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile'] - builtins_base = ( - 'Arg', 'Conj', 'Cstack_info', 'Encoding', 'FALSE', - 'Filter', 'Find', 'I', 'ISOdate', 'ISOdatetime', 'Im', 'Inf', - 'La.svd', 'Map', 'Math.Date', 'Math.POSIXt', 'Math.data.frame', - 'Math.difftime', 'Math.factor', 'Mod', 'NA_character_', - 'NA_complex_', 'NA_real_', 'NCOL', 'NROW', 'NULLNA_integer_', 'NaN', - 'Negate', 'NextMethod', 'Ops.Date', 'Ops.POSIXt', 'Ops.data.frame', - 'Ops.difftime', 'Ops.factor', 'Ops.numeric_version', 'Ops.ordered', - 'Position', 'R.Version', 'R.home', 'R.version', 'R.version.string', - 'RNGkind', 'RNGversion', 'R_system_version', 'Re', 'Recall', - 'Reduce', 'Summary.Date', 'Summary.POSIXct', 'Summary.POSIXlt', - 'Summary.data.frame', 'Summary.difftime', 'Summary.factor', - 'Summary.numeric_version', 'Summary.ordered', 'Sys.Date', - 'Sys.chmod', 'Sys.getenv', 'Sys.getlocale', 'Sys.getpid', - 'Sys.glob', 'Sys.info', 'Sys.localeconv', 'Sys.readlink', - 'Sys.setFileTime', 'Sys.setenv', 'Sys.setlocale', 'Sys.sleep', - 'Sys.time', 'Sys.timezone', 'Sys.umask', 'Sys.unsetenv', - 'Sys.which', 'TRUE', 'UseMethod', 'Vectorize', 'abbreviate', 'abs', - 'acos', 'acosh', 'addNA', 'addTaskCallback', 'agrep', 'alist', - 'all', 'all.equal', 'all.equal.POSIXct', 'all.equal.character', - 'all.equal.default', 'all.equal.factor', 'all.equal.formula', - 'all.equal.language', 'all.equal.list', 'all.equal.numeric', - 'all.equal.raw', 'all.names', 'all.vars', 'any', 'anyDuplicated', - 'anyDuplicated.array', 'anyDuplicated.data.frame', - 'anyDuplicated.default', 'anyDuplicated.matrix', 'aperm', - 'aperm.default', 'aperm.table', 'append', 'apply', 'args', - 'arrayInd', 'as.Date', 'as.Date.POSIXct', 'as.Date.POSIXlt', - 'as.Date.character', 'as.Date.date', 'as.Date.dates', - 'as.Date.default', 'as.Date.factor', 'as.Date.numeric', - 'as.POSIXct', 'as.POSIXct.Date', 'as.POSIXct.POSIXlt', - 'as.POSIXct.date', 'as.POSIXct.dates', 'as.POSIXct.default', - 'as.POSIXct.numeric', 'as.POSIXlt', 'as.POSIXlt.Date', - 'as.POSIXlt.POSIXct', 'as.POSIXlt.character', 'as.POSIXlt.date', - 'as.POSIXlt.dates', 'as.POSIXlt.default', 'as.POSIXlt.factor', - 'as.POSIXlt.numeric', 'as.array', 'as.array.default', 'as.call', - 'as.character', 'as.character.Date', 'as.character.POSIXt', - 'as.character.condition', 'as.character.default', - 'as.character.error', 'as.character.factor', 'as.character.hexmode', - 'as.character.numeric_version', 'as.character.octmode', - 'as.character.srcref', 'as.complex', 'as.data.frame', - 'as.data.frame.AsIs', 'as.data.frame.Date', 'as.data.frame.POSIXct', - 'as.data.frame.POSIXlt', 'as.data.frame.array', - 'as.data.frame.character', 'as.data.frame.complex', - 'as.data.frame.data.frame', 'as.data.frame.default', - 'as.data.frame.difftime', 'as.data.frame.factor', - 'as.data.frame.integer', 'as.data.frame.list', - 'as.data.frame.logical', 'as.data.frame.matrix', - 'as.data.frame.model.matrix', 'as.data.frame.numeric', - 'as.data.frame.numeric_version', 'as.data.frame.ordered', - 'as.data.frame.raw', 'as.data.frame.table', 'as.data.frame.ts', - 'as.data.frame.vector', 'as.difftime', 'as.double', - 'as.double.POSIXlt', 'as.double.difftime', 'as.environment', - 'as.expression', 'as.expression.default', 'as.factor', - 'as.function', 'as.function.default', 'as.hexmode', 'as.integer', - 'as.list', 'as.list.Date', 'as.list.POSIXct', 'as.list.data.frame', - 'as.list.default', 'as.list.environment', 'as.list.factor', - 'as.list.function', 'as.list.numeric_version', 'as.logical', - 'as.logical.factor', 'as.matrix', 'as.matrix.POSIXlt', - 'as.matrix.data.frame', 'as.matrix.default', 'as.matrix.noquote', - 'as.name', 'as.null', 'as.null.default', 'as.numeric', - 'as.numeric_version', 'as.octmode', 'as.ordered', - 'as.package_version', 'as.pairlist', 'as.qr', 'as.raw', 'as.single', - 'as.single.default', 'as.symbol', 'as.table', 'as.table.default', - 'as.vector', 'as.vector.factor', 'asNamespace', 'asS3', 'asS4', - 'asin', 'asinh', 'assign', 'atan', 'atan2', 'atanh', - 'attachNamespace', 'attr', 'attr.all.equal', 'attributes', - 'autoload', 'autoloader', 'backsolve', 'baseenv', 'basename', - 'besselI', 'besselJ', 'besselK', 'besselY', 'beta', - 'bindingIsActive', 'bindingIsLocked', 'bindtextdomain', 'bitwAnd', - 'bitwNot', 'bitwOr', 'bitwShiftL', 'bitwShiftR', 'bitwXor', 'body', - 'bquote', 'browser', 'browserCondition', 'browserSetDebug', - 'browserText', 'builtins', 'by', 'by.data.frame', 'by.default', - 'bzfile', 'c.Date', 'c.POSIXct', 'c.POSIXlt', 'c.noquote', - 'c.numeric_version', 'call', 'callCC', 'capabilities', 'casefold', - 'cat', 'category', 'cbind', 'cbind.data.frame', 'ceiling', - 'char.expand', 'charToRaw', 'charmatch', 'chartr', 'check_tzones', - 'chol', 'chol.default', 'chol2inv', 'choose', 'class', - 'clearPushBack', 'close', 'close.connection', 'close.srcfile', - 'close.srcfilealias', 'closeAllConnections', 'col', 'colMeans', - 'colSums', 'colnames', 'commandArgs', 'comment', 'computeRestarts', - 'conditionCall', 'conditionCall.condition', 'conditionMessage', - 'conditionMessage.condition', 'conflicts', 'contributors', 'cos', - 'cosh', 'crossprod', 'cummax', 'cummin', 'cumprod', 'cumsum', 'cut', - 'cut.Date', 'cut.POSIXt', 'cut.default', 'dQuote', 'data.class', - 'data.matrix', 'date', 'debug', 'debugonce', - 'default.stringsAsFactors', 'delayedAssign', 'deparse', 'det', - 'determinant', 'determinant.matrix', 'dget', 'diag', 'diff', - 'diff.Date', 'diff.POSIXt', 'diff.default', 'difftime', 'digamma', - 'dim', 'dim.data.frame', 'dimnames', 'dimnames.data.frame', 'dir', - 'dir.create', 'dirname', 'do.call', 'dput', 'drop', 'droplevels', - 'droplevels.data.frame', 'droplevels.factor', 'dump', 'duplicated', - 'duplicated.POSIXlt', 'duplicated.array', 'duplicated.data.frame', - 'duplicated.default', 'duplicated.matrix', - 'duplicated.numeric_version', 'dyn.load', 'dyn.unload', 'eapply', - 'eigen', 'else', 'emptyenv', 'enc2native', 'enc2utf8', - 'encodeString', 'enquote', 'env.profile', 'environment', - 'environmentIsLocked', 'environmentName', 'eval', 'eval.parent', - 'evalq', 'exists', 'exp', 'expand.grid', 'expm1', 'expression', - 'factor', 'factorial', 'fifo', 'file', 'file.access', 'file.append', - 'file.choose', 'file.copy', 'file.create', 'file.exists', - 'file.info', 'file.link', 'file.path', 'file.remove', 'file.rename', - 'file.show', 'file.symlink', 'find.package', 'findInterval', - 'findPackageEnv', 'findRestart', 'floor', 'flush', - 'flush.connection', 'force', 'formals', 'format', - 'format.AsIs', 'format.Date', 'format.POSIXct', 'format.POSIXlt', - 'format.data.frame', 'format.default', 'format.difftime', - 'format.factor', 'format.hexmode', 'format.info', - 'format.libraryIQR', 'format.numeric_version', 'format.octmode', - 'format.packageInfo', 'format.pval', 'format.summaryDefault', - 'formatC', 'formatDL', 'forwardsolve', 'gamma', 'gc', 'gc.time', - 'gcinfo', 'gctorture', 'gctorture2', 'get', 'getAllConnections', - 'getCallingDLL', 'getCallingDLLe', 'getConnection', - 'getDLLRegisteredRoutines', 'getDLLRegisteredRoutines.DLLInfo', - 'getDLLRegisteredRoutines.character', 'getElement', - 'getExportedValue', 'getHook', 'getLoadedDLLs', 'getNamespace', - 'getNamespaceExports', 'getNamespaceImports', 'getNamespaceInfo', - 'getNamespaceName', 'getNamespaceUsers', 'getNamespaceVersion', - 'getNativeSymbolInfo', 'getOption', 'getRversion', 'getSrcLines', - 'getTaskCallbackNames', 'geterrmessage', 'gettext', 'gettextf', - 'getwd', 'gl', 'globalenv', 'gregexpr', 'grep', 'grepRaw', 'grepl', - 'gsub', 'gzcon', 'gzfile', 'head', 'iconv', 'iconvlist', - 'icuSetCollate', 'identical', 'identity', 'ifelse', 'importIntoEnv', - 'in', 'inherits', 'intToBits', 'intToUtf8', 'interaction', 'interactive', - 'intersect', 'inverse.rle', 'invisible', 'invokeRestart', - 'invokeRestartInteractively', 'is.R', 'is.array', 'is.atomic', - 'is.call', 'is.character', 'is.complex', 'is.data.frame', - 'is.double', 'is.element', 'is.environment', 'is.expression', - 'is.factor', 'is.finite', 'is.function', 'is.infinite', - 'is.integer', 'is.language', 'is.list', 'is.loaded', 'is.logical', - 'is.matrix', 'is.na', 'is.na.POSIXlt', 'is.na.data.frame', - 'is.na.numeric_version', 'is.name', 'is.nan', 'is.null', - 'is.numeric', 'is.numeric.Date', 'is.numeric.POSIXt', - 'is.numeric.difftime', 'is.numeric_version', 'is.object', - 'is.ordered', 'is.package_version', 'is.pairlist', 'is.primitive', - 'is.qr', 'is.raw', 'is.recursive', 'is.single', 'is.symbol', - 'is.table', 'is.unsorted', 'is.vector', 'isBaseNamespace', - 'isIncomplete', 'isNamespace', 'isOpen', 'isRestart', 'isS4', - 'isSeekable', 'isSymmetric', 'isSymmetric.matrix', 'isTRUE', - 'isatty', 'isdebugged', 'jitter', 'julian', 'julian.Date', - 'julian.POSIXt', 'kappa', 'kappa.default', 'kappa.lm', 'kappa.qr', - 'kronecker', 'l10n_info', 'labels', 'labels.default', 'lapply', - 'lazyLoad', 'lazyLoadDBexec', 'lazyLoadDBfetch', 'lbeta', 'lchoose', - 'length', 'length.POSIXlt', 'letters', 'levels', 'levels.default', - 'lfactorial', 'lgamma', 'library.dynam', 'library.dynam.unload', - 'licence', 'license', 'list.dirs', 'list.files', 'list2env', 'load', - 'loadNamespace', 'loadedNamespaces', 'loadingNamespaceInfo', - 'local', 'lockBinding', 'lockEnvironment', 'log', 'log10', 'log1p', - 'log2', 'logb', 'lower.tri', 'ls', 'make.names', 'make.unique', - 'makeActiveBinding', 'mapply', 'margin.table', 'mat.or.vec', - 'match', 'match.arg', 'match.call', 'match.fun', 'max', 'max.col', - 'mean', 'mean.Date', 'mean.POSIXct', 'mean.POSIXlt', 'mean.default', - 'mean.difftime', 'mem.limits', 'memCompress', 'memDecompress', - 'memory.profile', 'merge', 'merge.data.frame', 'merge.default', - 'message', 'mget', 'min', 'missing', 'mode', 'month.abb', - 'month.name', 'months', 'months.Date', 'months.POSIXt', - 'months.abb', 'months.nameletters', 'names', 'names.POSIXlt', - 'namespaceExport', 'namespaceImport', 'namespaceImportClasses', - 'namespaceImportFrom', 'namespaceImportMethods', 'nargs', 'nchar', - 'ncol', 'new.env', 'ngettext', 'nlevels', 'noquote', 'norm', - 'normalizePath', 'nrow', 'numeric_version', 'nzchar', 'objects', - 'oldClass', 'on.exit', 'open', 'open.connection', 'open.srcfile', - 'open.srcfilealias', 'open.srcfilecopy', 'options', 'order', - 'ordered', 'outer', 'packBits', 'packageEvent', - 'packageHasNamespace', 'packageStartupMessage', 'package_version', - 'pairlist', 'parent.env', 'parent.frame', 'parse', - 'parseNamespaceFile', 'paste', 'paste0', 'path.expand', - 'path.package', 'pipe', 'pmatch', 'pmax', 'pmax.int', 'pmin', - 'pmin.int', 'polyroot', 'pos.to.env', 'pretty', 'pretty.default', - 'prettyNum', 'print', 'print.AsIs', 'print.DLLInfo', - 'print.DLLInfoList', 'print.DLLRegisteredRoutines', 'print.Date', - 'print.NativeRoutineList', 'print.POSIXct', 'print.POSIXlt', - 'print.by', 'print.condition', 'print.connection', - 'print.data.frame', 'print.default', 'print.difftime', - 'print.factor', 'print.function', 'print.hexmode', - 'print.libraryIQR', 'print.listof', 'print.noquote', - 'print.numeric_version', 'print.octmode', 'print.packageInfo', - 'print.proc_time', 'print.restart', 'print.rle', - 'print.simple.list', 'print.srcfile', 'print.srcref', - 'print.summary.table', 'print.summaryDefault', 'print.table', - 'print.warnings', 'prmatrix', 'proc.time', 'prod', 'prop.table', - 'provideDimnames', 'psigamma', 'pushBack', 'pushBackLength', 'q', - 'qr', 'qr.Q', 'qr.R', 'qr.X', 'qr.coef', 'qr.default', 'qr.fitted', - 'qr.qty', 'qr.qy', 'qr.resid', 'qr.solve', 'quarters', - 'quarters.Date', 'quarters.POSIXt', 'quit', 'quote', 'range', - 'range.default', 'rank', 'rapply', 'raw', 'rawConnection', - 'rawConnectionValue', 'rawShift', 'rawToBits', 'rawToChar', 'rbind', - 'rbind.data.frame', 'rcond', 'read.dcf', 'readBin', 'readChar', - 'readLines', 'readRDS', 'readRenviron', 'readline', 'reg.finalizer', - 'regexec', 'regexpr', 'registerS3method', 'registerS3methods', - 'regmatches', 'remove', 'removeTaskCallback', 'rep', 'rep.Date', - 'rep.POSIXct', 'rep.POSIXlt', 'rep.factor', 'rep.int', - 'rep.numeric_version', 'rep_len', 'replace', 'replicate', - 'requireNamespace', 'restartDescription', 'restartFormals', - 'retracemem', 'rev', 'rev.default', 'rle', 'rm', 'round', - 'round.Date', 'round.POSIXt', 'row', 'row.names', - 'row.names.data.frame', 'row.names.default', 'rowMeans', 'rowSums', - 'rownames', 'rowsum', 'rowsum.data.frame', 'rowsum.default', - 'sQuote', 'sample', 'sample.int', 'sapply', 'save', 'save.image', - 'saveRDS', 'scale', 'scale.default', 'scan', 'search', - 'searchpaths', 'seek', 'seek.connection', 'seq', 'seq.Date', - 'seq.POSIXt', 'seq.default', 'seq.int', 'seq_along', 'seq_len', - 'sequence', 'serialize', 'set.seed', 'setHook', 'setNamespaceInfo', - 'setSessionTimeLimit', 'setTimeLimit', 'setdiff', 'setequal', - 'setwd', 'shQuote', 'showConnections', 'sign', 'signalCondition', - 'signif', 'simpleCondition', 'simpleError', 'simpleMessage', - 'simpleWarning', 'simplify2array', 'sin', 'single', - 'sinh', 'sink', 'sink.number', 'slice.index', 'socketConnection', - 'socketSelect', 'solve', 'solve.default', 'solve.qr', 'sort', - 'sort.POSIXlt', 'sort.default', 'sort.int', 'sort.list', 'split', - 'split.Date', 'split.POSIXct', 'split.data.frame', 'split.default', - 'sprintf', 'sqrt', 'srcfile', 'srcfilealias', 'srcfilecopy', - 'srcref', 'standardGeneric', 'stderr', 'stdin', 'stdout', 'stop', - 'stopifnot', 'storage.mode', 'strftime', 'strptime', 'strsplit', - 'strtoi', 'strtrim', 'structure', 'strwrap', 'sub', 'subset', - 'subset.data.frame', 'subset.default', 'subset.matrix', - 'substitute', 'substr', 'substring', 'sum', 'summary', - 'summary.Date', 'summary.POSIXct', 'summary.POSIXlt', - 'summary.connection', 'summary.data.frame', 'summary.default', - 'summary.factor', 'summary.matrix', 'summary.proc_time', - 'summary.srcfile', 'summary.srcref', 'summary.table', - 'suppressMessages', 'suppressPackageStartupMessages', - 'suppressWarnings', 'svd', 'sweep', 'sys.call', 'sys.calls', - 'sys.frame', 'sys.frames', 'sys.function', 'sys.load.image', - 'sys.nframe', 'sys.on.exit', 'sys.parent', 'sys.parents', - 'sys.save.image', 'sys.source', 'sys.status', 'system', - 'system.file', 'system.time', 'system2', 't', 't.data.frame', - 't.default', 'table', 'tabulate', 'tail', 'tan', 'tanh', 'tapply', - 'taskCallbackManager', 'tcrossprod', 'tempdir', 'tempfile', - 'testPlatformEquivalence', 'textConnection', 'textConnectionValue', - 'toString', 'toString.default', 'tolower', 'topenv', 'toupper', - 'trace', 'traceback', 'tracemem', 'tracingState', 'transform', - 'transform.data.frame', 'transform.default', 'trigamma', 'trunc', - 'trunc.Date', 'trunc.POSIXt', 'truncate', 'truncate.connection', - 'try', 'tryCatch', 'typeof', 'unclass', 'undebug', 'union', - 'unique', 'unique.POSIXlt', 'unique.array', 'unique.data.frame', - 'unique.default', 'unique.matrix', 'unique.numeric_version', - 'units', 'units.difftime', 'unix.time', 'unlink', 'unlist', - 'unloadNamespace', 'unlockBinding', 'unname', 'unserialize', - 'unsplit', 'untrace', 'untracemem', 'unz', 'upper.tri', 'url', - 'utf8ToInt', 'vapply', 'version', 'warning', 'warnings', 'weekdays', - 'weekdays.Date', 'weekdays.POSIXt', 'which', 'which.max', - 'which.min', 'with', 'with.default', 'withCallingHandlers', - 'withRestarts', 'withVisible', 'within', 'within.data.frame', - 'within.list', 'write', 'write.dcf', 'writeBin', 'writeChar', - 'writeLines', 'xor', 'xor.hexmode', 'xor.octmode', - 'xpdrows.data.frame', 'xtfrm', 'xtfrm.AsIs', 'xtfrm.Date', - 'xtfrm.POSIXct', 'xtfrm.POSIXlt', 'xtfrm.Surv', 'xtfrm.default', - 'xtfrm.difftime', 'xtfrm.factor', 'xtfrm.numeric_version', 'xzfile', - 'zapsmall' - ) - + valid_name = r'(?:`[^`\\]*(?:\\.[^`\\]*)*`)|(?:(?:[a-zA-z]|[_.][^0-9])[\w_.]*)' tokens = { 'comments': [ (r'#.*$', Comment.Single), ], 'valid_name': [ - (r'[a-zA-Z][\w.]*', Text), - # can begin with ., but not if that is followed by a digit - (r'\.[a-zA-Z_][\w.]*', Text), + (valid_name, Name), ], 'punctuation': [ (r'\[{1,2}|\]{1,2}|\(|\)|;|,', Punctuation), ], 'keywords': [ - (words(builtins_base, suffix=r'(?![\w. =])'), - Keyword.Pseudo), (r'(if|else|for|while|repeat|in|next|break|return|switch|function)' r'(?![\w.])', Keyword.Reserved), - (r'(array|category|character|complex|double|function|integer|list|' - r'logical|matrix|numeric|vector|data.frame|c)' - r'(?![\w.])', - Keyword.Type), - (r'(library|require|attach|detach|source)' - r'(?![\w.])', - Keyword.Namespace) ], 'operators': [ (r'<<?-|->>?|-|==|<=|>=|<|>|&&?|!=|\|\|?|\?', Operator), - (r'\*|\+|\^|/|!|%[^%]*%|=|~|\$|@|:{1,3}', Operator) + (r'\*|\+|\^|/|!|%[^%]*%|=|~|\$|@|:{1,3}', Operator), ], 'builtin_symbols': [ (r'(NULL|NA(_(integer|real|complex|character)_)?|' @@ -379,17 +118,18 @@ class SLexer(RegexLexer): include('comments'), # whitespaces (r'\s+', Text), - (r'`.*?`', String.Backtick), (r'\'', String, 'string_squote'), (r'\"', String, 'string_dquote'), include('builtin_symbols'), + include('valid_name'), include('numbers'), include('keywords'), include('punctuation'), include('operators'), - include('valid_name'), ], 'root': [ + # calls: + (r'(%s)\s*(?=\()' % valid_name, Name.Function), include('statements'), # blocks: (r'\{|\}', Punctuation), diff --git a/pygments/lexers/rdf.py b/pygments/lexers/rdf.py index d0f8778a..27bbe154 100644 --- a/pygments/lexers/rdf.py +++ b/pygments/lexers/rdf.py @@ -97,7 +97,7 @@ class SparqlLexer(RegexLexer): 'root': [ (r'\s+', Text), # keywords :: - (r'((?i)select|construct|describe|ask|where|filter|group\s+by|minus|' + (r'(?i)(select|construct|describe|ask|where|filter|group\s+by|minus|' r'distinct|reduced|from\s+named|from|order\s+by|desc|asc|limit|' r'offset|bindings|load|clear|drop|create|add|move|copy|' r'insert\s+data|delete\s+data|delete\s+where|delete|insert|' @@ -111,10 +111,10 @@ class SparqlLexer(RegexLexer): # # variables :: ('[?$]' + VARNAME, Name.Variable), # prefixed names :: - (r'(' + PN_PREFIX + ')?(\:)(' + PN_LOCAL + ')?', + (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + r')?', bygroups(Name.Namespace, Punctuation, Name.Tag)), # function names :: - (r'((?i)str|lang|langmatches|datatype|bound|iri|uri|bnode|rand|abs|' + (r'(?i)(str|lang|langmatches|datatype|bound|iri|uri|bnode|rand|abs|' r'ceil|floor|round|concat|strlen|ucase|lcase|encode_for_uri|' r'contains|strstarts|strends|strbefore|strafter|year|month|day|' r'hours|minutes|seconds|timezone|tz|now|md5|sha1|sha256|sha384|' @@ -125,7 +125,7 @@ class SparqlLexer(RegexLexer): # boolean literals :: (r'(true|false)', Keyword.Constant), # double literals :: - (r'[+\-]?(\d+\.\d*' + EXPONENT + '|\.?\d+' + EXPONENT + ')', Number.Float), + (r'[+\-]?(\d+\.\d*' + EXPONENT + r'|\.?\d+' + EXPONENT + ')', Number.Float), # decimal literals :: (r'[+\-]?(\d+\.\d*|\.\d+)', Number.Float), # integer literals :: diff --git a/pygments/lexers/rebol.py b/pygments/lexers/rebol.py index f3d00200..4d24daaa 100644 --- a/pygments/lexers/rebol.py +++ b/pygments/lexers/rebol.py @@ -102,12 +102,12 @@ class RebolLexer(RegexLexer): yield match.start(), Generic.Heading, word elif re.match("to-.*", word): yield match.start(), Keyword, word - elif re.match('(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|<>|<|>|<=|>=)$', + elif re.match(r'(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|<>|<|>|<=|>=)$', word): yield match.start(), Operator, word - elif re.match(".*\?$", word): + elif re.match(r".*\?$", word): yield match.start(), Keyword, word - elif re.match(".*\!$", word): + elif re.match(r".*\!$", word): yield match.start(), Keyword.Type, word elif re.match("'.*", word): yield match.start(), Name.Variable.Instance, word # lit-word @@ -297,10 +297,10 @@ class RedLexer(RegexLexer): yield match.start(), Keyword.Namespace, word elif re.match("to-.*", word): yield match.start(), Keyword, word - elif re.match('(\+|-\*\*|-|\*\*|//|/|\*|and|or|xor|=\?|===|==|=|<>|<=|>=|' - '<<<|>>>|<<|>>|<|>%)$', word): + elif re.match(r'(\+|-\*\*|-|\*\*|//|/|\*|and|or|xor|=\?|===|==|=|<>|<=|>=|' + r'<<<|>>>|<<|>>|<|>%)$', word): yield match.start(), Operator, word - elif re.match(".*\!$", word): + elif re.match(r".*\!$", word): yield match.start(), Keyword.Type, word elif re.match("'.*", word): yield match.start(), Name.Variable.Instance, word # lit-word diff --git a/pygments/lexers/robotframework.py b/pygments/lexers/robotframework.py index e868127b..5bacffa3 100644 --- a/pygments/lexers/robotframework.py +++ b/pygments/lexers/robotframework.py @@ -161,7 +161,7 @@ class RowTokenizer(object): class RowSplitter(object): _space_splitter = re.compile('( {2,})') - _pipe_splitter = re.compile('((?:^| +)\|(?: +|$))') + _pipe_splitter = re.compile(r'((?:^| +)\|(?: +|$))') def split(self, row): splitter = (row.startswith('| ') and self._split_from_pipes diff --git a/pygments/lexers/ruby.py b/pygments/lexers/ruby.py index fe750f1a..ce2fc7a7 100644 --- a/pygments/lexers/ruby.py +++ b/pygments/lexers/ruby.py @@ -403,8 +403,8 @@ class RubyConsoleLexer(Lexer): aliases = ['rbcon', 'irb'] mimetypes = ['text/x-ruby-shellsession'] - _prompt_re = re.compile('irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] ' - '|>> |\?> ') + _prompt_re = re.compile(r'irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] ' + r'|>> |\?> ') def get_tokens_unprocessed(self, text): rblexer = RubyLexer(**self.options) @@ -498,11 +498,11 @@ class FancyLexer(RegexLexer): (r'[a-zA-Z](\w|[-+?!=*/^><%])*:', Name.Function), # operators, must be below functions (r'[-+*/~,<>=&!?%^\[\].$]+', Operator), - ('[A-Z]\w*', Name.Constant), - ('@[a-zA-Z_]\w*', Name.Variable.Instance), - ('@@[a-zA-Z_]\w*', Name.Variable.Class), + (r'[A-Z]\w*', Name.Constant), + (r'@[a-zA-Z_]\w*', Name.Variable.Instance), + (r'@@[a-zA-Z_]\w*', Name.Variable.Class), ('@@?', Operator), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), # numbers - / checks are necessary to avoid mismarking regexes, # see comment in RubyLexer (r'(0[oO]?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?', diff --git a/pygments/lexers/rust.py b/pygments/lexers/rust.py index 6914f54d..10097fba 100644 --- a/pygments/lexers/rust.py +++ b/pygments/lexers/rust.py @@ -24,7 +24,7 @@ class RustLexer(RegexLexer): """ name = 'Rust' filenames = ['*.rs', '*.rs.in'] - aliases = ['rust'] + aliases = ['rust', 'rs'] mimetypes = ['text/rust'] keyword_types = ( diff --git a/pygments/lexers/scripting.py b/pygments/lexers/scripting.py index b3af606e..28c37db8 100644 --- a/pygments/lexers/scripting.py +++ b/pygments/lexers/scripting.py @@ -104,7 +104,7 @@ class LuaLexer(RegexLexer): (r'%s(?=%s*[.:])' % (_name, _s), Name.Class), (_name, Name.Function, '#pop'), # inline function - ('\(', Punctuation, '#pop'), + (r'\(', Punctuation, '#pop'), ], 'goto': [ @@ -696,8 +696,8 @@ class AppleScriptLexer(RegexLexer): (r'[-+]?\d+', Number.Integer), ], 'comment': [ - ('\(\*', Comment.Multiline, '#push'), - ('\*\)', Comment.Multiline, '#pop'), + (r'\(\*', Comment.Multiline, '#push'), + (r'\*\)', Comment.Multiline, '#pop'), ('[^*(]+', Comment.Multiline), ('[*(]', Comment.Multiline), ], diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py index ceb6f14d..68150811 100644 --- a/pygments/lexers/shell.py +++ b/pygments/lexers/shell.py @@ -638,13 +638,29 @@ class PowerShellLexer(RegexLexer): 'wildcard').split() verbs = ( - 'write where wait use update unregister undo trace test tee take ' - 'suspend stop start split sort skip show set send select scroll resume ' - 'restore restart resolve resize reset rename remove register receive ' - 'read push pop ping out new move measure limit join invoke import ' - 'group get format foreach export expand exit enter enable disconnect ' - 'disable debug cxnew copy convertto convertfrom convert connect ' - 'complete compare clear checkpoint aggregate add').split() + 'write where watch wait use update unregister unpublish unprotect ' + 'unlock uninstall undo unblock trace test tee take sync switch ' + 'suspend submit stop step start split sort skip show set send select ' + 'search scroll save revoke resume restore restart resolve resize ' + 'reset request repair rename remove register redo receive read push ' + 'publish protect pop ping out optimize open new move mount merge ' + 'measure lock limit join invoke install initialize import hide group ' + 'grant get format foreach find export expand exit enter enable edit ' + 'dismount disconnect disable deny debug cxnew copy convertto ' + 'convertfrom convert connect confirm compress complete compare close ' + 'clear checkpoint block backup assert approve aggregate add').split() + + aliases = ( + 'ac asnp cat cd cfs chdir clc clear clhy cli clp cls clv cnsn ' + 'compare copy cp cpi cpp curl cvpa dbp del diff dir dnsn ebp echo epal ' + 'epcsv epsn erase etsn exsn fc fhx fl foreach ft fw gal gbp gc gci gcm ' + 'gcs gdr ghy gi gjb gl gm gmo gp gps gpv group gsn gsnp gsv gu gv gwmi ' + 'h history icm iex ihy ii ipal ipcsv ipmo ipsn irm ise iwmi iwr kill lp ' + 'ls man md measure mi mount move mp mv nal ndr ni nmo npssc nsn nv ogv ' + 'oh popd ps pushd pwd r rbp rcjb rcsn rd rdr ren ri rjb rm rmdir rmo ' + 'rni rnp rp rsn rsnp rujb rv rvpa rwmi sajb sal saps sasv sbp sc select ' + 'set shcm si sl sleep sls sort sp spjb spps spsv start sujb sv swmi tee ' + 'trcm type wget where wjb write').split() commenthelp = ( 'component description example externalhelp forwardhelpcategory ' @@ -672,6 +688,7 @@ class PowerShellLexer(RegexLexer): (r'(%s)\b' % '|'.join(keywords), Keyword), (r'-(%s)\b' % '|'.join(operators), Operator), (r'(%s)-[a-z_]\w*\b' % '|'.join(verbs), Name.Builtin), + (r'(%s)\s' % '|'.join(aliases), Name.Builtin), (r'\[[a-z_\[][\w. `,\[\]]*\]', Name.Constant), # .net [type]s (r'-[a-z_]\w*', Name), (r'\w+', Name), diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index 7507c0fc..7dd856b2 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -155,7 +155,7 @@ class PostgresLexer(PostgresBase, RegexLexer): (r'\s+', Text), (r'--.*\n?', Comment.Single), (r'/\*', Comment.Multiline, 'multiline-comments'), - (r'(' + '|'.join(s.replace(" ", "\s+") + (r'(' + '|'.join(s.replace(" ", r"\s+") for s in DATATYPES + PSEUDO_TYPES) + r')\b', Name.Builtin), (words(KEYWORDS, suffix=r'\b'), Keyword), @@ -308,14 +308,7 @@ class PostgresConsoleLexer(Lexer): # and continue until the end of command is detected curcode = '' insertions = [] - while 1: - try: - line = next(lines) - except StopIteration: - # allow the emission of partially collected items - # the repl loop will be broken below - break - + for line in lines: # Identify a shell prompt in case of psql commandline example if line.startswith('$') and not curcode: lexer = get_lexer_by_name('console', **self.options) @@ -346,8 +339,7 @@ class PostgresConsoleLexer(Lexer): # Emit the output lines out_token = Generic.Output - while 1: - line = next(lines) + for line in lines: mprompt = re_prompt.match(line) if mprompt is not None: # push the line back to have it processed by the prompt @@ -363,6 +355,8 @@ class PostgresConsoleLexer(Lexer): yield (mmsg.start(2), out_token, mmsg.group(2)) else: yield (0, out_token, line) + else: + return class SqlLexer(RegexLexer): @@ -499,7 +493,7 @@ class TransactSqlLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Whitespace), - (r'--(?m).*?$\n?', Comment.Single), + (r'(?m)--.*?$\n?', Comment.Single), (r'/\*', Comment.Multiline, 'multiline-comments'), (words(_tsql_builtins.OPERATORS), Operator), (words(_tsql_builtins.OPERATOR_WORDS, suffix=r'\b'), Operator.Word), diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py index 83c57db8..c184b2dd 100644 --- a/pygments/lexers/templates.py +++ b/pygments/lexers/templates.py @@ -187,13 +187,13 @@ class SmartyLexer(RegexLexer): def analyse_text(text): rv = 0.0 - if re.search('\{if\s+.*?\}.*?\{/if\}', text): + if re.search(r'\{if\s+.*?\}.*?\{/if\}', text): rv += 0.15 - if re.search('\{include\s+file=.*?\}', text): + if re.search(r'\{include\s+file=.*?\}', text): rv += 0.15 - if re.search('\{foreach\s+.*?\}.*?\{/foreach\}', text): + if re.search(r'\{foreach\s+.*?\}.*?\{/foreach\}', text): rv += 0.15 - if re.search('\{\$.*?\}', text): + if re.search(r'\{\$.*?\}', text): rv += 0.01 return rv @@ -421,18 +421,18 @@ class MyghtyLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (r'(<%(?:def|method))(\s*)(.*?)(>)(.*?)(</%\2\s*>)(?s)', + (r'(?s)(<%(?:def|method))(\s*)(.*?)(>)(.*?)(</%\2\s*>)', bygroups(Name.Tag, Text, Name.Function, Name.Tag, using(this), Name.Tag)), - (r'(<%\w+)(.*?)(>)(.*?)(</%\2\s*>)(?s)', + (r'(?s)(<%\w+)(.*?)(>)(.*?)(</%\2\s*>)', bygroups(Name.Tag, Name.Function, Name.Tag, using(PythonLexer), Name.Tag)), (r'(<&[^|])(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PythonLexer), Name.Tag)), - (r'(<&\|)(.*?)(,.*?)?(&>)(?s)', + (r'(?s)(<&\|)(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PythonLexer), Name.Tag)), (r'</&>', Name.Tag), - (r'(<%!?)(.*?)(%>)(?s)', + (r'(?s)(<%!?)(.*?)(%>)', bygroups(Name.Tag, using(PythonLexer), Name.Tag)), (r'(?<=^)#[^\n]*(\n|\Z)', Comment), (r'(?<=^)(%)([^\n]*)(\n|\Z)', @@ -538,20 +538,20 @@ class MasonLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (r'(<%doc>)(.*?)(</%doc>)(?s)', + (r'(?s)(<%doc>)(.*?)(</%doc>)', bygroups(Name.Tag, Comment.Multiline, Name.Tag)), - (r'(<%(?:def|method))(\s*)(.*?)(>)(.*?)(</%\2\s*>)(?s)', + (r'(?s)(<%(?:def|method))(\s*)(.*?)(>)(.*?)(</%\2\s*>)', bygroups(Name.Tag, Text, Name.Function, Name.Tag, using(this), Name.Tag)), - (r'(<%\w+)(.*?)(>)(.*?)(</%\2\s*>)(?s)', + (r'(?s)(<%\w+)(.*?)(>)(.*?)(</%\2\s*>)', bygroups(Name.Tag, Name.Function, Name.Tag, using(PerlLexer), Name.Tag)), - (r'(<&[^|])(.*?)(,.*?)?(&>)(?s)', + (r'(?s)(<&[^|])(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PerlLexer), Name.Tag)), - (r'(<&\|)(.*?)(,.*?)?(&>)(?s)', + (r'(?s)(<&\|)(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PerlLexer), Name.Tag)), (r'</&>', Name.Tag), - (r'(<%!?)(.*?)(%>)(?s)', + (r'(?s)(<%!?)(.*?)(%>)', bygroups(Name.Tag, using(PerlLexer), Name.Tag)), (r'(?<=^)#[^\n]*(\n|\Z)', Comment), (r'(?<=^)(%)([^\n]*)(\n|\Z)', @@ -607,7 +607,7 @@ class MakoLexer(RegexLexer): (r'(</%)([\w.:]+)(>)', bygroups(Comment.Preproc, Name.Builtin, Comment.Preproc)), (r'<%(?=([\w.:]+))', Comment.Preproc, 'ondeftags'), - (r'(<%(?:!?))(.*?)(%>)(?s)', + (r'(?s)(<%(?:!?))(.*?)(%>)', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), (r'(\$\{)(.*?)(\})', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), @@ -759,7 +759,7 @@ class CheetahLexer(RegexLexer): # TODO support other Python syntax like $foo['bar'] (r'(\$)([a-zA-Z_][\w.]*\w)', bygroups(Comment.Preproc, using(CheetahPythonLexer))), - (r'(\$\{!?)(.*?)(\})(?s)', + (r'(?s)(\$\{!?)(.*?)(\})', bygroups(Comment.Preproc, using(CheetahPythonLexer), Comment.Preproc)), (r'''(?sx) @@ -942,9 +942,9 @@ class HtmlGenshiLexer(DelegatingLexer): def analyse_text(text): rv = 0.0 - if re.search('\$\{.*?\}', text) is not None: + if re.search(r'\$\{.*?\}', text) is not None: rv += 0.2 - if re.search('py:(.*?)=["\']', text) is not None: + if re.search(r'py:(.*?)=["\']', text) is not None: rv += 0.2 return rv + HtmlLexer.analyse_text(text) - 0.01 @@ -967,9 +967,9 @@ class GenshiLexer(DelegatingLexer): def analyse_text(text): rv = 0.0 - if re.search('\$\{.*?\}', text) is not None: + if re.search(r'\$\{.*?\}', text) is not None: rv += 0.2 - if re.search('py:(.*?)=["\']', text) is not None: + if re.search(r'py:(.*?)=["\']', text) is not None: rv += 0.2 return rv + XmlLexer.analyse_text(text) - 0.01 @@ -1627,7 +1627,7 @@ class SspLexer(DelegatingLexer): def analyse_text(text): rv = 0.0 - if re.search('val \w+\s*:', text): + if re.search(r'val \w+\s*:', text): rv += 0.6 if looks_like_xml(text): rv += 0.2 @@ -1955,7 +1955,7 @@ class LiquidLexer(RegexLexer): 'output': [ include('whitespace'), - ('\}\}', Punctuation, '#pop'), # end of output + (r'\}\}', Punctuation, '#pop'), # end of output (r'\|', Punctuation, 'filters') ], diff --git a/pygments/lexers/testing.py b/pygments/lexers/testing.py index 1e0795b1..86e60f25 100644 --- a/pygments/lexers/testing.py +++ b/pygments/lexers/testing.py @@ -29,7 +29,7 @@ class GherkinLexer(RegexLexer): feature_keywords = u'^(기능|機能|功能|フィーチャ|خاصية|תכונה|Функціонал|Функционалност|Функционал|Фича|Особина|Могућност|Özellik|Właściwość|Tính năng|Trajto|Savybė|Požiadavka|Požadavek|Osobina|Ominaisuus|Omadus|OH HAI|Mogućnost|Mogucnost|Jellemző|Fīča|Funzionalità|Funktionalität|Funkcionalnost|Funkcionalitāte|Funcționalitate|Functionaliteit|Functionalitate|Funcionalitat|Funcionalidade|Fonctionnalité|Fitur|Feature|Egenskap|Egenskab|Crikey|Característica|Arwedd)(:)(.*)$' feature_element_keywords = u'^(\\s*)(시나리오 개요|시나리오|배경|背景|場景大綱|場景|场景大纲|场景|劇本大綱|劇本|剧本大纲|剧本|テンプレ|シナリオテンプレート|シナリオテンプレ|シナリオアウトライン|シナリオ|سيناريو مخطط|سيناريو|الخلفية|תרחיש|תבנית תרחיש|רקע|Тарих|Сценарій|Сценарио|Сценарий структураси|Сценарий|Структура сценарію|Структура сценарија|Структура сценария|Скица|Рамка на сценарий|Пример|Предыстория|Предистория|Позадина|Передумова|Основа|Концепт|Контекст|Założenia|Wharrimean is|Tình huống|The thing of it is|Tausta|Taust|Tapausaihio|Tapaus|Szenariogrundriss|Szenario|Szablon scenariusza|Stsenaarium|Struktura scenarija|Skica|Skenario konsep|Skenario|Situācija|Senaryo taslağı|Senaryo|Scénář|Scénario|Schema dello scenario|Scenārijs pēc parauga|Scenārijs|Scenár|Scenaro|Scenariusz|Scenariul de şablon|Scenariul de sablon|Scenariu|Scenario Outline|Scenario Amlinellol|Scenario|Scenarijus|Scenarijaus šablonas|Scenarij|Scenarie|Rerefons|Raamstsenaarium|Primer|Pozadí|Pozadina|Pozadie|Plan du scénario|Plan du Scénario|Osnova scénáře|Osnova|Náčrt Scénáře|Náčrt Scenáru|Mate|MISHUN SRSLY|MISHUN|Kịch bản|Konturo de la scenaro|Kontext|Konteksts|Kontekstas|Kontekst|Koncept|Khung tình huống|Khung kịch bản|Háttér|Grundlage|Geçmiş|Forgatókönyv vázlat|Forgatókönyv|Fono|Esquema do Cenário|Esquema do Cenario|Esquema del escenario|Esquema de l\'escenari|Escenario|Escenari|Dis is what went down|Dasar|Contexto|Contexte|Contesto|Condiţii|Conditii|Cenário|Cenario|Cefndir|Bối cảnh|Blokes|Bakgrunn|Bakgrund|Baggrund|Background|B4|Antecedents|Antecedentes|All y\'all|Achtergrond|Abstrakt Scenario|Abstract Scenario)(:)(.*)$' examples_keywords = u'^(\\s*)(예|例子|例|サンプル|امثلة|דוגמאות|Сценарији|Примери|Приклади|Мисоллар|Значения|Örnekler|Voorbeelden|Variantai|Tapaukset|Scenarios|Scenariji|Scenarijai|Příklady|Példák|Príklady|Przykłady|Primjeri|Primeri|Piemēri|Pavyzdžiai|Paraugs|Juhtumid|Exemplos|Exemples|Exemplele|Exempel|Examples|Esempi|Enghreifftiau|Ekzemploj|Eksempler|Ejemplos|EXAMPLZ|Dữ liệu|Contoh|Cobber|Beispiele)(:)(.*)$' - step_keywords = u'^(\\s*)(하지만|조건|먼저|만일|만약|단|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假设|假如|假定|但是|但し|並且|并且|同時|同时|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Якщо |Унда |То |Припустимо, що |Припустимо |Онда |Но |Нехай |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Але |Агар |А |І |Și |És |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Youse know when youse got |Youse know like when |Yna |Ya know how |Ya gotta |Y |Wun |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Stel |Soit |Siis |Si |Sed |Se |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kaj |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givun |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Dun |Donitaĵo |Donat |Donada |Do |Diyelim ki |Dengan |Den youse gotta |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |Cal |But y\'all |But |Buh |Biết |Bet |BUT |Atès |Atunci |Atesa |Anrhegedig a |Angenommen |And y\'all |And |An |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\* )' + step_keywords = u'^(\\s*)(하지만|조건|먼저|만일|만약|단|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假设|假如|假定|但是|但し|並且|并且|同時|同时|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Якщо |Унда |То |Припустимо, що |Припустимо |Онда |Но |Нехай |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Але |Агар |А |І |Și |És |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Youse know when youse got |Youse know like when |Yna |Ya know how |Ya gotta |Y |Wun |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Stel |Soit |Siis |Si |Sed |Se |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kaj |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givun |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Dun |Donitaĵo |Donat |Donada |Do |Diyelim ki |Dengan |Den youse gotta |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |Cal |But y\'all |But |Buh |Biết |Bet |BUT |Atès |Atunci |Atesa |Anrhegedig a |Angenommen |And y\'all |And |An |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\\* )' tokens = { 'comments': [ diff --git a/pygments/lexers/textfmts.py b/pygments/lexers/textfmts.py index bb8124ef..b70c2ad6 100644 --- a/pygments/lexers/textfmts.py +++ b/pygments/lexers/textfmts.py @@ -266,7 +266,7 @@ class TodotxtLexer(RegexLexer): # 5. Leading project (project_regex, Project, 'incomplete'), # 6. Non-whitespace catch-all - ('\S+', IncompleteTaskText, 'incomplete'), + (r'\S+', IncompleteTaskText, 'incomplete'), ], # Parse a complete task @@ -277,9 +277,9 @@ class TodotxtLexer(RegexLexer): (context_regex, Context), (project_regex, Project), # Tokenize non-whitespace text - ('\S+', CompleteTaskText), + (r'\S+', CompleteTaskText), # Tokenize whitespace not containing a newline - ('\s+', CompleteTaskText), + (r'\s+', CompleteTaskText), ], # Parse an incomplete task @@ -290,8 +290,8 @@ class TodotxtLexer(RegexLexer): (context_regex, Context), (project_regex, Project), # Tokenize non-whitespace text - ('\S+', IncompleteTaskText), + (r'\S+', IncompleteTaskText), # Tokenize whitespace not containing a newline - ('\s+', IncompleteTaskText), + (r'\s+', IncompleteTaskText), ], } diff --git a/pygments/lexers/typoscript.py b/pygments/lexers/typoscript.py index e358af07..7da87c75 100644 --- a/pygments/lexers/typoscript.py +++ b/pygments/lexers/typoscript.py @@ -132,7 +132,7 @@ class TypoScriptLexer(RegexLexer): ], 'keywords': [ # Conditions - (r'(\[)(?i)(browser|compatVersion|dayofmonth|dayofweek|dayofyear|' + (r'(?i)(\[)(browser|compatVersion|dayofmonth|dayofweek|dayofyear|' r'device|ELSE|END|GLOBAL|globalString|globalVar|hostname|hour|IP|' r'language|loginUser|loginuser|minute|month|page|PIDinRootline|' r'PIDupinRootline|system|treeLevel|useragent|userFunc|usergroup|' @@ -172,7 +172,7 @@ class TypoScriptLexer(RegexLexer): 'html': [ (r'<\S[^\n>]*>', using(TypoScriptHtmlDataLexer)), (r'&[^;\n]*;', String), - (r'(_CSS_DEFAULT_STYLE)(\s*)(\()(?s)(.*(?=\n\)))', + (r'(?s)(_CSS_DEFAULT_STYLE)(\s*)(\()(.*(?=\n\)))', bygroups(Name.Class, Text, String.Symbol, using(TypoScriptCssDataLexer))), ], 'literal': [ diff --git a/pygments/lexers/varnish.py b/pygments/lexers/varnish.py index 44521422..f3b37d60 100644 --- a/pygments/lexers/varnish.py +++ b/pygments/lexers/varnish.py @@ -36,7 +36,7 @@ class VCLLexer(RegexLexer): # Skip over comments and blank lines # This is accurate enough that returning 0.9 is reasonable. # Almost no VCL files start without some comments. - elif '\nvcl 4\.0;' in text[:1000]: + elif '\nvcl 4.0;' in text[:1000]: return 0.9 tokens = { @@ -120,7 +120,7 @@ class VCLLexer(RegexLexer): r'([a-zA-Z_]\w*)' r'(\s*\(.*\))', bygroups(Name.Function, Punctuation, Name.Function, using(this))), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'comment': [ (r'[^*/]+', Comment.Multiline), diff --git a/pygments/lexers/webmisc.py b/pygments/lexers/webmisc.py index 712c8246..67aefe23 100644 --- a/pygments/lexers/webmisc.py +++ b/pygments/lexers/webmisc.py @@ -438,7 +438,7 @@ class XQueryLexer(ExtendedRegexLexer): ], 'varname': [ (r'\(:', Comment, 'comment'), - (r'(' + qname + ')(\()?', bygroups(Name, Punctuation), 'operator'), + (r'(' + qname + r')(\()?', bygroups(Name, Punctuation), 'operator'), ], 'singletype': [ include('whitespace'), @@ -643,9 +643,9 @@ class XQueryLexer(ExtendedRegexLexer): bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'operator'), (r'(declare)(\s+)(context)(\s+)(item)', bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'operator'), - (ncname + ':\*', Name, 'operator'), - ('\*:'+ncname, Name.Tag, 'operator'), - ('\*', Name.Tag, 'operator'), + (ncname + r':\*', Name, 'operator'), + (r'\*:'+ncname, Name.Tag, 'operator'), + (r'\*', Name.Tag, 'operator'), (stringdouble, String.Double, 'operator'), (stringsingle, String.Single, 'operator'), @@ -661,7 +661,8 @@ class XQueryLexer(ExtendedRegexLexer): # NAMESPACE KEYWORD (r'(declare)(\s+)(default)(\s+)(element|function)', - bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'namespacekeyword'), + bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), + 'namespacekeyword'), (r'(import)(\s+)(schema|module)', bygroups(Keyword.Pseudo, Text, Keyword.Pseudo), 'namespacekeyword'), (r'(declare)(\s+)(copy-namespaces)', @@ -861,7 +862,7 @@ class QmlLexer(RegexLexer): class CirruLexer(RegexLexer): - """ + r""" Syntax rules of Cirru can be found at: http://cirru.org/ diff --git a/pygments/lexers/xorg.py b/pygments/lexers/xorg.py index 89475a80..3bba930f 100644 --- a/pygments/lexers/xorg.py +++ b/pygments/lexers/xorg.py @@ -16,6 +16,7 @@ __all__ = ['XorgLexer'] class XorgLexer(RegexLexer): + """Lexer for xorg.conf file.""" name = 'Xorg' aliases = ['xorg.conf'] filenames = ['xorg.conf'] @@ -26,8 +27,8 @@ class XorgLexer(RegexLexer): (r'\s+', Text), (r'#.*$', Comment), - (r'((|Sub)Section)(\s+)("\w+")', - bygroups(String.Escape, String.Escape, Text, String.Escape)), + (r'((?:Sub)?Section)(\s+)("\w+")', + bygroups(String.Escape, Text, String.Escape)), (r'(End(|Sub)Section)', String.Escape), (r'(\w+)(\s+)([^\n#]+)', diff --git a/pygments/plugin.py b/pygments/plugin.py index 7987d646..08d9b5b4 100644 --- a/pygments/plugin.py +++ b/pygments/plugin.py @@ -40,14 +40,16 @@ FORMATTER_ENTRY_POINT = 'pygments.formatters' STYLE_ENTRY_POINT = 'pygments.styles' FILTER_ENTRY_POINT = 'pygments.filters' + def iter_entry_points(group_name): try: import pkg_resources - except ImportError: + except (ImportError, IOError): return [] return pkg_resources.iter_entry_points(group_name) + def find_plugin_lexers(): for entrypoint in iter_entry_points(LEXER_ENTRY_POINT): yield entrypoint.load() diff --git a/pygments/sphinxext.py b/pygments/sphinxext.py index f962f8c6..5c8ac8b9 100644 --- a/pygments/sphinxext.py +++ b/pygments/sphinxext.py @@ -16,7 +16,7 @@ import sys from docutils import nodes from docutils.statemachine import ViewList -from sphinx.util.compat import Directive +from docutils.parsers.rst import Directive from sphinx.util.nodes import nested_parse_with_titles diff --git a/pygments/unistring.py b/pygments/unistring.py index 6096d110..ba0e41de 100644 --- a/pygments/unistring.py +++ b/pygments/unistring.py @@ -16,9 +16,9 @@ import sys Cc = u'\x00-\x1f\x7f-\x9f' -Cf = u'\xad\u0600-\u0604\u061c\u06dd\u070f\u180e\u200b-\u200f\u202a-\u202e\u2060-\u2064\u2066-\u206f\ufeff\ufff9-\ufffb' +Cf = u'\xad\u0600-\u0605\u061c\u06dd\u070f\u08e2\u180e\u200b-\u200f\u202a-\u202e\u2060-\u2064\u2066-\u206f\ufeff\ufff9-\ufffb' -Cn = u'\u0378-\u0379\u037f-\u0383\u038b\u038d\u03a2\u0528-\u0530\u0557-\u0558\u0560\u0588\u058b-\u058e\u0590\u05c8-\u05cf\u05eb-\u05ef\u05f5-\u05ff\u0605\u061d\u070e\u074b-\u074c\u07b2-\u07bf\u07fb-\u07ff\u082e-\u082f\u083f\u085c-\u085d\u085f-\u089f\u08a1\u08ad-\u08e3\u08ff\u0978\u0980\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09c5-\u09c6\u09c9-\u09ca\u09cf-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09fc-\u0a00\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a50\u0a52-\u0a58\u0a5d\u0a5f-\u0a65\u0a76-\u0a80\u0a84\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0acf\u0ad1-\u0adf\u0ae4-\u0ae5\u0af2-\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34\u0b3a-\u0b3b\u0b45-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b64-\u0b65\u0b78-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bcf\u0bd1-\u0bd6\u0bd8-\u0be5\u0bfb-\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c3c\u0c45\u0c49\u0c4e-\u0c54\u0c57\u0c5a-\u0c5f\u0c64-\u0c65\u0c70-\u0c77\u0c80-\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbb\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce4-\u0ce5\u0cf0\u0cf3-\u0d01\u0d04\u0d0d\u0d11\u0d3b-\u0d3c\u0d45\u0d49\u0d4f-\u0d56\u0d58-\u0d5f\u0d64-\u0d65\u0d76-\u0d78\u0d80-\u0d81\u0d84\u0d97-\u0d99\u0db2\u0dbc\u0dbe-\u0dbf\u0dc7-\u0dc9\u0dcb-\u0dce\u0dd5\u0dd7\u0de0-\u0df1\u0df5-\u0e00\u0e3b-\u0e3e\u0e5c-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0edb\u0ee0-\u0eff\u0f48\u0f6d-\u0f70\u0f98\u0fbd\u0fcd\u0fdb-\u0fff\u10c6\u10c8-\u10cc\u10ce-\u10cf\u1249\u124e-\u124f\u1257\u1259\u125e-\u125f\u1289\u128e-\u128f\u12b1\u12b6-\u12b7\u12bf\u12c1\u12c6-\u12c7\u12d7\u1311\u1316-\u1317\u135b-\u135c\u137d-\u137f\u139a-\u139f\u13f5-\u13ff\u169d-\u169f\u16f1-\u16ff\u170d\u1715-\u171f\u1737-\u173f\u1754-\u175f\u176d\u1771\u1774-\u177f\u17de-\u17df\u17ea-\u17ef\u17fa-\u17ff\u180f\u181a-\u181f\u1878-\u187f\u18ab-\u18af\u18f6-\u18ff\u191d-\u191f\u192c-\u192f\u193c-\u193f\u1941-\u1943\u196e-\u196f\u1975-\u197f\u19ac-\u19af\u19ca-\u19cf\u19db-\u19dd\u1a1c-\u1a1d\u1a5f\u1a7d-\u1a7e\u1a8a-\u1a8f\u1a9a-\u1a9f\u1aae-\u1aff\u1b4c-\u1b4f\u1b7d-\u1b7f\u1bf4-\u1bfb\u1c38-\u1c3a\u1c4a-\u1c4c\u1c80-\u1cbf\u1cc8-\u1ccf\u1cf7-\u1cff\u1de7-\u1dfb\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fc5\u1fd4-\u1fd5\u1fdc\u1ff0-\u1ff1\u1ff5\u1fff\u2065\u2072-\u2073\u208f\u209d-\u209f\u20bb-\u20cf\u20f1-\u20ff\u218a-\u218f\u23f4-\u23ff\u2427-\u243f\u244b-\u245f\u2700\u2b4d-\u2b4f\u2b5a-\u2bff\u2c2f\u2c5f\u2cf4-\u2cf8\u2d26\u2d28-\u2d2c\u2d2e-\u2d2f\u2d68-\u2d6e\u2d71-\u2d7e\u2d97-\u2d9f\u2da7\u2daf\u2db7\u2dbf\u2dc7\u2dcf\u2dd7\u2ddf\u2e3c-\u2e7f\u2e9a\u2ef4-\u2eff\u2fd6-\u2fef\u2ffc-\u2fff\u3040\u3097-\u3098\u3100-\u3104\u312e-\u3130\u318f\u31bb-\u31bf\u31e4-\u31ef\u321f\u32ff\u4db6-\u4dbf\u9fcd-\u9fff\ua48d-\ua48f\ua4c7-\ua4cf\ua62c-\ua63f\ua698-\ua69e\ua6f8-\ua6ff\ua78f\ua794-\ua79f\ua7ab-\ua7f7\ua82c-\ua82f\ua83a-\ua83f\ua878-\ua87f\ua8c5-\ua8cd\ua8da-\ua8df\ua8fc-\ua8ff\ua954-\ua95e\ua97d-\ua97f\ua9ce\ua9da-\ua9dd\ua9e0-\ua9ff\uaa37-\uaa3f\uaa4e-\uaa4f\uaa5a-\uaa5b\uaa7c-\uaa7f\uaac3-\uaada\uaaf7-\uab00\uab07-\uab08\uab0f-\uab10\uab17-\uab1f\uab27\uab2f-\uabbf\uabee-\uabef\uabfa-\uabff\ud7a4-\ud7af\ud7c7-\ud7ca\ud7fc-\ud7ff\ufa6e-\ufa6f\ufada-\ufaff\ufb07-\ufb12\ufb18-\ufb1c\ufb37\ufb3d\ufb3f\ufb42\ufb45\ufbc2-\ufbd2\ufd40-\ufd4f\ufd90-\ufd91\ufdc8-\ufdef\ufdfe-\ufdff\ufe1a-\ufe1f\ufe27-\ufe2f\ufe53\ufe67\ufe6c-\ufe6f\ufe75\ufefd-\ufefe\uff00\uffbf-\uffc1\uffc8-\uffc9\uffd0-\uffd1\uffd8-\uffd9\uffdd-\uffdf\uffe7\uffef-\ufff8\ufffe-\uffff' +Cn = u'\u0378-\u0379\u0380-\u0383\u038b\u038d\u03a2\u0530\u0557-\u0558\u058b-\u058c\u0590\u05c8-\u05cf\u05eb-\u05ee\u05f5-\u05ff\u061d\u070e\u074b-\u074c\u07b2-\u07bf\u07fb-\u07fc\u082e-\u082f\u083f\u085c-\u085d\u085f\u086b-\u089f\u08b5\u08be-\u08d2\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09c5-\u09c6\u09c9-\u09ca\u09cf-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09ff-\u0a00\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a50\u0a52-\u0a58\u0a5d\u0a5f-\u0a65\u0a77-\u0a80\u0a84\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0acf\u0ad1-\u0adf\u0ae4-\u0ae5\u0af2-\u0af8\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34\u0b3a-\u0b3b\u0b45-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b64-\u0b65\u0b78-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bcf\u0bd1-\u0bd6\u0bd8-\u0be5\u0bfb-\u0bff\u0c0d\u0c11\u0c29\u0c3a-\u0c3c\u0c45\u0c49\u0c4e-\u0c54\u0c57\u0c5b-\u0c5f\u0c64-\u0c65\u0c70-\u0c77\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbb\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce4-\u0ce5\u0cf0\u0cf3-\u0cff\u0d04\u0d0d\u0d11\u0d45\u0d49\u0d50-\u0d53\u0d64-\u0d65\u0d80-\u0d81\u0d84\u0d97-\u0d99\u0db2\u0dbc\u0dbe-\u0dbf\u0dc7-\u0dc9\u0dcb-\u0dce\u0dd5\u0dd7\u0de0-\u0de5\u0df0-\u0df1\u0df5-\u0e00\u0e3b-\u0e3e\u0e5c-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0edb\u0ee0-\u0eff\u0f48\u0f6d-\u0f70\u0f98\u0fbd\u0fcd\u0fdb-\u0fff\u10c6\u10c8-\u10cc\u10ce-\u10cf\u1249\u124e-\u124f\u1257\u1259\u125e-\u125f\u1289\u128e-\u128f\u12b1\u12b6-\u12b7\u12bf\u12c1\u12c6-\u12c7\u12d7\u1311\u1316-\u1317\u135b-\u135c\u137d-\u137f\u139a-\u139f\u13f6-\u13f7\u13fe-\u13ff\u169d-\u169f\u16f9-\u16ff\u170d\u1715-\u171f\u1737-\u173f\u1754-\u175f\u176d\u1771\u1774-\u177f\u17de-\u17df\u17ea-\u17ef\u17fa-\u17ff\u180f\u181a-\u181f\u1879-\u187f\u18ab-\u18af\u18f6-\u18ff\u191f\u192c-\u192f\u193c-\u193f\u1941-\u1943\u196e-\u196f\u1975-\u197f\u19ac-\u19af\u19ca-\u19cf\u19db-\u19dd\u1a1c-\u1a1d\u1a5f\u1a7d-\u1a7e\u1a8a-\u1a8f\u1a9a-\u1a9f\u1aae-\u1aaf\u1abf-\u1aff\u1b4c-\u1b4f\u1b7d-\u1b7f\u1bf4-\u1bfb\u1c38-\u1c3a\u1c4a-\u1c4c\u1c89-\u1c8f\u1cbb-\u1cbc\u1cc8-\u1ccf\u1cfa-\u1cff\u1dfa\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fc5\u1fd4-\u1fd5\u1fdc\u1ff0-\u1ff1\u1ff5\u1fff\u2065\u2072-\u2073\u208f\u209d-\u209f\u20c0-\u20cf\u20f1-\u20ff\u218c-\u218f\u2427-\u243f\u244b-\u245f\u2b74-\u2b75\u2b96-\u2b97\u2bc9\u2bff\u2c2f\u2c5f\u2cf4-\u2cf8\u2d26\u2d28-\u2d2c\u2d2e-\u2d2f\u2d68-\u2d6e\u2d71-\u2d7e\u2d97-\u2d9f\u2da7\u2daf\u2db7\u2dbf\u2dc7\u2dcf\u2dd7\u2ddf\u2e4f-\u2e7f\u2e9a\u2ef4-\u2eff\u2fd6-\u2fef\u2ffc-\u2fff\u3040\u3097-\u3098\u3100-\u3104\u3130\u318f\u31bb-\u31bf\u31e4-\u31ef\u321f\u32ff\u4db6-\u4dbf\u9ff0-\u9fff\ua48d-\ua48f\ua4c7-\ua4cf\ua62c-\ua63f\ua6f8-\ua6ff\ua7ba-\ua7f6\ua82c-\ua82f\ua83a-\ua83f\ua878-\ua87f\ua8c6-\ua8cd\ua8da-\ua8df\ua954-\ua95e\ua97d-\ua97f\ua9ce\ua9da-\ua9dd\ua9ff\uaa37-\uaa3f\uaa4e-\uaa4f\uaa5a-\uaa5b\uaac3-\uaada\uaaf7-\uab00\uab07-\uab08\uab0f-\uab10\uab17-\uab1f\uab27\uab2f\uab66-\uab6f\uabee-\uabef\uabfa-\uabff\ud7a4-\ud7af\ud7c7-\ud7ca\ud7fc-\ud7ff\ufa6e-\ufa6f\ufada-\ufaff\ufb07-\ufb12\ufb18-\ufb1c\ufb37\ufb3d\ufb3f\ufb42\ufb45\ufbc2-\ufbd2\ufd40-\ufd4f\ufd90-\ufd91\ufdc8-\ufdef\ufdfe-\ufdff\ufe1a-\ufe1f\ufe53\ufe67\ufe6c-\ufe6f\ufe75\ufefd-\ufefe\uff00\uffbf-\uffc1\uffc8-\uffc9\uffd0-\uffd1\uffd8-\uffd9\uffdd-\uffdf\uffe7\uffef-\ufff8\ufffe-\uffff' Co = u'\ue000-\uf8ff' @@ -27,49 +27,49 @@ try: except UnicodeDecodeError: Cs = '' # Jython can't handle isolated surrogates -Ll = u'a-z\xb5\xdf-\xf6\xf8-\xff\u0101\u0103\u0105\u0107\u0109\u010b\u010d\u010f\u0111\u0113\u0115\u0117\u0119\u011b\u011d\u011f\u0121\u0123\u0125\u0127\u0129\u012b\u012d\u012f\u0131\u0133\u0135\u0137-\u0138\u013a\u013c\u013e\u0140\u0142\u0144\u0146\u0148-\u0149\u014b\u014d\u014f\u0151\u0153\u0155\u0157\u0159\u015b\u015d\u015f\u0161\u0163\u0165\u0167\u0169\u016b\u016d\u016f\u0171\u0173\u0175\u0177\u017a\u017c\u017e-\u0180\u0183\u0185\u0188\u018c-\u018d\u0192\u0195\u0199-\u019b\u019e\u01a1\u01a3\u01a5\u01a8\u01aa-\u01ab\u01ad\u01b0\u01b4\u01b6\u01b9-\u01ba\u01bd-\u01bf\u01c6\u01c9\u01cc\u01ce\u01d0\u01d2\u01d4\u01d6\u01d8\u01da\u01dc-\u01dd\u01df\u01e1\u01e3\u01e5\u01e7\u01e9\u01eb\u01ed\u01ef-\u01f0\u01f3\u01f5\u01f9\u01fb\u01fd\u01ff\u0201\u0203\u0205\u0207\u0209\u020b\u020d\u020f\u0211\u0213\u0215\u0217\u0219\u021b\u021d\u021f\u0221\u0223\u0225\u0227\u0229\u022b\u022d\u022f\u0231\u0233-\u0239\u023c\u023f-\u0240\u0242\u0247\u0249\u024b\u024d\u024f-\u0293\u0295-\u02af\u0371\u0373\u0377\u037b-\u037d\u0390\u03ac-\u03ce\u03d0-\u03d1\u03d5-\u03d7\u03d9\u03db\u03dd\u03df\u03e1\u03e3\u03e5\u03e7\u03e9\u03eb\u03ed\u03ef-\u03f3\u03f5\u03f8\u03fb-\u03fc\u0430-\u045f\u0461\u0463\u0465\u0467\u0469\u046b\u046d\u046f\u0471\u0473\u0475\u0477\u0479\u047b\u047d\u047f\u0481\u048b\u048d\u048f\u0491\u0493\u0495\u0497\u0499\u049b\u049d\u049f\u04a1\u04a3\u04a5\u04a7\u04a9\u04ab\u04ad\u04af\u04b1\u04b3\u04b5\u04b7\u04b9\u04bb\u04bd\u04bf\u04c2\u04c4\u04c6\u04c8\u04ca\u04cc\u04ce-\u04cf\u04d1\u04d3\u04d5\u04d7\u04d9\u04db\u04dd\u04df\u04e1\u04e3\u04e5\u04e7\u04e9\u04eb\u04ed\u04ef\u04f1\u04f3\u04f5\u04f7\u04f9\u04fb\u04fd\u04ff\u0501\u0503\u0505\u0507\u0509\u050b\u050d\u050f\u0511\u0513\u0515\u0517\u0519\u051b\u051d\u051f\u0521\u0523\u0525\u0527\u0561-\u0587\u1d00-\u1d2b\u1d6b-\u1d77\u1d79-\u1d9a\u1e01\u1e03\u1e05\u1e07\u1e09\u1e0b\u1e0d\u1e0f\u1e11\u1e13\u1e15\u1e17\u1e19\u1e1b\u1e1d\u1e1f\u1e21\u1e23\u1e25\u1e27\u1e29\u1e2b\u1e2d\u1e2f\u1e31\u1e33\u1e35\u1e37\u1e39\u1e3b\u1e3d\u1e3f\u1e41\u1e43\u1e45\u1e47\u1e49\u1e4b\u1e4d\u1e4f\u1e51\u1e53\u1e55\u1e57\u1e59\u1e5b\u1e5d\u1e5f\u1e61\u1e63\u1e65\u1e67\u1e69\u1e6b\u1e6d\u1e6f\u1e71\u1e73\u1e75\u1e77\u1e79\u1e7b\u1e7d\u1e7f\u1e81\u1e83\u1e85\u1e87\u1e89\u1e8b\u1e8d\u1e8f\u1e91\u1e93\u1e95-\u1e9d\u1e9f\u1ea1\u1ea3\u1ea5\u1ea7\u1ea9\u1eab\u1ead\u1eaf\u1eb1\u1eb3\u1eb5\u1eb7\u1eb9\u1ebb\u1ebd\u1ebf\u1ec1\u1ec3\u1ec5\u1ec7\u1ec9\u1ecb\u1ecd\u1ecf\u1ed1\u1ed3\u1ed5\u1ed7\u1ed9\u1edb\u1edd\u1edf\u1ee1\u1ee3\u1ee5\u1ee7\u1ee9\u1eeb\u1eed\u1eef\u1ef1\u1ef3\u1ef5\u1ef7\u1ef9\u1efb\u1efd\u1eff-\u1f07\u1f10-\u1f15\u1f20-\u1f27\u1f30-\u1f37\u1f40-\u1f45\u1f50-\u1f57\u1f60-\u1f67\u1f70-\u1f7d\u1f80-\u1f87\u1f90-\u1f97\u1fa0-\u1fa7\u1fb0-\u1fb4\u1fb6-\u1fb7\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fc7\u1fd0-\u1fd3\u1fd6-\u1fd7\u1fe0-\u1fe7\u1ff2-\u1ff4\u1ff6-\u1ff7\u210a\u210e-\u210f\u2113\u212f\u2134\u2139\u213c-\u213d\u2146-\u2149\u214e\u2184\u2c30-\u2c5e\u2c61\u2c65-\u2c66\u2c68\u2c6a\u2c6c\u2c71\u2c73-\u2c74\u2c76-\u2c7b\u2c81\u2c83\u2c85\u2c87\u2c89\u2c8b\u2c8d\u2c8f\u2c91\u2c93\u2c95\u2c97\u2c99\u2c9b\u2c9d\u2c9f\u2ca1\u2ca3\u2ca5\u2ca7\u2ca9\u2cab\u2cad\u2caf\u2cb1\u2cb3\u2cb5\u2cb7\u2cb9\u2cbb\u2cbd\u2cbf\u2cc1\u2cc3\u2cc5\u2cc7\u2cc9\u2ccb\u2ccd\u2ccf\u2cd1\u2cd3\u2cd5\u2cd7\u2cd9\u2cdb\u2cdd\u2cdf\u2ce1\u2ce3-\u2ce4\u2cec\u2cee\u2cf3\u2d00-\u2d25\u2d27\u2d2d\ua641\ua643\ua645\ua647\ua649\ua64b\ua64d\ua64f\ua651\ua653\ua655\ua657\ua659\ua65b\ua65d\ua65f\ua661\ua663\ua665\ua667\ua669\ua66b\ua66d\ua681\ua683\ua685\ua687\ua689\ua68b\ua68d\ua68f\ua691\ua693\ua695\ua697\ua723\ua725\ua727\ua729\ua72b\ua72d\ua72f-\ua731\ua733\ua735\ua737\ua739\ua73b\ua73d\ua73f\ua741\ua743\ua745\ua747\ua749\ua74b\ua74d\ua74f\ua751\ua753\ua755\ua757\ua759\ua75b\ua75d\ua75f\ua761\ua763\ua765\ua767\ua769\ua76b\ua76d\ua76f\ua771-\ua778\ua77a\ua77c\ua77f\ua781\ua783\ua785\ua787\ua78c\ua78e\ua791\ua793\ua7a1\ua7a3\ua7a5\ua7a7\ua7a9\ua7fa\ufb00-\ufb06\ufb13-\ufb17\uff41-\uff5a' +Ll = u'a-z\xb5\xdf-\xf6\xf8-\xff\u0101\u0103\u0105\u0107\u0109\u010b\u010d\u010f\u0111\u0113\u0115\u0117\u0119\u011b\u011d\u011f\u0121\u0123\u0125\u0127\u0129\u012b\u012d\u012f\u0131\u0133\u0135\u0137-\u0138\u013a\u013c\u013e\u0140\u0142\u0144\u0146\u0148-\u0149\u014b\u014d\u014f\u0151\u0153\u0155\u0157\u0159\u015b\u015d\u015f\u0161\u0163\u0165\u0167\u0169\u016b\u016d\u016f\u0171\u0173\u0175\u0177\u017a\u017c\u017e-\u0180\u0183\u0185\u0188\u018c-\u018d\u0192\u0195\u0199-\u019b\u019e\u01a1\u01a3\u01a5\u01a8\u01aa-\u01ab\u01ad\u01b0\u01b4\u01b6\u01b9-\u01ba\u01bd-\u01bf\u01c6\u01c9\u01cc\u01ce\u01d0\u01d2\u01d4\u01d6\u01d8\u01da\u01dc-\u01dd\u01df\u01e1\u01e3\u01e5\u01e7\u01e9\u01eb\u01ed\u01ef-\u01f0\u01f3\u01f5\u01f9\u01fb\u01fd\u01ff\u0201\u0203\u0205\u0207\u0209\u020b\u020d\u020f\u0211\u0213\u0215\u0217\u0219\u021b\u021d\u021f\u0221\u0223\u0225\u0227\u0229\u022b\u022d\u022f\u0231\u0233-\u0239\u023c\u023f-\u0240\u0242\u0247\u0249\u024b\u024d\u024f-\u0293\u0295-\u02af\u0371\u0373\u0377\u037b-\u037d\u0390\u03ac-\u03ce\u03d0-\u03d1\u03d5-\u03d7\u03d9\u03db\u03dd\u03df\u03e1\u03e3\u03e5\u03e7\u03e9\u03eb\u03ed\u03ef-\u03f3\u03f5\u03f8\u03fb-\u03fc\u0430-\u045f\u0461\u0463\u0465\u0467\u0469\u046b\u046d\u046f\u0471\u0473\u0475\u0477\u0479\u047b\u047d\u047f\u0481\u048b\u048d\u048f\u0491\u0493\u0495\u0497\u0499\u049b\u049d\u049f\u04a1\u04a3\u04a5\u04a7\u04a9\u04ab\u04ad\u04af\u04b1\u04b3\u04b5\u04b7\u04b9\u04bb\u04bd\u04bf\u04c2\u04c4\u04c6\u04c8\u04ca\u04cc\u04ce-\u04cf\u04d1\u04d3\u04d5\u04d7\u04d9\u04db\u04dd\u04df\u04e1\u04e3\u04e5\u04e7\u04e9\u04eb\u04ed\u04ef\u04f1\u04f3\u04f5\u04f7\u04f9\u04fb\u04fd\u04ff\u0501\u0503\u0505\u0507\u0509\u050b\u050d\u050f\u0511\u0513\u0515\u0517\u0519\u051b\u051d\u051f\u0521\u0523\u0525\u0527\u0529\u052b\u052d\u052f\u0560-\u0588\u10d0-\u10fa\u10fd-\u10ff\u13f8-\u13fd\u1c80-\u1c88\u1d00-\u1d2b\u1d6b-\u1d77\u1d79-\u1d9a\u1e01\u1e03\u1e05\u1e07\u1e09\u1e0b\u1e0d\u1e0f\u1e11\u1e13\u1e15\u1e17\u1e19\u1e1b\u1e1d\u1e1f\u1e21\u1e23\u1e25\u1e27\u1e29\u1e2b\u1e2d\u1e2f\u1e31\u1e33\u1e35\u1e37\u1e39\u1e3b\u1e3d\u1e3f\u1e41\u1e43\u1e45\u1e47\u1e49\u1e4b\u1e4d\u1e4f\u1e51\u1e53\u1e55\u1e57\u1e59\u1e5b\u1e5d\u1e5f\u1e61\u1e63\u1e65\u1e67\u1e69\u1e6b\u1e6d\u1e6f\u1e71\u1e73\u1e75\u1e77\u1e79\u1e7b\u1e7d\u1e7f\u1e81\u1e83\u1e85\u1e87\u1e89\u1e8b\u1e8d\u1e8f\u1e91\u1e93\u1e95-\u1e9d\u1e9f\u1ea1\u1ea3\u1ea5\u1ea7\u1ea9\u1eab\u1ead\u1eaf\u1eb1\u1eb3\u1eb5\u1eb7\u1eb9\u1ebb\u1ebd\u1ebf\u1ec1\u1ec3\u1ec5\u1ec7\u1ec9\u1ecb\u1ecd\u1ecf\u1ed1\u1ed3\u1ed5\u1ed7\u1ed9\u1edb\u1edd\u1edf\u1ee1\u1ee3\u1ee5\u1ee7\u1ee9\u1eeb\u1eed\u1eef\u1ef1\u1ef3\u1ef5\u1ef7\u1ef9\u1efb\u1efd\u1eff-\u1f07\u1f10-\u1f15\u1f20-\u1f27\u1f30-\u1f37\u1f40-\u1f45\u1f50-\u1f57\u1f60-\u1f67\u1f70-\u1f7d\u1f80-\u1f87\u1f90-\u1f97\u1fa0-\u1fa7\u1fb0-\u1fb4\u1fb6-\u1fb7\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fc7\u1fd0-\u1fd3\u1fd6-\u1fd7\u1fe0-\u1fe7\u1ff2-\u1ff4\u1ff6-\u1ff7\u210a\u210e-\u210f\u2113\u212f\u2134\u2139\u213c-\u213d\u2146-\u2149\u214e\u2184\u2c30-\u2c5e\u2c61\u2c65-\u2c66\u2c68\u2c6a\u2c6c\u2c71\u2c73-\u2c74\u2c76-\u2c7b\u2c81\u2c83\u2c85\u2c87\u2c89\u2c8b\u2c8d\u2c8f\u2c91\u2c93\u2c95\u2c97\u2c99\u2c9b\u2c9d\u2c9f\u2ca1\u2ca3\u2ca5\u2ca7\u2ca9\u2cab\u2cad\u2caf\u2cb1\u2cb3\u2cb5\u2cb7\u2cb9\u2cbb\u2cbd\u2cbf\u2cc1\u2cc3\u2cc5\u2cc7\u2cc9\u2ccb\u2ccd\u2ccf\u2cd1\u2cd3\u2cd5\u2cd7\u2cd9\u2cdb\u2cdd\u2cdf\u2ce1\u2ce3-\u2ce4\u2cec\u2cee\u2cf3\u2d00-\u2d25\u2d27\u2d2d\ua641\ua643\ua645\ua647\ua649\ua64b\ua64d\ua64f\ua651\ua653\ua655\ua657\ua659\ua65b\ua65d\ua65f\ua661\ua663\ua665\ua667\ua669\ua66b\ua66d\ua681\ua683\ua685\ua687\ua689\ua68b\ua68d\ua68f\ua691\ua693\ua695\ua697\ua699\ua69b\ua723\ua725\ua727\ua729\ua72b\ua72d\ua72f-\ua731\ua733\ua735\ua737\ua739\ua73b\ua73d\ua73f\ua741\ua743\ua745\ua747\ua749\ua74b\ua74d\ua74f\ua751\ua753\ua755\ua757\ua759\ua75b\ua75d\ua75f\ua761\ua763\ua765\ua767\ua769\ua76b\ua76d\ua76f\ua771-\ua778\ua77a\ua77c\ua77f\ua781\ua783\ua785\ua787\ua78c\ua78e\ua791\ua793-\ua795\ua797\ua799\ua79b\ua79d\ua79f\ua7a1\ua7a3\ua7a5\ua7a7\ua7a9\ua7af\ua7b5\ua7b7\ua7b9\ua7fa\uab30-\uab5a\uab60-\uab65\uab70-\uabbf\ufb00-\ufb06\ufb13-\ufb17\uff41-\uff5a' -Lm = u'\u02b0-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0374\u037a\u0559\u0640\u06e5-\u06e6\u07f4-\u07f5\u07fa\u081a\u0824\u0828\u0971\u0e46\u0ec6\u10fc\u17d7\u1843\u1aa7\u1c78-\u1c7d\u1d2c-\u1d6a\u1d78\u1d9b-\u1dbf\u2071\u207f\u2090-\u209c\u2c7c-\u2c7d\u2d6f\u2e2f\u3005\u3031-\u3035\u303b\u309d-\u309e\u30fc-\u30fe\ua015\ua4f8-\ua4fd\ua60c\ua67f\ua717-\ua71f\ua770\ua788\ua7f8-\ua7f9\ua9cf\uaa70\uaadd\uaaf3-\uaaf4\uff70\uff9e-\uff9f' +Lm = u'\u02b0-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0374\u037a\u0559\u0640\u06e5-\u06e6\u07f4-\u07f5\u07fa\u081a\u0824\u0828\u0971\u0e46\u0ec6\u10fc\u17d7\u1843\u1aa7\u1c78-\u1c7d\u1d2c-\u1d6a\u1d78\u1d9b-\u1dbf\u2071\u207f\u2090-\u209c\u2c7c-\u2c7d\u2d6f\u2e2f\u3005\u3031-\u3035\u303b\u309d-\u309e\u30fc-\u30fe\ua015\ua4f8-\ua4fd\ua60c\ua67f\ua69c-\ua69d\ua717-\ua71f\ua770\ua788\ua7f8-\ua7f9\ua9cf\ua9e6\uaa70\uaadd\uaaf3-\uaaf4\uab5c-\uab5f\uff70\uff9e-\uff9f' -Lo = u'\xaa\xba\u01bb\u01c0-\u01c3\u0294\u05d0-\u05ea\u05f0-\u05f2\u0620-\u063f\u0641-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u0800-\u0815\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0972-\u0977\u0979-\u097f\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58-\u0c59\u0c60-\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32-\u0e33\u0e40-\u0e45\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2-\u0eb3\u0ebd\u0ec0-\u0ec4\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10d0-\u10fa\u10fd-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17dc\u1820-\u1842\u1844-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c77\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u2135-\u2138\u2d30-\u2d67\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3006\u303c\u3041-\u3096\u309f\u30a1-\u30fa\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua014\ua016-\ua48c\ua4d0-\ua4f7\ua500-\ua60b\ua610-\ua61f\ua62a-\ua62b\ua66e\ua6a0-\ua6e5\ua7fb-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa6f\uaa71-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadc\uaae0-\uaaea\uaaf2\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff66-\uff6f\uff71-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' +Lo = u'\xaa\xba\u01bb\u01c0-\u01c3\u0294\u05d0-\u05ea\u05ef-\u05f2\u0620-\u063f\u0641-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u0800-\u0815\u0840-\u0858\u0860-\u086a\u08a0-\u08b4\u08b6-\u08bd\u0904-\u0939\u093d\u0950\u0958-\u0961\u0972-\u0980\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u09fc\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0af9\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c39\u0c3d\u0c58-\u0c5a\u0c60-\u0c61\u0c80\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d54-\u0d56\u0d5f-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32-\u0e33\u0e40-\u0e45\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2-\u0eb3\u0ebd\u0ec0-\u0ec4\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u1100-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16f1-\u16f8\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17dc\u1820-\u1842\u1844-\u1878\u1880-\u1884\u1887-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191e\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u1a00-\u1a16\u1a20-\u1a54\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c77\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u2135-\u2138\u2d30-\u2d67\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3006\u303c\u3041-\u3096\u309f\u30a1-\u30fa\u30ff\u3105-\u312f\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fef\ua000-\ua014\ua016-\ua48c\ua4d0-\ua4f7\ua500-\ua60b\ua610-\ua61f\ua62a-\ua62b\ua66e\ua6a0-\ua6e5\ua78f\ua7f7\ua7fb-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua8fd-\ua8fe\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9e0-\ua9e4\ua9e7-\ua9ef\ua9fa-\ua9fe\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa6f\uaa71-\uaa76\uaa7a\uaa7e-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadc\uaae0-\uaaea\uaaf2\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff66-\uff6f\uff71-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' Lt = u'\u01c5\u01c8\u01cb\u01f2\u1f88-\u1f8f\u1f98-\u1f9f\u1fa8-\u1faf\u1fbc\u1fcc\u1ffc' -Lu = u'A-Z\xc0-\xd6\xd8-\xde\u0100\u0102\u0104\u0106\u0108\u010a\u010c\u010e\u0110\u0112\u0114\u0116\u0118\u011a\u011c\u011e\u0120\u0122\u0124\u0126\u0128\u012a\u012c\u012e\u0130\u0132\u0134\u0136\u0139\u013b\u013d\u013f\u0141\u0143\u0145\u0147\u014a\u014c\u014e\u0150\u0152\u0154\u0156\u0158\u015a\u015c\u015e\u0160\u0162\u0164\u0166\u0168\u016a\u016c\u016e\u0170\u0172\u0174\u0176\u0178-\u0179\u017b\u017d\u0181-\u0182\u0184\u0186-\u0187\u0189-\u018b\u018e-\u0191\u0193-\u0194\u0196-\u0198\u019c-\u019d\u019f-\u01a0\u01a2\u01a4\u01a6-\u01a7\u01a9\u01ac\u01ae-\u01af\u01b1-\u01b3\u01b5\u01b7-\u01b8\u01bc\u01c4\u01c7\u01ca\u01cd\u01cf\u01d1\u01d3\u01d5\u01d7\u01d9\u01db\u01de\u01e0\u01e2\u01e4\u01e6\u01e8\u01ea\u01ec\u01ee\u01f1\u01f4\u01f6-\u01f8\u01fa\u01fc\u01fe\u0200\u0202\u0204\u0206\u0208\u020a\u020c\u020e\u0210\u0212\u0214\u0216\u0218\u021a\u021c\u021e\u0220\u0222\u0224\u0226\u0228\u022a\u022c\u022e\u0230\u0232\u023a-\u023b\u023d-\u023e\u0241\u0243-\u0246\u0248\u024a\u024c\u024e\u0370\u0372\u0376\u0386\u0388-\u038a\u038c\u038e-\u038f\u0391-\u03a1\u03a3-\u03ab\u03cf\u03d2-\u03d4\u03d8\u03da\u03dc\u03de\u03e0\u03e2\u03e4\u03e6\u03e8\u03ea\u03ec\u03ee\u03f4\u03f7\u03f9-\u03fa\u03fd-\u042f\u0460\u0462\u0464\u0466\u0468\u046a\u046c\u046e\u0470\u0472\u0474\u0476\u0478\u047a\u047c\u047e\u0480\u048a\u048c\u048e\u0490\u0492\u0494\u0496\u0498\u049a\u049c\u049e\u04a0\u04a2\u04a4\u04a6\u04a8\u04aa\u04ac\u04ae\u04b0\u04b2\u04b4\u04b6\u04b8\u04ba\u04bc\u04be\u04c0-\u04c1\u04c3\u04c5\u04c7\u04c9\u04cb\u04cd\u04d0\u04d2\u04d4\u04d6\u04d8\u04da\u04dc\u04de\u04e0\u04e2\u04e4\u04e6\u04e8\u04ea\u04ec\u04ee\u04f0\u04f2\u04f4\u04f6\u04f8\u04fa\u04fc\u04fe\u0500\u0502\u0504\u0506\u0508\u050a\u050c\u050e\u0510\u0512\u0514\u0516\u0518\u051a\u051c\u051e\u0520\u0522\u0524\u0526\u0531-\u0556\u10a0-\u10c5\u10c7\u10cd\u1e00\u1e02\u1e04\u1e06\u1e08\u1e0a\u1e0c\u1e0e\u1e10\u1e12\u1e14\u1e16\u1e18\u1e1a\u1e1c\u1e1e\u1e20\u1e22\u1e24\u1e26\u1e28\u1e2a\u1e2c\u1e2e\u1e30\u1e32\u1e34\u1e36\u1e38\u1e3a\u1e3c\u1e3e\u1e40\u1e42\u1e44\u1e46\u1e48\u1e4a\u1e4c\u1e4e\u1e50\u1e52\u1e54\u1e56\u1e58\u1e5a\u1e5c\u1e5e\u1e60\u1e62\u1e64\u1e66\u1e68\u1e6a\u1e6c\u1e6e\u1e70\u1e72\u1e74\u1e76\u1e78\u1e7a\u1e7c\u1e7e\u1e80\u1e82\u1e84\u1e86\u1e88\u1e8a\u1e8c\u1e8e\u1e90\u1e92\u1e94\u1e9e\u1ea0\u1ea2\u1ea4\u1ea6\u1ea8\u1eaa\u1eac\u1eae\u1eb0\u1eb2\u1eb4\u1eb6\u1eb8\u1eba\u1ebc\u1ebe\u1ec0\u1ec2\u1ec4\u1ec6\u1ec8\u1eca\u1ecc\u1ece\u1ed0\u1ed2\u1ed4\u1ed6\u1ed8\u1eda\u1edc\u1ede\u1ee0\u1ee2\u1ee4\u1ee6\u1ee8\u1eea\u1eec\u1eee\u1ef0\u1ef2\u1ef4\u1ef6\u1ef8\u1efa\u1efc\u1efe\u1f08-\u1f0f\u1f18-\u1f1d\u1f28-\u1f2f\u1f38-\u1f3f\u1f48-\u1f4d\u1f59\u1f5b\u1f5d\u1f5f\u1f68-\u1f6f\u1fb8-\u1fbb\u1fc8-\u1fcb\u1fd8-\u1fdb\u1fe8-\u1fec\u1ff8-\u1ffb\u2102\u2107\u210b-\u210d\u2110-\u2112\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u2130-\u2133\u213e-\u213f\u2145\u2183\u2c00-\u2c2e\u2c60\u2c62-\u2c64\u2c67\u2c69\u2c6b\u2c6d-\u2c70\u2c72\u2c75\u2c7e-\u2c80\u2c82\u2c84\u2c86\u2c88\u2c8a\u2c8c\u2c8e\u2c90\u2c92\u2c94\u2c96\u2c98\u2c9a\u2c9c\u2c9e\u2ca0\u2ca2\u2ca4\u2ca6\u2ca8\u2caa\u2cac\u2cae\u2cb0\u2cb2\u2cb4\u2cb6\u2cb8\u2cba\u2cbc\u2cbe\u2cc0\u2cc2\u2cc4\u2cc6\u2cc8\u2cca\u2ccc\u2cce\u2cd0\u2cd2\u2cd4\u2cd6\u2cd8\u2cda\u2cdc\u2cde\u2ce0\u2ce2\u2ceb\u2ced\u2cf2\ua640\ua642\ua644\ua646\ua648\ua64a\ua64c\ua64e\ua650\ua652\ua654\ua656\ua658\ua65a\ua65c\ua65e\ua660\ua662\ua664\ua666\ua668\ua66a\ua66c\ua680\ua682\ua684\ua686\ua688\ua68a\ua68c\ua68e\ua690\ua692\ua694\ua696\ua722\ua724\ua726\ua728\ua72a\ua72c\ua72e\ua732\ua734\ua736\ua738\ua73a\ua73c\ua73e\ua740\ua742\ua744\ua746\ua748\ua74a\ua74c\ua74e\ua750\ua752\ua754\ua756\ua758\ua75a\ua75c\ua75e\ua760\ua762\ua764\ua766\ua768\ua76a\ua76c\ua76e\ua779\ua77b\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\ua78d\ua790\ua792\ua7a0\ua7a2\ua7a4\ua7a6\ua7a8\ua7aa\uff21-\uff3a' +Lu = u'A-Z\xc0-\xd6\xd8-\xde\u0100\u0102\u0104\u0106\u0108\u010a\u010c\u010e\u0110\u0112\u0114\u0116\u0118\u011a\u011c\u011e\u0120\u0122\u0124\u0126\u0128\u012a\u012c\u012e\u0130\u0132\u0134\u0136\u0139\u013b\u013d\u013f\u0141\u0143\u0145\u0147\u014a\u014c\u014e\u0150\u0152\u0154\u0156\u0158\u015a\u015c\u015e\u0160\u0162\u0164\u0166\u0168\u016a\u016c\u016e\u0170\u0172\u0174\u0176\u0178-\u0179\u017b\u017d\u0181-\u0182\u0184\u0186-\u0187\u0189-\u018b\u018e-\u0191\u0193-\u0194\u0196-\u0198\u019c-\u019d\u019f-\u01a0\u01a2\u01a4\u01a6-\u01a7\u01a9\u01ac\u01ae-\u01af\u01b1-\u01b3\u01b5\u01b7-\u01b8\u01bc\u01c4\u01c7\u01ca\u01cd\u01cf\u01d1\u01d3\u01d5\u01d7\u01d9\u01db\u01de\u01e0\u01e2\u01e4\u01e6\u01e8\u01ea\u01ec\u01ee\u01f1\u01f4\u01f6-\u01f8\u01fa\u01fc\u01fe\u0200\u0202\u0204\u0206\u0208\u020a\u020c\u020e\u0210\u0212\u0214\u0216\u0218\u021a\u021c\u021e\u0220\u0222\u0224\u0226\u0228\u022a\u022c\u022e\u0230\u0232\u023a-\u023b\u023d-\u023e\u0241\u0243-\u0246\u0248\u024a\u024c\u024e\u0370\u0372\u0376\u037f\u0386\u0388-\u038a\u038c\u038e-\u038f\u0391-\u03a1\u03a3-\u03ab\u03cf\u03d2-\u03d4\u03d8\u03da\u03dc\u03de\u03e0\u03e2\u03e4\u03e6\u03e8\u03ea\u03ec\u03ee\u03f4\u03f7\u03f9-\u03fa\u03fd-\u042f\u0460\u0462\u0464\u0466\u0468\u046a\u046c\u046e\u0470\u0472\u0474\u0476\u0478\u047a\u047c\u047e\u0480\u048a\u048c\u048e\u0490\u0492\u0494\u0496\u0498\u049a\u049c\u049e\u04a0\u04a2\u04a4\u04a6\u04a8\u04aa\u04ac\u04ae\u04b0\u04b2\u04b4\u04b6\u04b8\u04ba\u04bc\u04be\u04c0-\u04c1\u04c3\u04c5\u04c7\u04c9\u04cb\u04cd\u04d0\u04d2\u04d4\u04d6\u04d8\u04da\u04dc\u04de\u04e0\u04e2\u04e4\u04e6\u04e8\u04ea\u04ec\u04ee\u04f0\u04f2\u04f4\u04f6\u04f8\u04fa\u04fc\u04fe\u0500\u0502\u0504\u0506\u0508\u050a\u050c\u050e\u0510\u0512\u0514\u0516\u0518\u051a\u051c\u051e\u0520\u0522\u0524\u0526\u0528\u052a\u052c\u052e\u0531-\u0556\u10a0-\u10c5\u10c7\u10cd\u13a0-\u13f5\u1c90-\u1cba\u1cbd-\u1cbf\u1e00\u1e02\u1e04\u1e06\u1e08\u1e0a\u1e0c\u1e0e\u1e10\u1e12\u1e14\u1e16\u1e18\u1e1a\u1e1c\u1e1e\u1e20\u1e22\u1e24\u1e26\u1e28\u1e2a\u1e2c\u1e2e\u1e30\u1e32\u1e34\u1e36\u1e38\u1e3a\u1e3c\u1e3e\u1e40\u1e42\u1e44\u1e46\u1e48\u1e4a\u1e4c\u1e4e\u1e50\u1e52\u1e54\u1e56\u1e58\u1e5a\u1e5c\u1e5e\u1e60\u1e62\u1e64\u1e66\u1e68\u1e6a\u1e6c\u1e6e\u1e70\u1e72\u1e74\u1e76\u1e78\u1e7a\u1e7c\u1e7e\u1e80\u1e82\u1e84\u1e86\u1e88\u1e8a\u1e8c\u1e8e\u1e90\u1e92\u1e94\u1e9e\u1ea0\u1ea2\u1ea4\u1ea6\u1ea8\u1eaa\u1eac\u1eae\u1eb0\u1eb2\u1eb4\u1eb6\u1eb8\u1eba\u1ebc\u1ebe\u1ec0\u1ec2\u1ec4\u1ec6\u1ec8\u1eca\u1ecc\u1ece\u1ed0\u1ed2\u1ed4\u1ed6\u1ed8\u1eda\u1edc\u1ede\u1ee0\u1ee2\u1ee4\u1ee6\u1ee8\u1eea\u1eec\u1eee\u1ef0\u1ef2\u1ef4\u1ef6\u1ef8\u1efa\u1efc\u1efe\u1f08-\u1f0f\u1f18-\u1f1d\u1f28-\u1f2f\u1f38-\u1f3f\u1f48-\u1f4d\u1f59\u1f5b\u1f5d\u1f5f\u1f68-\u1f6f\u1fb8-\u1fbb\u1fc8-\u1fcb\u1fd8-\u1fdb\u1fe8-\u1fec\u1ff8-\u1ffb\u2102\u2107\u210b-\u210d\u2110-\u2112\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u2130-\u2133\u213e-\u213f\u2145\u2183\u2c00-\u2c2e\u2c60\u2c62-\u2c64\u2c67\u2c69\u2c6b\u2c6d-\u2c70\u2c72\u2c75\u2c7e-\u2c80\u2c82\u2c84\u2c86\u2c88\u2c8a\u2c8c\u2c8e\u2c90\u2c92\u2c94\u2c96\u2c98\u2c9a\u2c9c\u2c9e\u2ca0\u2ca2\u2ca4\u2ca6\u2ca8\u2caa\u2cac\u2cae\u2cb0\u2cb2\u2cb4\u2cb6\u2cb8\u2cba\u2cbc\u2cbe\u2cc0\u2cc2\u2cc4\u2cc6\u2cc8\u2cca\u2ccc\u2cce\u2cd0\u2cd2\u2cd4\u2cd6\u2cd8\u2cda\u2cdc\u2cde\u2ce0\u2ce2\u2ceb\u2ced\u2cf2\ua640\ua642\ua644\ua646\ua648\ua64a\ua64c\ua64e\ua650\ua652\ua654\ua656\ua658\ua65a\ua65c\ua65e\ua660\ua662\ua664\ua666\ua668\ua66a\ua66c\ua680\ua682\ua684\ua686\ua688\ua68a\ua68c\ua68e\ua690\ua692\ua694\ua696\ua698\ua69a\ua722\ua724\ua726\ua728\ua72a\ua72c\ua72e\ua732\ua734\ua736\ua738\ua73a\ua73c\ua73e\ua740\ua742\ua744\ua746\ua748\ua74a\ua74c\ua74e\ua750\ua752\ua754\ua756\ua758\ua75a\ua75c\ua75e\ua760\ua762\ua764\ua766\ua768\ua76a\ua76c\ua76e\ua779\ua77b\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\ua78d\ua790\ua792\ua796\ua798\ua79a\ua79c\ua79e\ua7a0\ua7a2\ua7a4\ua7a6\ua7a8\ua7aa-\ua7ae\ua7b0-\ua7b4\ua7b6\ua7b8\uff21-\uff3a' -Mc = u'\u0903\u093b\u093e-\u0940\u0949-\u094c\u094e-\u094f\u0982-\u0983\u09be-\u09c0\u09c7-\u09c8\u09cb-\u09cc\u09d7\u0a03\u0a3e-\u0a40\u0a83\u0abe-\u0ac0\u0ac9\u0acb-\u0acc\u0b02-\u0b03\u0b3e\u0b40\u0b47-\u0b48\u0b4b-\u0b4c\u0b57\u0bbe-\u0bbf\u0bc1-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcc\u0bd7\u0c01-\u0c03\u0c41-\u0c44\u0c82-\u0c83\u0cbe\u0cc0-\u0cc4\u0cc7-\u0cc8\u0cca-\u0ccb\u0cd5-\u0cd6\u0d02-\u0d03\u0d3e-\u0d40\u0d46-\u0d48\u0d4a-\u0d4c\u0d57\u0d82-\u0d83\u0dcf-\u0dd1\u0dd8-\u0ddf\u0df2-\u0df3\u0f3e-\u0f3f\u0f7f\u102b-\u102c\u1031\u1038\u103b-\u103c\u1056-\u1057\u1062-\u1064\u1067-\u106d\u1083-\u1084\u1087-\u108c\u108f\u109a-\u109c\u17b6\u17be-\u17c5\u17c7-\u17c8\u1923-\u1926\u1929-\u192b\u1930-\u1931\u1933-\u1938\u19b0-\u19c0\u19c8-\u19c9\u1a19-\u1a1a\u1a55\u1a57\u1a61\u1a63-\u1a64\u1a6d-\u1a72\u1b04\u1b35\u1b3b\u1b3d-\u1b41\u1b43-\u1b44\u1b82\u1ba1\u1ba6-\u1ba7\u1baa\u1bac-\u1bad\u1be7\u1bea-\u1bec\u1bee\u1bf2-\u1bf3\u1c24-\u1c2b\u1c34-\u1c35\u1ce1\u1cf2-\u1cf3\u302e-\u302f\ua823-\ua824\ua827\ua880-\ua881\ua8b4-\ua8c3\ua952-\ua953\ua983\ua9b4-\ua9b5\ua9ba-\ua9bb\ua9bd-\ua9c0\uaa2f-\uaa30\uaa33-\uaa34\uaa4d\uaa7b\uaaeb\uaaee-\uaaef\uaaf5\uabe3-\uabe4\uabe6-\uabe7\uabe9-\uabea\uabec' +Mc = u'\u0903\u093b\u093e-\u0940\u0949-\u094c\u094e-\u094f\u0982-\u0983\u09be-\u09c0\u09c7-\u09c8\u09cb-\u09cc\u09d7\u0a03\u0a3e-\u0a40\u0a83\u0abe-\u0ac0\u0ac9\u0acb-\u0acc\u0b02-\u0b03\u0b3e\u0b40\u0b47-\u0b48\u0b4b-\u0b4c\u0b57\u0bbe-\u0bbf\u0bc1-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcc\u0bd7\u0c01-\u0c03\u0c41-\u0c44\u0c82-\u0c83\u0cbe\u0cc0-\u0cc4\u0cc7-\u0cc8\u0cca-\u0ccb\u0cd5-\u0cd6\u0d02-\u0d03\u0d3e-\u0d40\u0d46-\u0d48\u0d4a-\u0d4c\u0d57\u0d82-\u0d83\u0dcf-\u0dd1\u0dd8-\u0ddf\u0df2-\u0df3\u0f3e-\u0f3f\u0f7f\u102b-\u102c\u1031\u1038\u103b-\u103c\u1056-\u1057\u1062-\u1064\u1067-\u106d\u1083-\u1084\u1087-\u108c\u108f\u109a-\u109c\u17b6\u17be-\u17c5\u17c7-\u17c8\u1923-\u1926\u1929-\u192b\u1930-\u1931\u1933-\u1938\u1a19-\u1a1a\u1a55\u1a57\u1a61\u1a63-\u1a64\u1a6d-\u1a72\u1b04\u1b35\u1b3b\u1b3d-\u1b41\u1b43-\u1b44\u1b82\u1ba1\u1ba6-\u1ba7\u1baa\u1be7\u1bea-\u1bec\u1bee\u1bf2-\u1bf3\u1c24-\u1c2b\u1c34-\u1c35\u1ce1\u1cf2-\u1cf3\u1cf7\u302e-\u302f\ua823-\ua824\ua827\ua880-\ua881\ua8b4-\ua8c3\ua952-\ua953\ua983\ua9b4-\ua9b5\ua9ba-\ua9bb\ua9bd-\ua9c0\uaa2f-\uaa30\uaa33-\uaa34\uaa4d\uaa7b\uaa7d\uaaeb\uaaee-\uaaef\uaaf5\uabe3-\uabe4\uabe6-\uabe7\uabe9-\uabea\uabec' -Me = u'\u0488-\u0489\u20dd-\u20e0\u20e2-\u20e4\ua670-\ua672' +Me = u'\u0488-\u0489\u1abe\u20dd-\u20e0\u20e2-\u20e4\ua670-\ua672' -Mn = u'\u0300-\u036f\u0483-\u0487\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u0610-\u061a\u064b-\u065f\u0670\u06d6-\u06dc\u06df-\u06e4\u06e7-\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0859-\u085b\u08e4-\u08fe\u0900-\u0902\u093a\u093c\u0941-\u0948\u094d\u0951-\u0957\u0962-\u0963\u0981\u09bc\u09c1-\u09c4\u09cd\u09e2-\u09e3\u0a01-\u0a02\u0a3c\u0a41-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a70-\u0a71\u0a75\u0a81-\u0a82\u0abc\u0ac1-\u0ac5\u0ac7-\u0ac8\u0acd\u0ae2-\u0ae3\u0b01\u0b3c\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b62-\u0b63\u0b82\u0bc0\u0bcd\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c62-\u0c63\u0cbc\u0cbf\u0cc6\u0ccc-\u0ccd\u0ce2-\u0ce3\u0d41-\u0d44\u0d4d\u0d62-\u0d63\u0dca\u0dd2-\u0dd4\u0dd6\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb-\u0ebc\u0ec8-\u0ecd\u0f18-\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86-\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039-\u103a\u103d-\u103e\u1058-\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085-\u1086\u108d\u109d\u135d-\u135f\u1712-\u1714\u1732-\u1734\u1752-\u1753\u1772-\u1773\u17b4-\u17b5\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u18a9\u1920-\u1922\u1927-\u1928\u1932\u1939-\u193b\u1a17-\u1a18\u1a1b\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80-\u1b81\u1ba2-\u1ba5\u1ba8-\u1ba9\u1bab\u1be6\u1be8-\u1be9\u1bed\u1bef-\u1bf1\u1c2c-\u1c33\u1c36-\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1cf4\u1dc0-\u1de6\u1dfc-\u1dff\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2cef-\u2cf1\u2d7f\u2de0-\u2dff\u302a-\u302d\u3099-\u309a\ua66f\ua674-\ua67d\ua69f\ua6f0-\ua6f1\ua802\ua806\ua80b\ua825-\ua826\ua8c4\ua8e0-\ua8f1\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\uaa29-\uaa2e\uaa31-\uaa32\uaa35-\uaa36\uaa43\uaa4c\uaab0\uaab2-\uaab4\uaab7-\uaab8\uaabe-\uaabf\uaac1\uaaec-\uaaed\uaaf6\uabe5\uabe8\uabed\ufb1e\ufe00-\ufe0f\ufe20-\ufe26' +Mn = u'\u0300-\u036f\u0483-\u0487\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u0610-\u061a\u064b-\u065f\u0670\u06d6-\u06dc\u06df-\u06e4\u06e7-\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u07fd\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0859-\u085b\u08d3-\u08e1\u08e3-\u0902\u093a\u093c\u0941-\u0948\u094d\u0951-\u0957\u0962-\u0963\u0981\u09bc\u09c1-\u09c4\u09cd\u09e2-\u09e3\u09fe\u0a01-\u0a02\u0a3c\u0a41-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a70-\u0a71\u0a75\u0a81-\u0a82\u0abc\u0ac1-\u0ac5\u0ac7-\u0ac8\u0acd\u0ae2-\u0ae3\u0afa-\u0aff\u0b01\u0b3c\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b62-\u0b63\u0b82\u0bc0\u0bcd\u0c00\u0c04\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c62-\u0c63\u0c81\u0cbc\u0cbf\u0cc6\u0ccc-\u0ccd\u0ce2-\u0ce3\u0d00-\u0d01\u0d3b-\u0d3c\u0d41-\u0d44\u0d4d\u0d62-\u0d63\u0dca\u0dd2-\u0dd4\u0dd6\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb-\u0ebc\u0ec8-\u0ecd\u0f18-\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86-\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039-\u103a\u103d-\u103e\u1058-\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085-\u1086\u108d\u109d\u135d-\u135f\u1712-\u1714\u1732-\u1734\u1752-\u1753\u1772-\u1773\u17b4-\u17b5\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u1885-\u1886\u18a9\u1920-\u1922\u1927-\u1928\u1932\u1939-\u193b\u1a17-\u1a18\u1a1b\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1ab0-\u1abd\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80-\u1b81\u1ba2-\u1ba5\u1ba8-\u1ba9\u1bab-\u1bad\u1be6\u1be8-\u1be9\u1bed\u1bef-\u1bf1\u1c2c-\u1c33\u1c36-\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1cf4\u1cf8-\u1cf9\u1dc0-\u1df9\u1dfb-\u1dff\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2cef-\u2cf1\u2d7f\u2de0-\u2dff\u302a-\u302d\u3099-\u309a\ua66f\ua674-\ua67d\ua69e-\ua69f\ua6f0-\ua6f1\ua802\ua806\ua80b\ua825-\ua826\ua8c4-\ua8c5\ua8e0-\ua8f1\ua8ff\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\ua9e5\uaa29-\uaa2e\uaa31-\uaa32\uaa35-\uaa36\uaa43\uaa4c\uaa7c\uaab0\uaab2-\uaab4\uaab7-\uaab8\uaabe-\uaabf\uaac1\uaaec-\uaaed\uaaf6\uabe5\uabe8\uabed\ufb1e\ufe00-\ufe0f\ufe20-\ufe2f' -Nd = u'0-9\u0660-\u0669\u06f0-\u06f9\u07c0-\u07c9\u0966-\u096f\u09e6-\u09ef\u0a66-\u0a6f\u0ae6-\u0aef\u0b66-\u0b6f\u0be6-\u0bef\u0c66-\u0c6f\u0ce6-\u0cef\u0d66-\u0d6f\u0e50-\u0e59\u0ed0-\u0ed9\u0f20-\u0f29\u1040-\u1049\u1090-\u1099\u17e0-\u17e9\u1810-\u1819\u1946-\u194f\u19d0-\u19d9\u1a80-\u1a89\u1a90-\u1a99\u1b50-\u1b59\u1bb0-\u1bb9\u1c40-\u1c49\u1c50-\u1c59\ua620-\ua629\ua8d0-\ua8d9\ua900-\ua909\ua9d0-\ua9d9\uaa50-\uaa59\uabf0-\uabf9\uff10-\uff19' +Nd = u'0-9\u0660-\u0669\u06f0-\u06f9\u07c0-\u07c9\u0966-\u096f\u09e6-\u09ef\u0a66-\u0a6f\u0ae6-\u0aef\u0b66-\u0b6f\u0be6-\u0bef\u0c66-\u0c6f\u0ce6-\u0cef\u0d66-\u0d6f\u0de6-\u0def\u0e50-\u0e59\u0ed0-\u0ed9\u0f20-\u0f29\u1040-\u1049\u1090-\u1099\u17e0-\u17e9\u1810-\u1819\u1946-\u194f\u19d0-\u19d9\u1a80-\u1a89\u1a90-\u1a99\u1b50-\u1b59\u1bb0-\u1bb9\u1c40-\u1c49\u1c50-\u1c59\ua620-\ua629\ua8d0-\ua8d9\ua900-\ua909\ua9d0-\ua9d9\ua9f0-\ua9f9\uaa50-\uaa59\uabf0-\uabf9\uff10-\uff19' Nl = u'\u16ee-\u16f0\u2160-\u2182\u2185-\u2188\u3007\u3021-\u3029\u3038-\u303a\ua6e6-\ua6ef' -No = u'\xb2-\xb3\xb9\xbc-\xbe\u09f4-\u09f9\u0b72-\u0b77\u0bf0-\u0bf2\u0c78-\u0c7e\u0d70-\u0d75\u0f2a-\u0f33\u1369-\u137c\u17f0-\u17f9\u19da\u2070\u2074-\u2079\u2080-\u2089\u2150-\u215f\u2189\u2460-\u249b\u24ea-\u24ff\u2776-\u2793\u2cfd\u3192-\u3195\u3220-\u3229\u3248-\u324f\u3251-\u325f\u3280-\u3289\u32b1-\u32bf\ua830-\ua835' +No = u'\xb2-\xb3\xb9\xbc-\xbe\u09f4-\u09f9\u0b72-\u0b77\u0bf0-\u0bf2\u0c78-\u0c7e\u0d58-\u0d5e\u0d70-\u0d78\u0f2a-\u0f33\u1369-\u137c\u17f0-\u17f9\u19da\u2070\u2074-\u2079\u2080-\u2089\u2150-\u215f\u2189\u2460-\u249b\u24ea-\u24ff\u2776-\u2793\u2cfd\u3192-\u3195\u3220-\u3229\u3248-\u324f\u3251-\u325f\u3280-\u3289\u32b1-\u32bf\ua830-\ua835' Pc = u'_\u203f-\u2040\u2054\ufe33-\ufe34\ufe4d-\ufe4f\uff3f' -Pd = u'\\-\u058a\u05be\u1400\u1806\u2010-\u2015\u2e17\u2e1a\u2e3a-\u2e3b\u301c\u3030\u30a0\ufe31-\ufe32\ufe58\ufe63\uff0d' +Pd = u'\\-\u058a\u05be\u1400\u1806\u2010-\u2015\u2e17\u2e1a\u2e3a-\u2e3b\u2e40\u301c\u3030\u30a0\ufe31-\ufe32\ufe58\ufe63\uff0d' -Pe = u')\\]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u2309\u230b\u232a\u2769\u276b\u276d\u276f\u2771\u2773\u2775\u27c6\u27e7\u27e9\u27eb\u27ed\u27ef\u2984\u2986\u2988\u298a\u298c\u298e\u2990\u2992\u2994\u2996\u2998\u29d9\u29db\u29fd\u2e23\u2e25\u2e27\u2e29\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e-\u301f\ufd3f\ufe18\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42\ufe44\ufe48\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff60\uff63' +Pe = u')\\]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u2309\u230b\u232a\u2769\u276b\u276d\u276f\u2771\u2773\u2775\u27c6\u27e7\u27e9\u27eb\u27ed\u27ef\u2984\u2986\u2988\u298a\u298c\u298e\u2990\u2992\u2994\u2996\u2998\u29d9\u29db\u29fd\u2e23\u2e25\u2e27\u2e29\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e-\u301f\ufd3e\ufe18\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42\ufe44\ufe48\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff60\uff63' Pf = u'\xbb\u2019\u201d\u203a\u2e03\u2e05\u2e0a\u2e0d\u2e1d\u2e21' Pi = u'\xab\u2018\u201b-\u201c\u201f\u2039\u2e02\u2e04\u2e09\u2e0c\u2e1c\u2e20' -Po = u"!-#%-'*,.-/:-;?-@\\\\\xa1\xa7\xb6-\xb7\xbf\u037e\u0387\u055a-\u055f\u0589\u05c0\u05c3\u05c6\u05f3-\u05f4\u0609-\u060a\u060c-\u060d\u061b\u061e-\u061f\u066a-\u066d\u06d4\u0700-\u070d\u07f7-\u07f9\u0830-\u083e\u085e\u0964-\u0965\u0970\u0af0\u0df4\u0e4f\u0e5a-\u0e5b\u0f04-\u0f12\u0f14\u0f85\u0fd0-\u0fd4\u0fd9-\u0fda\u104a-\u104f\u10fb\u1360-\u1368\u166d-\u166e\u16eb-\u16ed\u1735-\u1736\u17d4-\u17d6\u17d8-\u17da\u1800-\u1805\u1807-\u180a\u1944-\u1945\u1a1e-\u1a1f\u1aa0-\u1aa6\u1aa8-\u1aad\u1b5a-\u1b60\u1bfc-\u1bff\u1c3b-\u1c3f\u1c7e-\u1c7f\u1cc0-\u1cc7\u1cd3\u2016-\u2017\u2020-\u2027\u2030-\u2038\u203b-\u203e\u2041-\u2043\u2047-\u2051\u2053\u2055-\u205e\u2cf9-\u2cfc\u2cfe-\u2cff\u2d70\u2e00-\u2e01\u2e06-\u2e08\u2e0b\u2e0e-\u2e16\u2e18-\u2e19\u2e1b\u2e1e-\u2e1f\u2e2a-\u2e2e\u2e30-\u2e39\u3001-\u3003\u303d\u30fb\ua4fe-\ua4ff\ua60d-\ua60f\ua673\ua67e\ua6f2-\ua6f7\ua874-\ua877\ua8ce-\ua8cf\ua8f8-\ua8fa\ua92e-\ua92f\ua95f\ua9c1-\ua9cd\ua9de-\ua9df\uaa5c-\uaa5f\uaade-\uaadf\uaaf0-\uaaf1\uabeb\ufe10-\ufe16\ufe19\ufe30\ufe45-\ufe46\ufe49-\ufe4c\ufe50-\ufe52\ufe54-\ufe57\ufe5f-\ufe61\ufe68\ufe6a-\ufe6b\uff01-\uff03\uff05-\uff07\uff0a\uff0c\uff0e-\uff0f\uff1a-\uff1b\uff1f-\uff20\uff3c\uff61\uff64-\uff65" +Po = u"!-#%-'*,.-/:-;?-@\\\\\xa1\xa7\xb6-\xb7\xbf\u037e\u0387\u055a-\u055f\u0589\u05c0\u05c3\u05c6\u05f3-\u05f4\u0609-\u060a\u060c-\u060d\u061b\u061e-\u061f\u066a-\u066d\u06d4\u0700-\u070d\u07f7-\u07f9\u0830-\u083e\u085e\u0964-\u0965\u0970\u09fd\u0a76\u0af0\u0c84\u0df4\u0e4f\u0e5a-\u0e5b\u0f04-\u0f12\u0f14\u0f85\u0fd0-\u0fd4\u0fd9-\u0fda\u104a-\u104f\u10fb\u1360-\u1368\u166d-\u166e\u16eb-\u16ed\u1735-\u1736\u17d4-\u17d6\u17d8-\u17da\u1800-\u1805\u1807-\u180a\u1944-\u1945\u1a1e-\u1a1f\u1aa0-\u1aa6\u1aa8-\u1aad\u1b5a-\u1b60\u1bfc-\u1bff\u1c3b-\u1c3f\u1c7e-\u1c7f\u1cc0-\u1cc7\u1cd3\u2016-\u2017\u2020-\u2027\u2030-\u2038\u203b-\u203e\u2041-\u2043\u2047-\u2051\u2053\u2055-\u205e\u2cf9-\u2cfc\u2cfe-\u2cff\u2d70\u2e00-\u2e01\u2e06-\u2e08\u2e0b\u2e0e-\u2e16\u2e18-\u2e19\u2e1b\u2e1e-\u2e1f\u2e2a-\u2e2e\u2e30-\u2e39\u2e3c-\u2e3f\u2e41\u2e43-\u2e4e\u3001-\u3003\u303d\u30fb\ua4fe-\ua4ff\ua60d-\ua60f\ua673\ua67e\ua6f2-\ua6f7\ua874-\ua877\ua8ce-\ua8cf\ua8f8-\ua8fa\ua8fc\ua92e-\ua92f\ua95f\ua9c1-\ua9cd\ua9de-\ua9df\uaa5c-\uaa5f\uaade-\uaadf\uaaf0-\uaaf1\uabeb\ufe10-\ufe16\ufe19\ufe30\ufe45-\ufe46\ufe49-\ufe4c\ufe50-\ufe52\ufe54-\ufe57\ufe5f-\ufe61\ufe68\ufe6a-\ufe6b\uff01-\uff03\uff05-\uff07\uff0a\uff0c\uff0e-\uff0f\uff1a-\uff1b\uff1f-\uff20\uff3c\uff61\uff64-\uff65" -Ps = u'(\\[{\u0f3a\u0f3c\u169b\u201a\u201e\u2045\u207d\u208d\u2308\u230a\u2329\u2768\u276a\u276c\u276e\u2770\u2772\u2774\u27c5\u27e6\u27e8\u27ea\u27ec\u27ee\u2983\u2985\u2987\u2989\u298b\u298d\u298f\u2991\u2993\u2995\u2997\u29d8\u29da\u29fc\u2e22\u2e24\u2e26\u2e28\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d\ufd3e\ufe17\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe47\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff5f\uff62' +Ps = u'(\\[{\u0f3a\u0f3c\u169b\u201a\u201e\u2045\u207d\u208d\u2308\u230a\u2329\u2768\u276a\u276c\u276e\u2770\u2772\u2774\u27c5\u27e6\u27e8\u27ea\u27ec\u27ee\u2983\u2985\u2987\u2989\u298b\u298d\u298f\u2991\u2993\u2995\u2997\u29d8\u29da\u29fc\u2e22\u2e24\u2e26\u2e28\u2e42\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d\ufd3f\ufe17\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe47\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff5f\uff62' -Sc = u'$\xa2-\xa5\u058f\u060b\u09f2-\u09f3\u09fb\u0af1\u0bf9\u0e3f\u17db\u20a0-\u20ba\ua838\ufdfc\ufe69\uff04\uffe0-\uffe1\uffe5-\uffe6' +Sc = u'$\xa2-\xa5\u058f\u060b\u07fe-\u07ff\u09f2-\u09f3\u09fb\u0af1\u0bf9\u0e3f\u17db\u20a0-\u20bf\ua838\ufdfc\ufe69\uff04\uffe0-\uffe1\uffe5-\uffe6' -Sk = u'\\^`\xa8\xaf\xb4\xb8\u02c2-\u02c5\u02d2-\u02df\u02e5-\u02eb\u02ed\u02ef-\u02ff\u0375\u0384-\u0385\u1fbd\u1fbf-\u1fc1\u1fcd-\u1fcf\u1fdd-\u1fdf\u1fed-\u1fef\u1ffd-\u1ffe\u309b-\u309c\ua700-\ua716\ua720-\ua721\ua789-\ua78a\ufbb2-\ufbc1\uff3e\uff40\uffe3' +Sk = u'\\^`\xa8\xaf\xb4\xb8\u02c2-\u02c5\u02d2-\u02df\u02e5-\u02eb\u02ed\u02ef-\u02ff\u0375\u0384-\u0385\u1fbd\u1fbf-\u1fc1\u1fcd-\u1fcf\u1fdd-\u1fdf\u1fed-\u1fef\u1ffd-\u1ffe\u309b-\u309c\ua700-\ua716\ua720-\ua721\ua789-\ua78a\uab5b\ufbb2-\ufbc1\uff3e\uff40\uffe3' Sm = u'+<->|~\xac\xb1\xd7\xf7\u03f6\u0606-\u0608\u2044\u2052\u207a-\u207c\u208a-\u208c\u2118\u2140-\u2144\u214b\u2190-\u2194\u219a-\u219b\u21a0\u21a3\u21a6\u21ae\u21ce-\u21cf\u21d2\u21d4\u21f4-\u22ff\u2320-\u2321\u237c\u239b-\u23b3\u23dc-\u23e1\u25b7\u25c1\u25f8-\u25ff\u266f\u27c0-\u27c4\u27c7-\u27e5\u27f0-\u27ff\u2900-\u2982\u2999-\u29d7\u29dc-\u29fb\u29fe-\u2aff\u2b30-\u2b44\u2b47-\u2b4c\ufb29\ufe62\ufe64-\ufe66\uff0b\uff1c-\uff1e\uff5c\uff5e\uffe2\uffe9-\uffec' -So = u'\xa6\xa9\xae\xb0\u0482\u060e-\u060f\u06de\u06e9\u06fd-\u06fe\u07f6\u09fa\u0b70\u0bf3-\u0bf8\u0bfa\u0c7f\u0d79\u0f01-\u0f03\u0f13\u0f15-\u0f17\u0f1a-\u0f1f\u0f34\u0f36\u0f38\u0fbe-\u0fc5\u0fc7-\u0fcc\u0fce-\u0fcf\u0fd5-\u0fd8\u109e-\u109f\u1390-\u1399\u1940\u19de-\u19ff\u1b61-\u1b6a\u1b74-\u1b7c\u2100-\u2101\u2103-\u2106\u2108-\u2109\u2114\u2116-\u2117\u211e-\u2123\u2125\u2127\u2129\u212e\u213a-\u213b\u214a\u214c-\u214d\u214f\u2195-\u2199\u219c-\u219f\u21a1-\u21a2\u21a4-\u21a5\u21a7-\u21ad\u21af-\u21cd\u21d0-\u21d1\u21d3\u21d5-\u21f3\u2300-\u2307\u230c-\u231f\u2322-\u2328\u232b-\u237b\u237d-\u239a\u23b4-\u23db\u23e2-\u23f3\u2400-\u2426\u2440-\u244a\u249c-\u24e9\u2500-\u25b6\u25b8-\u25c0\u25c2-\u25f7\u2600-\u266e\u2670-\u26ff\u2701-\u2767\u2794-\u27bf\u2800-\u28ff\u2b00-\u2b2f\u2b45-\u2b46\u2b50-\u2b59\u2ce5-\u2cea\u2e80-\u2e99\u2e9b-\u2ef3\u2f00-\u2fd5\u2ff0-\u2ffb\u3004\u3012-\u3013\u3020\u3036-\u3037\u303e-\u303f\u3190-\u3191\u3196-\u319f\u31c0-\u31e3\u3200-\u321e\u322a-\u3247\u3250\u3260-\u327f\u328a-\u32b0\u32c0-\u32fe\u3300-\u33ff\u4dc0-\u4dff\ua490-\ua4c6\ua828-\ua82b\ua836-\ua837\ua839\uaa77-\uaa79\ufdfd\uffe4\uffe8\uffed-\uffee\ufffc-\ufffd' +So = u'\xa6\xa9\xae\xb0\u0482\u058d-\u058e\u060e-\u060f\u06de\u06e9\u06fd-\u06fe\u07f6\u09fa\u0b70\u0bf3-\u0bf8\u0bfa\u0c7f\u0d4f\u0d79\u0f01-\u0f03\u0f13\u0f15-\u0f17\u0f1a-\u0f1f\u0f34\u0f36\u0f38\u0fbe-\u0fc5\u0fc7-\u0fcc\u0fce-\u0fcf\u0fd5-\u0fd8\u109e-\u109f\u1390-\u1399\u1940\u19de-\u19ff\u1b61-\u1b6a\u1b74-\u1b7c\u2100-\u2101\u2103-\u2106\u2108-\u2109\u2114\u2116-\u2117\u211e-\u2123\u2125\u2127\u2129\u212e\u213a-\u213b\u214a\u214c-\u214d\u214f\u218a-\u218b\u2195-\u2199\u219c-\u219f\u21a1-\u21a2\u21a4-\u21a5\u21a7-\u21ad\u21af-\u21cd\u21d0-\u21d1\u21d3\u21d5-\u21f3\u2300-\u2307\u230c-\u231f\u2322-\u2328\u232b-\u237b\u237d-\u239a\u23b4-\u23db\u23e2-\u2426\u2440-\u244a\u249c-\u24e9\u2500-\u25b6\u25b8-\u25c0\u25c2-\u25f7\u2600-\u266e\u2670-\u2767\u2794-\u27bf\u2800-\u28ff\u2b00-\u2b2f\u2b45-\u2b46\u2b4d-\u2b73\u2b76-\u2b95\u2b98-\u2bc8\u2bca-\u2bfe\u2ce5-\u2cea\u2e80-\u2e99\u2e9b-\u2ef3\u2f00-\u2fd5\u2ff0-\u2ffb\u3004\u3012-\u3013\u3020\u3036-\u3037\u303e-\u303f\u3190-\u3191\u3196-\u319f\u31c0-\u31e3\u3200-\u321e\u322a-\u3247\u3250\u3260-\u327f\u328a-\u32b0\u32c0-\u32fe\u3300-\u33ff\u4dc0-\u4dff\ua490-\ua4c6\ua828-\ua82b\ua836-\ua837\ua839\uaa77-\uaa79\ufdfd\uffe4\uffe8\uffed-\uffee\ufffc-\ufffd' Zl = u'\u2028' @@ -77,49 +77,53 @@ Zp = u'\u2029' Zs = u' \xa0\u1680\u2000-\u200a\u202f\u205f\u3000' -xid_continue = u'0-9A-Z_a-z\xaa\xb5\xb7\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0300-\u0374\u0376-\u0377\u037b-\u037d\u0386-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u0483-\u0487\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u05d0-\u05ea\u05f0-\u05f2\u0610-\u061a\u0620-\u0669\u066e-\u06d3\u06d5-\u06dc\u06df-\u06e8\u06ea-\u06fc\u06ff\u0710-\u074a\u074d-\u07b1\u07c0-\u07f5\u07fa\u0800-\u082d\u0840-\u085b\u08a0\u08a2-\u08ac\u08e4-\u08fe\u0900-\u0963\u0966-\u096f\u0971-\u0977\u0979-\u097f\u0981-\u0983\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bc-\u09c4\u09c7-\u09c8\u09cb-\u09ce\u09d7\u09dc-\u09dd\u09df-\u09e3\u09e6-\u09f1\u0a01-\u0a03\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a3c\u0a3e-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a59-\u0a5c\u0a5e\u0a66-\u0a75\u0a81-\u0a83\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abc-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ad0\u0ae0-\u0ae3\u0ae6-\u0aef\u0b01-\u0b03\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3c-\u0b44\u0b47-\u0b48\u0b4b-\u0b4d\u0b56-\u0b57\u0b5c-\u0b5d\u0b5f-\u0b63\u0b66-\u0b6f\u0b71\u0b82-\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd0\u0bd7\u0be6-\u0bef\u0c01-\u0c03\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d-\u0c44\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c58-\u0c59\u0c60-\u0c63\u0c66-\u0c6f\u0c82-\u0c83\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbc-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5-\u0cd6\u0cde\u0ce0-\u0ce3\u0ce6-\u0cef\u0cf1-\u0cf2\u0d02-\u0d03\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d-\u0d44\u0d46-\u0d48\u0d4a-\u0d4e\u0d57\u0d60-\u0d63\u0d66-\u0d6f\u0d7a-\u0d7f\u0d82-\u0d83\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0df2-\u0df3\u0e01-\u0e3a\u0e40-\u0e4e\u0e50-\u0e59\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb9\u0ebb-\u0ebd\u0ec0-\u0ec4\u0ec6\u0ec8-\u0ecd\u0ed0-\u0ed9\u0edc-\u0edf\u0f00\u0f18-\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f3e-\u0f47\u0f49-\u0f6c\u0f71-\u0f84\u0f86-\u0f97\u0f99-\u0fbc\u0fc6\u1000-\u1049\u1050-\u109d\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u135d-\u135f\u1369-\u1371\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1714\u1720-\u1734\u1740-\u1753\u1760-\u176c\u176e-\u1770\u1772-\u1773\u1780-\u17d3\u17d7\u17dc-\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u1820-\u1877\u1880-\u18aa\u18b0-\u18f5\u1900-\u191c\u1920-\u192b\u1930-\u193b\u1946-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u19d0-\u19da\u1a00-\u1a1b\u1a20-\u1a5e\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1aa7\u1b00-\u1b4b\u1b50-\u1b59\u1b6b-\u1b73\u1b80-\u1bf3\u1c00-\u1c37\u1c40-\u1c49\u1c4d-\u1c7d\u1cd0-\u1cd2\u1cd4-\u1cf6\u1d00-\u1de6\u1dfc-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u203f-\u2040\u2054\u2071\u207f\u2090-\u209c\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d7f-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2de0-\u2dff\u3005-\u3007\u3021-\u302f\u3031-\u3035\u3038-\u303c\u3041-\u3096\u3099-\u309a\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua62b\ua640-\ua66f\ua674-\ua67d\ua67f-\ua697\ua69f-\ua6f1\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua827\ua840-\ua873\ua880-\ua8c4\ua8d0-\ua8d9\ua8e0-\ua8f7\ua8fb\ua900-\ua92d\ua930-\ua953\ua960-\ua97c\ua980-\ua9c0\ua9cf-\ua9d9\uaa00-\uaa36\uaa40-\uaa4d\uaa50-\uaa59\uaa60-\uaa76\uaa7a-\uaa7b\uaa80-\uaac2\uaadb-\uaadd\uaae0-\uaaef\uaaf2-\uaaf6\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabea\uabec-\uabed\uabf0-\uabf9\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe00-\ufe0f\ufe20-\ufe26\ufe33-\ufe34\ufe4d-\ufe4f\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff10-\uff19\uff21-\uff3a\uff3f\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' +xid_continue = u'0-9A-Z_a-z\xaa\xb5\xb7\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0300-\u0374\u0376-\u0377\u037b-\u037d\u037f\u0386-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u0483-\u0487\u048a-\u052f\u0531-\u0556\u0559\u0560-\u0588\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u05d0-\u05ea\u05ef-\u05f2\u0610-\u061a\u0620-\u0669\u066e-\u06d3\u06d5-\u06dc\u06df-\u06e8\u06ea-\u06fc\u06ff\u0710-\u074a\u074d-\u07b1\u07c0-\u07f5\u07fa\u07fd\u0800-\u082d\u0840-\u085b\u0860-\u086a\u08a0-\u08b4\u08b6-\u08bd\u08d3-\u08e1\u08e3-\u0963\u0966-\u096f\u0971-\u0983\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bc-\u09c4\u09c7-\u09c8\u09cb-\u09ce\u09d7\u09dc-\u09dd\u09df-\u09e3\u09e6-\u09f1\u09fc\u09fe\u0a01-\u0a03\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a3c\u0a3e-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a59-\u0a5c\u0a5e\u0a66-\u0a75\u0a81-\u0a83\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abc-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ad0\u0ae0-\u0ae3\u0ae6-\u0aef\u0af9-\u0aff\u0b01-\u0b03\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3c-\u0b44\u0b47-\u0b48\u0b4b-\u0b4d\u0b56-\u0b57\u0b5c-\u0b5d\u0b5f-\u0b63\u0b66-\u0b6f\u0b71\u0b82-\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd0\u0bd7\u0be6-\u0bef\u0c00-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c39\u0c3d-\u0c44\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c58-\u0c5a\u0c60-\u0c63\u0c66-\u0c6f\u0c80-\u0c83\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbc-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5-\u0cd6\u0cde\u0ce0-\u0ce3\u0ce6-\u0cef\u0cf1-\u0cf2\u0d00-\u0d03\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d44\u0d46-\u0d48\u0d4a-\u0d4e\u0d54-\u0d57\u0d5f-\u0d63\u0d66-\u0d6f\u0d7a-\u0d7f\u0d82-\u0d83\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0de6-\u0def\u0df2-\u0df3\u0e01-\u0e3a\u0e40-\u0e4e\u0e50-\u0e59\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb9\u0ebb-\u0ebd\u0ec0-\u0ec4\u0ec6\u0ec8-\u0ecd\u0ed0-\u0ed9\u0edc-\u0edf\u0f00\u0f18-\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f3e-\u0f47\u0f49-\u0f6c\u0f71-\u0f84\u0f86-\u0f97\u0f99-\u0fbc\u0fc6\u1000-\u1049\u1050-\u109d\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u135d-\u135f\u1369-\u1371\u1380-\u138f\u13a0-\u13f5\u13f8-\u13fd\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f8\u1700-\u170c\u170e-\u1714\u1720-\u1734\u1740-\u1753\u1760-\u176c\u176e-\u1770\u1772-\u1773\u1780-\u17d3\u17d7\u17dc-\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u1820-\u1878\u1880-\u18aa\u18b0-\u18f5\u1900-\u191e\u1920-\u192b\u1930-\u193b\u1946-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u19d0-\u19da\u1a00-\u1a1b\u1a20-\u1a5e\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1aa7\u1ab0-\u1abd\u1b00-\u1b4b\u1b50-\u1b59\u1b6b-\u1b73\u1b80-\u1bf3\u1c00-\u1c37\u1c40-\u1c49\u1c4d-\u1c7d\u1c80-\u1c88\u1c90-\u1cba\u1cbd-\u1cbf\u1cd0-\u1cd2\u1cd4-\u1cf9\u1d00-\u1df9\u1dfb-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u203f-\u2040\u2054\u2071\u207f\u2090-\u209c\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d7f-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2de0-\u2dff\u3005-\u3007\u3021-\u302f\u3031-\u3035\u3038-\u303c\u3041-\u3096\u3099-\u309a\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312f\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fef\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua62b\ua640-\ua66f\ua674-\ua67d\ua67f-\ua6f1\ua717-\ua71f\ua722-\ua788\ua78b-\ua7b9\ua7f7-\ua827\ua840-\ua873\ua880-\ua8c5\ua8d0-\ua8d9\ua8e0-\ua8f7\ua8fb\ua8fd-\ua92d\ua930-\ua953\ua960-\ua97c\ua980-\ua9c0\ua9cf-\ua9d9\ua9e0-\ua9fe\uaa00-\uaa36\uaa40-\uaa4d\uaa50-\uaa59\uaa60-\uaa76\uaa7a-\uaac2\uaadb-\uaadd\uaae0-\uaaef\uaaf2-\uaaf6\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uab30-\uab5a\uab5c-\uab65\uab70-\uabea\uabec-\uabed\uabf0-\uabf9\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe00-\ufe0f\ufe20-\ufe2f\ufe33-\ufe34\ufe4d-\ufe4f\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff10-\uff19\uff21-\uff3a\uff3f\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' -xid_start = u'A-Z_a-z\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376-\u0377\u037b-\u037d\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u05d0-\u05ea\u05f0-\u05f2\u0620-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06e5-\u06e6\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4-\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0977\u0979-\u097f\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58-\u0c59\u0c60-\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e40-\u0e46\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a-\ua62b\ua640-\ua66e\ua67f-\ua697\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' +xid_start = u'A-Z_a-z\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376-\u0377\u037b-\u037d\u037f\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u052f\u0531-\u0556\u0559\u0560-\u0588\u05d0-\u05ea\u05ef-\u05f2\u0620-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06e5-\u06e6\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4-\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u0860-\u086a\u08a0-\u08b4\u08b6-\u08bd\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0980\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u09fc\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0af9\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c39\u0c3d\u0c58-\u0c5a\u0c60-\u0c61\u0c80\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d54-\u0d56\u0d5f-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e40-\u0e46\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f5\u13f8-\u13fd\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f8\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1878\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191e\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1c80-\u1c88\u1c90-\u1cba\u1cbd-\u1cbf\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312f\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fef\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a-\ua62b\ua640-\ua66e\ua67f-\ua69d\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua7b9\ua7f7-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua8fd-\ua8fe\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\ua9e0-\ua9e4\ua9e6-\ua9ef\ua9fa-\ua9fe\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa7e-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uab30-\uab5a\uab5c-\uab65\uab70-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' if sys.maxunicode > 0xFFFF: # non-BMP characters, use only on wide Unicode builds - Cf += u'\U000110bd\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f' + Cf += u'\U000110bd\U000110cd\U0001bca0-\U0001bca3\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f' - Cn += u'\U0001000c\U00010027\U0001003b\U0001003e\U0001004e-\U0001004f\U0001005e-\U0001007f\U000100fb-\U000100ff\U00010103-\U00010106\U00010134-\U00010136\U0001018b-\U0001018f\U0001019c-\U000101cf\U000101fe-\U0001027f\U0001029d-\U0001029f\U000102d1-\U000102ff\U0001031f\U00010324-\U0001032f\U0001034b-\U0001037f\U0001039e\U000103c4-\U000103c7\U000103d6-\U000103ff\U0001049e-\U0001049f\U000104aa-\U000107ff\U00010806-\U00010807\U00010809\U00010836\U00010839-\U0001083b\U0001083d-\U0001083e\U00010856\U00010860-\U000108ff\U0001091c-\U0001091e\U0001093a-\U0001093e\U00010940-\U0001097f\U000109b8-\U000109bd\U000109c0-\U000109ff\U00010a04\U00010a07-\U00010a0b\U00010a14\U00010a18\U00010a34-\U00010a37\U00010a3b-\U00010a3e\U00010a48-\U00010a4f\U00010a59-\U00010a5f\U00010a80-\U00010aff\U00010b36-\U00010b38\U00010b56-\U00010b57\U00010b73-\U00010b77\U00010b80-\U00010bff\U00010c49-\U00010e5f\U00010e7f-\U00010fff\U0001104e-\U00011051\U00011070-\U0001107f\U000110c2-\U000110cf\U000110e9-\U000110ef\U000110fa-\U000110ff\U00011135\U00011144-\U0001117f\U000111c9-\U000111cf\U000111da-\U0001167f\U000116b8-\U000116bf\U000116ca-\U00011fff\U0001236f-\U000123ff\U00012463-\U0001246f\U00012474-\U00012fff\U0001342f-\U000167ff\U00016a39-\U00016eff\U00016f45-\U00016f4f\U00016f7f-\U00016f8e\U00016fa0-\U0001afff\U0001b002-\U0001cfff\U0001d0f6-\U0001d0ff\U0001d127-\U0001d128\U0001d1de-\U0001d1ff\U0001d246-\U0001d2ff\U0001d357-\U0001d35f\U0001d372-\U0001d3ff\U0001d455\U0001d49d\U0001d4a0-\U0001d4a1\U0001d4a3-\U0001d4a4\U0001d4a7-\U0001d4a8\U0001d4ad\U0001d4ba\U0001d4bc\U0001d4c4\U0001d506\U0001d50b-\U0001d50c\U0001d515\U0001d51d\U0001d53a\U0001d53f\U0001d545\U0001d547-\U0001d549\U0001d551\U0001d6a6-\U0001d6a7\U0001d7cc-\U0001d7cd\U0001d800-\U0001edff\U0001ee04\U0001ee20\U0001ee23\U0001ee25-\U0001ee26\U0001ee28\U0001ee33\U0001ee38\U0001ee3a\U0001ee3c-\U0001ee41\U0001ee43-\U0001ee46\U0001ee48\U0001ee4a\U0001ee4c\U0001ee50\U0001ee53\U0001ee55-\U0001ee56\U0001ee58\U0001ee5a\U0001ee5c\U0001ee5e\U0001ee60\U0001ee63\U0001ee65-\U0001ee66\U0001ee6b\U0001ee73\U0001ee78\U0001ee7d\U0001ee7f\U0001ee8a\U0001ee9c-\U0001eea0\U0001eea4\U0001eeaa\U0001eebc-\U0001eeef\U0001eef2-\U0001efff\U0001f02c-\U0001f02f\U0001f094-\U0001f09f\U0001f0af-\U0001f0b0\U0001f0bf-\U0001f0c0\U0001f0d0\U0001f0e0-\U0001f0ff\U0001f10b-\U0001f10f\U0001f12f\U0001f16c-\U0001f16f\U0001f19b-\U0001f1e5\U0001f203-\U0001f20f\U0001f23b-\U0001f23f\U0001f249-\U0001f24f\U0001f252-\U0001f2ff\U0001f321-\U0001f32f\U0001f336\U0001f37d-\U0001f37f\U0001f394-\U0001f39f\U0001f3c5\U0001f3cb-\U0001f3df\U0001f3f1-\U0001f3ff\U0001f43f\U0001f441\U0001f4f8\U0001f4fd-\U0001f4ff\U0001f53e-\U0001f53f\U0001f544-\U0001f54f\U0001f568-\U0001f5fa\U0001f641-\U0001f644\U0001f650-\U0001f67f\U0001f6c6-\U0001f6ff\U0001f774-\U0001ffff\U0002a6d7-\U0002a6ff\U0002b735-\U0002b73f\U0002b81e-\U0002f7ff\U0002fa1e-\U000e0000\U000e0002-\U000e001f\U000e0080-\U000e00ff\U000e01f0-\U000effff\U000ffffe-\U000fffff\U0010fffe-\U0010ffff' + Cn += u'\U0001000c\U00010027\U0001003b\U0001003e\U0001004e-\U0001004f\U0001005e-\U0001007f\U000100fb-\U000100ff\U00010103-\U00010106\U00010134-\U00010136\U0001018f\U0001019c-\U0001019f\U000101a1-\U000101cf\U000101fe-\U0001027f\U0001029d-\U0001029f\U000102d1-\U000102df\U000102fc-\U000102ff\U00010324-\U0001032c\U0001034b-\U0001034f\U0001037b-\U0001037f\U0001039e\U000103c4-\U000103c7\U000103d6-\U000103ff\U0001049e-\U0001049f\U000104aa-\U000104af\U000104d4-\U000104d7\U000104fc-\U000104ff\U00010528-\U0001052f\U00010564-\U0001056e\U00010570-\U000105ff\U00010737-\U0001073f\U00010756-\U0001075f\U00010768-\U000107ff\U00010806-\U00010807\U00010809\U00010836\U00010839-\U0001083b\U0001083d-\U0001083e\U00010856\U0001089f-\U000108a6\U000108b0-\U000108df\U000108f3\U000108f6-\U000108fa\U0001091c-\U0001091e\U0001093a-\U0001093e\U00010940-\U0001097f\U000109b8-\U000109bb\U000109d0-\U000109d1\U00010a04\U00010a07-\U00010a0b\U00010a14\U00010a18\U00010a36-\U00010a37\U00010a3b-\U00010a3e\U00010a49-\U00010a4f\U00010a59-\U00010a5f\U00010aa0-\U00010abf\U00010ae7-\U00010aea\U00010af7-\U00010aff\U00010b36-\U00010b38\U00010b56-\U00010b57\U00010b73-\U00010b77\U00010b92-\U00010b98\U00010b9d-\U00010ba8\U00010bb0-\U00010bff\U00010c49-\U00010c7f\U00010cb3-\U00010cbf\U00010cf3-\U00010cf9\U00010d28-\U00010d2f\U00010d3a-\U00010e5f\U00010e7f-\U00010eff\U00010f28-\U00010f2f\U00010f5a-\U00010fff\U0001104e-\U00011051\U00011070-\U0001107e\U000110c2-\U000110cc\U000110ce-\U000110cf\U000110e9-\U000110ef\U000110fa-\U000110ff\U00011135\U00011147-\U0001114f\U00011177-\U0001117f\U000111ce-\U000111cf\U000111e0\U000111f5-\U000111ff\U00011212\U0001123f-\U0001127f\U00011287\U00011289\U0001128e\U0001129e\U000112aa-\U000112af\U000112eb-\U000112ef\U000112fa-\U000112ff\U00011304\U0001130d-\U0001130e\U00011311-\U00011312\U00011329\U00011331\U00011334\U0001133a\U00011345-\U00011346\U00011349-\U0001134a\U0001134e-\U0001134f\U00011351-\U00011356\U00011358-\U0001135c\U00011364-\U00011365\U0001136d-\U0001136f\U00011375-\U000113ff\U0001145a\U0001145c\U0001145f-\U0001147f\U000114c8-\U000114cf\U000114da-\U0001157f\U000115b6-\U000115b7\U000115de-\U000115ff\U00011645-\U0001164f\U0001165a-\U0001165f\U0001166d-\U0001167f\U000116b8-\U000116bf\U000116ca-\U000116ff\U0001171b-\U0001171c\U0001172c-\U0001172f\U00011740-\U000117ff\U0001183c-\U0001189f\U000118f3-\U000118fe\U00011900-\U000119ff\U00011a48-\U00011a4f\U00011a84-\U00011a85\U00011aa3-\U00011abf\U00011af9-\U00011bff\U00011c09\U00011c37\U00011c46-\U00011c4f\U00011c6d-\U00011c6f\U00011c90-\U00011c91\U00011ca8\U00011cb7-\U00011cff\U00011d07\U00011d0a\U00011d37-\U00011d39\U00011d3b\U00011d3e\U00011d48-\U00011d4f\U00011d5a-\U00011d5f\U00011d66\U00011d69\U00011d8f\U00011d92\U00011d99-\U00011d9f\U00011daa-\U00011edf\U00011ef9-\U00011fff\U0001239a-\U000123ff\U0001246f\U00012475-\U0001247f\U00012544-\U00012fff\U0001342f-\U000143ff\U00014647-\U000167ff\U00016a39-\U00016a3f\U00016a5f\U00016a6a-\U00016a6d\U00016a70-\U00016acf\U00016aee-\U00016aef\U00016af6-\U00016aff\U00016b46-\U00016b4f\U00016b5a\U00016b62\U00016b78-\U00016b7c\U00016b90-\U00016e3f\U00016e9b-\U00016eff\U00016f45-\U00016f4f\U00016f7f-\U00016f8e\U00016fa0-\U00016fdf\U00016fe2-\U00016fff\U000187f2-\U000187ff\U00018af3-\U0001afff\U0001b11f-\U0001b16f\U0001b2fc-\U0001bbff\U0001bc6b-\U0001bc6f\U0001bc7d-\U0001bc7f\U0001bc89-\U0001bc8f\U0001bc9a-\U0001bc9b\U0001bca4-\U0001cfff\U0001d0f6-\U0001d0ff\U0001d127-\U0001d128\U0001d1e9-\U0001d1ff\U0001d246-\U0001d2df\U0001d2f4-\U0001d2ff\U0001d357-\U0001d35f\U0001d379-\U0001d3ff\U0001d455\U0001d49d\U0001d4a0-\U0001d4a1\U0001d4a3-\U0001d4a4\U0001d4a7-\U0001d4a8\U0001d4ad\U0001d4ba\U0001d4bc\U0001d4c4\U0001d506\U0001d50b-\U0001d50c\U0001d515\U0001d51d\U0001d53a\U0001d53f\U0001d545\U0001d547-\U0001d549\U0001d551\U0001d6a6-\U0001d6a7\U0001d7cc-\U0001d7cd\U0001da8c-\U0001da9a\U0001daa0\U0001dab0-\U0001dfff\U0001e007\U0001e019-\U0001e01a\U0001e022\U0001e025\U0001e02b-\U0001e7ff\U0001e8c5-\U0001e8c6\U0001e8d7-\U0001e8ff\U0001e94b-\U0001e94f\U0001e95a-\U0001e95d\U0001e960-\U0001ec70\U0001ecb5-\U0001edff\U0001ee04\U0001ee20\U0001ee23\U0001ee25-\U0001ee26\U0001ee28\U0001ee33\U0001ee38\U0001ee3a\U0001ee3c-\U0001ee41\U0001ee43-\U0001ee46\U0001ee48\U0001ee4a\U0001ee4c\U0001ee50\U0001ee53\U0001ee55-\U0001ee56\U0001ee58\U0001ee5a\U0001ee5c\U0001ee5e\U0001ee60\U0001ee63\U0001ee65-\U0001ee66\U0001ee6b\U0001ee73\U0001ee78\U0001ee7d\U0001ee7f\U0001ee8a\U0001ee9c-\U0001eea0\U0001eea4\U0001eeaa\U0001eebc-\U0001eeef\U0001eef2-\U0001efff\U0001f02c-\U0001f02f\U0001f094-\U0001f09f\U0001f0af-\U0001f0b0\U0001f0c0\U0001f0d0\U0001f0f6-\U0001f0ff\U0001f10d-\U0001f10f\U0001f16c-\U0001f16f\U0001f1ad-\U0001f1e5\U0001f203-\U0001f20f\U0001f23c-\U0001f23f\U0001f249-\U0001f24f\U0001f252-\U0001f25f\U0001f266-\U0001f2ff\U0001f6d5-\U0001f6df\U0001f6ed-\U0001f6ef\U0001f6fa-\U0001f6ff\U0001f774-\U0001f77f\U0001f7d9-\U0001f7ff\U0001f80c-\U0001f80f\U0001f848-\U0001f84f\U0001f85a-\U0001f85f\U0001f888-\U0001f88f\U0001f8ae-\U0001f8ff\U0001f90c-\U0001f90f\U0001f93f\U0001f971-\U0001f972\U0001f977-\U0001f979\U0001f97b\U0001f9a3-\U0001f9af\U0001f9ba-\U0001f9bf\U0001f9c3-\U0001f9cf\U0001fa00-\U0001fa5f\U0001fa6e-\U0001ffff\U0002a6d7-\U0002a6ff\U0002b735-\U0002b73f\U0002b81e-\U0002b81f\U0002cea2-\U0002ceaf\U0002ebe1-\U0002f7ff\U0002fa1e-\U000e0000\U000e0002-\U000e001f\U000e0080-\U000e00ff\U000e01f0-\U000effff\U000ffffe-\U000fffff\U0010fffe-\U0010ffff' Co += u'\U000f0000-\U000ffffd\U00100000-\U0010fffd' - Ll += u'\U00010428-\U0001044f\U0001d41a-\U0001d433\U0001d44e-\U0001d454\U0001d456-\U0001d467\U0001d482-\U0001d49b\U0001d4b6-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d4cf\U0001d4ea-\U0001d503\U0001d51e-\U0001d537\U0001d552-\U0001d56b\U0001d586-\U0001d59f\U0001d5ba-\U0001d5d3\U0001d5ee-\U0001d607\U0001d622-\U0001d63b\U0001d656-\U0001d66f\U0001d68a-\U0001d6a5\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6e1\U0001d6fc-\U0001d714\U0001d716-\U0001d71b\U0001d736-\U0001d74e\U0001d750-\U0001d755\U0001d770-\U0001d788\U0001d78a-\U0001d78f\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7c9\U0001d7cb' + Ll += u'\U00010428-\U0001044f\U000104d8-\U000104fb\U00010cc0-\U00010cf2\U000118c0-\U000118df\U00016e60-\U00016e7f\U0001d41a-\U0001d433\U0001d44e-\U0001d454\U0001d456-\U0001d467\U0001d482-\U0001d49b\U0001d4b6-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d4cf\U0001d4ea-\U0001d503\U0001d51e-\U0001d537\U0001d552-\U0001d56b\U0001d586-\U0001d59f\U0001d5ba-\U0001d5d3\U0001d5ee-\U0001d607\U0001d622-\U0001d63b\U0001d656-\U0001d66f\U0001d68a-\U0001d6a5\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6e1\U0001d6fc-\U0001d714\U0001d716-\U0001d71b\U0001d736-\U0001d74e\U0001d750-\U0001d755\U0001d770-\U0001d788\U0001d78a-\U0001d78f\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7c9\U0001d7cb\U0001e922-\U0001e943' - Lm += u'\U00016f93-\U00016f9f' + Lm += u'\U00016b40-\U00016b43\U00016f93-\U00016f9f\U00016fe0-\U00016fe1' - Lo += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U00010340\U00010342-\U00010349\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U00010450-\U0001049d\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011183-\U000111b2\U000111c1-\U000111c4\U00011680-\U000116aa\U00012000-\U0001236e\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50\U0001b000-\U0001b001\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d' + Lo += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031f\U0001032d-\U00010340\U00010342-\U00010349\U00010350-\U00010375\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U00010450-\U0001049d\U00010500-\U00010527\U00010530-\U00010563\U00010600-\U00010736\U00010740-\U00010755\U00010760-\U00010767\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010860-\U00010876\U00010880-\U0001089e\U000108e0-\U000108f2\U000108f4-\U000108f5\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a35\U00010a60-\U00010a7c\U00010a80-\U00010a9c\U00010ac0-\U00010ac7\U00010ac9-\U00010ae4\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010b80-\U00010b91\U00010c00-\U00010c48\U00010d00-\U00010d23\U00010f00-\U00010f1c\U00010f27\U00010f30-\U00010f45\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011144\U00011150-\U00011172\U00011176\U00011183-\U000111b2\U000111c1-\U000111c4\U000111da\U000111dc\U00011200-\U00011211\U00011213-\U0001122b\U00011280-\U00011286\U00011288\U0001128a-\U0001128d\U0001128f-\U0001129d\U0001129f-\U000112a8\U000112b0-\U000112de\U00011305-\U0001130c\U0001130f-\U00011310\U00011313-\U00011328\U0001132a-\U00011330\U00011332-\U00011333\U00011335-\U00011339\U0001133d\U00011350\U0001135d-\U00011361\U00011400-\U00011434\U00011447-\U0001144a\U00011480-\U000114af\U000114c4-\U000114c5\U000114c7\U00011580-\U000115ae\U000115d8-\U000115db\U00011600-\U0001162f\U00011644\U00011680-\U000116aa\U00011700-\U0001171a\U00011800-\U0001182b\U000118ff\U00011a00\U00011a0b-\U00011a32\U00011a3a\U00011a50\U00011a5c-\U00011a83\U00011a86-\U00011a89\U00011a9d\U00011ac0-\U00011af8\U00011c00-\U00011c08\U00011c0a-\U00011c2e\U00011c40\U00011c72-\U00011c8f\U00011d00-\U00011d06\U00011d08-\U00011d09\U00011d0b-\U00011d30\U00011d46\U00011d60-\U00011d65\U00011d67-\U00011d68\U00011d6a-\U00011d89\U00011d98\U00011ee0-\U00011ef2\U00012000-\U00012399\U00012480-\U00012543\U00013000-\U0001342e\U00014400-\U00014646\U00016800-\U00016a38\U00016a40-\U00016a5e\U00016ad0-\U00016aed\U00016b00-\U00016b2f\U00016b63-\U00016b77\U00016b7d-\U00016b8f\U00016f00-\U00016f44\U00016f50\U00017000-\U000187f1\U00018800-\U00018af2\U0001b000-\U0001b11e\U0001b170-\U0001b2fb\U0001bc00-\U0001bc6a\U0001bc70-\U0001bc7c\U0001bc80-\U0001bc88\U0001bc90-\U0001bc99\U0001e800-\U0001e8c4\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002b820-\U0002cea1\U0002ceb0-\U0002ebe0\U0002f800-\U0002fa1d' - Lu += u'\U00010400-\U00010427\U0001d400-\U0001d419\U0001d434-\U0001d44d\U0001d468-\U0001d481\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b5\U0001d4d0-\U0001d4e9\U0001d504-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d538-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d56c-\U0001d585\U0001d5a0-\U0001d5b9\U0001d5d4-\U0001d5ed\U0001d608-\U0001d621\U0001d63c-\U0001d655\U0001d670-\U0001d689\U0001d6a8-\U0001d6c0\U0001d6e2-\U0001d6fa\U0001d71c-\U0001d734\U0001d756-\U0001d76e\U0001d790-\U0001d7a8\U0001d7ca' + Lu += u'\U00010400-\U00010427\U000104b0-\U000104d3\U00010c80-\U00010cb2\U000118a0-\U000118bf\U00016e40-\U00016e5f\U0001d400-\U0001d419\U0001d434-\U0001d44d\U0001d468-\U0001d481\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b5\U0001d4d0-\U0001d4e9\U0001d504-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d538-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d56c-\U0001d585\U0001d5a0-\U0001d5b9\U0001d5d4-\U0001d5ed\U0001d608-\U0001d621\U0001d63c-\U0001d655\U0001d670-\U0001d689\U0001d6a8-\U0001d6c0\U0001d6e2-\U0001d6fa\U0001d71c-\U0001d734\U0001d756-\U0001d76e\U0001d790-\U0001d7a8\U0001d7ca\U0001e900-\U0001e921' - Mc += u'\U00011000\U00011002\U00011082\U000110b0-\U000110b2\U000110b7-\U000110b8\U0001112c\U00011182\U000111b3-\U000111b5\U000111bf-\U000111c0\U000116ac\U000116ae-\U000116af\U000116b6\U00016f51-\U00016f7e\U0001d165-\U0001d166\U0001d16d-\U0001d172' + Mc += u'\U00011000\U00011002\U00011082\U000110b0-\U000110b2\U000110b7-\U000110b8\U0001112c\U00011145-\U00011146\U00011182\U000111b3-\U000111b5\U000111bf-\U000111c0\U0001122c-\U0001122e\U00011232-\U00011233\U00011235\U000112e0-\U000112e2\U00011302-\U00011303\U0001133e-\U0001133f\U00011341-\U00011344\U00011347-\U00011348\U0001134b-\U0001134d\U00011357\U00011362-\U00011363\U00011435-\U00011437\U00011440-\U00011441\U00011445\U000114b0-\U000114b2\U000114b9\U000114bb-\U000114be\U000114c1\U000115af-\U000115b1\U000115b8-\U000115bb\U000115be\U00011630-\U00011632\U0001163b-\U0001163c\U0001163e\U000116ac\U000116ae-\U000116af\U000116b6\U00011720-\U00011721\U00011726\U0001182c-\U0001182e\U00011838\U00011a39\U00011a57-\U00011a58\U00011a97\U00011c2f\U00011c3e\U00011ca9\U00011cb1\U00011cb4\U00011d8a-\U00011d8e\U00011d93-\U00011d94\U00011d96\U00011ef5-\U00011ef6\U00016f51-\U00016f7e\U0001d165-\U0001d166\U0001d16d-\U0001d172' - Mn += u'\U000101fd\U00010a01-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a0f\U00010a38-\U00010a3a\U00010a3f\U00011001\U00011038-\U00011046\U00011080-\U00011081\U000110b3-\U000110b6\U000110b9-\U000110ba\U00011100-\U00011102\U00011127-\U0001112b\U0001112d-\U00011134\U00011180-\U00011181\U000111b6-\U000111be\U000116ab\U000116ad\U000116b0-\U000116b5\U000116b7\U00016f8f-\U00016f92\U0001d167-\U0001d169\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U000e0100-\U000e01ef' + Mn += u'\U000101fd\U000102e0\U00010376-\U0001037a\U00010a01-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a0f\U00010a38-\U00010a3a\U00010a3f\U00010ae5-\U00010ae6\U00010d24-\U00010d27\U00010f46-\U00010f50\U00011001\U00011038-\U00011046\U0001107f-\U00011081\U000110b3-\U000110b6\U000110b9-\U000110ba\U00011100-\U00011102\U00011127-\U0001112b\U0001112d-\U00011134\U00011173\U00011180-\U00011181\U000111b6-\U000111be\U000111c9-\U000111cc\U0001122f-\U00011231\U00011234\U00011236-\U00011237\U0001123e\U000112df\U000112e3-\U000112ea\U00011300-\U00011301\U0001133b-\U0001133c\U00011340\U00011366-\U0001136c\U00011370-\U00011374\U00011438-\U0001143f\U00011442-\U00011444\U00011446\U0001145e\U000114b3-\U000114b8\U000114ba\U000114bf-\U000114c0\U000114c2-\U000114c3\U000115b2-\U000115b5\U000115bc-\U000115bd\U000115bf-\U000115c0\U000115dc-\U000115dd\U00011633-\U0001163a\U0001163d\U0001163f-\U00011640\U000116ab\U000116ad\U000116b0-\U000116b5\U000116b7\U0001171d-\U0001171f\U00011722-\U00011725\U00011727-\U0001172b\U0001182f-\U00011837\U00011839-\U0001183a\U00011a01-\U00011a0a\U00011a33-\U00011a38\U00011a3b-\U00011a3e\U00011a47\U00011a51-\U00011a56\U00011a59-\U00011a5b\U00011a8a-\U00011a96\U00011a98-\U00011a99\U00011c30-\U00011c36\U00011c38-\U00011c3d\U00011c3f\U00011c92-\U00011ca7\U00011caa-\U00011cb0\U00011cb2-\U00011cb3\U00011cb5-\U00011cb6\U00011d31-\U00011d36\U00011d3a\U00011d3c-\U00011d3d\U00011d3f-\U00011d45\U00011d47\U00011d90-\U00011d91\U00011d95\U00011d97\U00011ef3-\U00011ef4\U00016af0-\U00016af4\U00016b30-\U00016b36\U00016f8f-\U00016f92\U0001bc9d-\U0001bc9e\U0001d167-\U0001d169\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U0001da00-\U0001da36\U0001da3b-\U0001da6c\U0001da75\U0001da84\U0001da9b-\U0001da9f\U0001daa1-\U0001daaf\U0001e000-\U0001e006\U0001e008-\U0001e018\U0001e01b-\U0001e021\U0001e023-\U0001e024\U0001e026-\U0001e02a\U0001e8d0-\U0001e8d6\U0001e944-\U0001e94a\U000e0100-\U000e01ef' - Nd += u'\U000104a0-\U000104a9\U00011066-\U0001106f\U000110f0-\U000110f9\U00011136-\U0001113f\U000111d0-\U000111d9\U000116c0-\U000116c9\U0001d7ce-\U0001d7ff' + Nd += u'\U000104a0-\U000104a9\U00010d30-\U00010d39\U00011066-\U0001106f\U000110f0-\U000110f9\U00011136-\U0001113f\U000111d0-\U000111d9\U000112f0-\U000112f9\U00011450-\U00011459\U000114d0-\U000114d9\U00011650-\U00011659\U000116c0-\U000116c9\U00011730-\U00011739\U000118e0-\U000118e9\U00011c50-\U00011c59\U00011d50-\U00011d59\U00011da0-\U00011da9\U00016a60-\U00016a69\U00016b50-\U00016b59\U0001d7ce-\U0001d7ff\U0001e950-\U0001e959' - Nl += u'\U00010140-\U00010174\U00010341\U0001034a\U000103d1-\U000103d5\U00012400-\U00012462' + Nl += u'\U00010140-\U00010174\U00010341\U0001034a\U000103d1-\U000103d5\U00012400-\U0001246e' - No += u'\U00010107-\U00010133\U00010175-\U00010178\U0001018a\U00010320-\U00010323\U00010858-\U0001085f\U00010916-\U0001091b\U00010a40-\U00010a47\U00010a7d-\U00010a7e\U00010b58-\U00010b5f\U00010b78-\U00010b7f\U00010e60-\U00010e7e\U00011052-\U00011065\U0001d360-\U0001d371\U0001f100-\U0001f10a' + No += u'\U00010107-\U00010133\U00010175-\U00010178\U0001018a-\U0001018b\U000102e1-\U000102fb\U00010320-\U00010323\U00010858-\U0001085f\U00010879-\U0001087f\U000108a7-\U000108af\U000108fb-\U000108ff\U00010916-\U0001091b\U000109bc-\U000109bd\U000109c0-\U000109cf\U000109d2-\U000109ff\U00010a40-\U00010a48\U00010a7d-\U00010a7e\U00010a9d-\U00010a9f\U00010aeb-\U00010aef\U00010b58-\U00010b5f\U00010b78-\U00010b7f\U00010ba9-\U00010baf\U00010cfa-\U00010cff\U00010e60-\U00010e7e\U00010f1d-\U00010f26\U00010f51-\U00010f54\U00011052-\U00011065\U000111e1-\U000111f4\U0001173a-\U0001173b\U000118ea-\U000118f2\U00011c5a-\U00011c6c\U00016b5b-\U00016b61\U00016e80-\U00016e96\U0001d2e0-\U0001d2f3\U0001d360-\U0001d378\U0001e8c7-\U0001e8cf\U0001ec71-\U0001ecab\U0001ecad-\U0001ecaf\U0001ecb1-\U0001ecb4\U0001f100-\U0001f10c' - Po += u'\U00010100-\U00010102\U0001039f\U000103d0\U00010857\U0001091f\U0001093f\U00010a50-\U00010a58\U00010a7f\U00010b39-\U00010b3f\U00011047-\U0001104d\U000110bb-\U000110bc\U000110be-\U000110c1\U00011140-\U00011143\U000111c5-\U000111c8\U00012470-\U00012473' + Po += u'\U00010100-\U00010102\U0001039f\U000103d0\U0001056f\U00010857\U0001091f\U0001093f\U00010a50-\U00010a58\U00010a7f\U00010af0-\U00010af6\U00010b39-\U00010b3f\U00010b99-\U00010b9c\U00010f55-\U00010f59\U00011047-\U0001104d\U000110bb-\U000110bc\U000110be-\U000110c1\U00011140-\U00011143\U00011174-\U00011175\U000111c5-\U000111c8\U000111cd\U000111db\U000111dd-\U000111df\U00011238-\U0001123d\U000112a9\U0001144b-\U0001144f\U0001145b\U0001145d\U000114c6\U000115c1-\U000115d7\U00011641-\U00011643\U00011660-\U0001166c\U0001173c-\U0001173e\U0001183b\U00011a3f-\U00011a46\U00011a9a-\U00011a9c\U00011a9e-\U00011aa2\U00011c41-\U00011c45\U00011c70-\U00011c71\U00011ef7-\U00011ef8\U00012470-\U00012474\U00016a6e-\U00016a6f\U00016af5\U00016b37-\U00016b3b\U00016b44\U00016e97-\U00016e9a\U0001bc9f\U0001da87-\U0001da8b\U0001e95e-\U0001e95f' + + Sc += u'\U0001ecb0' + + Sk += u'\U0001f3fb-\U0001f3ff' Sm += u'\U0001d6c1\U0001d6db\U0001d6fb\U0001d715\U0001d735\U0001d74f\U0001d76f\U0001d789\U0001d7a9\U0001d7c3\U0001eef0-\U0001eef1' - So += u'\U00010137-\U0001013f\U00010179-\U00010189\U00010190-\U0001019b\U000101d0-\U000101fc\U0001d000-\U0001d0f5\U0001d100-\U0001d126\U0001d129-\U0001d164\U0001d16a-\U0001d16c\U0001d183-\U0001d184\U0001d18c-\U0001d1a9\U0001d1ae-\U0001d1dd\U0001d200-\U0001d241\U0001d245\U0001d300-\U0001d356\U0001f000-\U0001f02b\U0001f030-\U0001f093\U0001f0a0-\U0001f0ae\U0001f0b1-\U0001f0be\U0001f0c1-\U0001f0cf\U0001f0d1-\U0001f0df\U0001f110-\U0001f12e\U0001f130-\U0001f16b\U0001f170-\U0001f19a\U0001f1e6-\U0001f202\U0001f210-\U0001f23a\U0001f240-\U0001f248\U0001f250-\U0001f251\U0001f300-\U0001f320\U0001f330-\U0001f335\U0001f337-\U0001f37c\U0001f380-\U0001f393\U0001f3a0-\U0001f3c4\U0001f3c6-\U0001f3ca\U0001f3e0-\U0001f3f0\U0001f400-\U0001f43e\U0001f440\U0001f442-\U0001f4f7\U0001f4f9-\U0001f4fc\U0001f500-\U0001f53d\U0001f540-\U0001f543\U0001f550-\U0001f567\U0001f5fb-\U0001f640\U0001f645-\U0001f64f\U0001f680-\U0001f6c5\U0001f700-\U0001f773' + So += u'\U00010137-\U0001013f\U00010179-\U00010189\U0001018c-\U0001018e\U00010190-\U0001019b\U000101a0\U000101d0-\U000101fc\U00010877-\U00010878\U00010ac8\U0001173f\U00016b3c-\U00016b3f\U00016b45\U0001bc9c\U0001d000-\U0001d0f5\U0001d100-\U0001d126\U0001d129-\U0001d164\U0001d16a-\U0001d16c\U0001d183-\U0001d184\U0001d18c-\U0001d1a9\U0001d1ae-\U0001d1e8\U0001d200-\U0001d241\U0001d245\U0001d300-\U0001d356\U0001d800-\U0001d9ff\U0001da37-\U0001da3a\U0001da6d-\U0001da74\U0001da76-\U0001da83\U0001da85-\U0001da86\U0001ecac\U0001f000-\U0001f02b\U0001f030-\U0001f093\U0001f0a0-\U0001f0ae\U0001f0b1-\U0001f0bf\U0001f0c1-\U0001f0cf\U0001f0d1-\U0001f0f5\U0001f110-\U0001f16b\U0001f170-\U0001f1ac\U0001f1e6-\U0001f202\U0001f210-\U0001f23b\U0001f240-\U0001f248\U0001f250-\U0001f251\U0001f260-\U0001f265\U0001f300-\U0001f3fa\U0001f400-\U0001f6d4\U0001f6e0-\U0001f6ec\U0001f6f0-\U0001f6f9\U0001f700-\U0001f773\U0001f780-\U0001f7d8\U0001f800-\U0001f80b\U0001f810-\U0001f847\U0001f850-\U0001f859\U0001f860-\U0001f887\U0001f890-\U0001f8ad\U0001f900-\U0001f90b\U0001f910-\U0001f93e\U0001f940-\U0001f970\U0001f973-\U0001f976\U0001f97a\U0001f97c-\U0001f9a2\U0001f9b0-\U0001f9b9\U0001f9c0-\U0001f9c2\U0001f9d0-\U0001f9ff\U0001fa60-\U0001fa6d' - xid_continue += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U000101fd\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U0001034a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U000104a0-\U000104a9\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a38-\U00010a3a\U00010a3f\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011000-\U00011046\U00011066-\U0001106f\U00011080-\U000110ba\U000110d0-\U000110e8\U000110f0-\U000110f9\U00011100-\U00011134\U00011136-\U0001113f\U00011180-\U000111c4\U000111d0-\U000111d9\U00011680-\U000116b7\U000116c0-\U000116c9\U00012000-\U0001236e\U00012400-\U00012462\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50-\U00016f7e\U00016f8f-\U00016f9f\U0001b000-\U0001b001\U0001d165-\U0001d169\U0001d16d-\U0001d172\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001d7ce-\U0001d7ff\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d\U000e0100-\U000e01ef' + xid_continue += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U000101fd\U00010280-\U0001029c\U000102a0-\U000102d0\U000102e0\U00010300-\U0001031f\U0001032d-\U0001034a\U00010350-\U0001037a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U000104a0-\U000104a9\U000104b0-\U000104d3\U000104d8-\U000104fb\U00010500-\U00010527\U00010530-\U00010563\U00010600-\U00010736\U00010740-\U00010755\U00010760-\U00010767\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010860-\U00010876\U00010880-\U0001089e\U000108e0-\U000108f2\U000108f4-\U000108f5\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a35\U00010a38-\U00010a3a\U00010a3f\U00010a60-\U00010a7c\U00010a80-\U00010a9c\U00010ac0-\U00010ac7\U00010ac9-\U00010ae6\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010b80-\U00010b91\U00010c00-\U00010c48\U00010c80-\U00010cb2\U00010cc0-\U00010cf2\U00010d00-\U00010d27\U00010d30-\U00010d39\U00010f00-\U00010f1c\U00010f27\U00010f30-\U00010f50\U00011000-\U00011046\U00011066-\U0001106f\U0001107f-\U000110ba\U000110d0-\U000110e8\U000110f0-\U000110f9\U00011100-\U00011134\U00011136-\U0001113f\U00011144-\U00011146\U00011150-\U00011173\U00011176\U00011180-\U000111c4\U000111c9-\U000111cc\U000111d0-\U000111da\U000111dc\U00011200-\U00011211\U00011213-\U00011237\U0001123e\U00011280-\U00011286\U00011288\U0001128a-\U0001128d\U0001128f-\U0001129d\U0001129f-\U000112a8\U000112b0-\U000112ea\U000112f0-\U000112f9\U00011300-\U00011303\U00011305-\U0001130c\U0001130f-\U00011310\U00011313-\U00011328\U0001132a-\U00011330\U00011332-\U00011333\U00011335-\U00011339\U0001133b-\U00011344\U00011347-\U00011348\U0001134b-\U0001134d\U00011350\U00011357\U0001135d-\U00011363\U00011366-\U0001136c\U00011370-\U00011374\U00011400-\U0001144a\U00011450-\U00011459\U0001145e\U00011480-\U000114c5\U000114c7\U000114d0-\U000114d9\U00011580-\U000115b5\U000115b8-\U000115c0\U000115d8-\U000115dd\U00011600-\U00011640\U00011644\U00011650-\U00011659\U00011680-\U000116b7\U000116c0-\U000116c9\U00011700-\U0001171a\U0001171d-\U0001172b\U00011730-\U00011739\U00011800-\U0001183a\U000118a0-\U000118e9\U000118ff\U00011a00-\U00011a3e\U00011a47\U00011a50-\U00011a83\U00011a86-\U00011a99\U00011a9d\U00011ac0-\U00011af8\U00011c00-\U00011c08\U00011c0a-\U00011c36\U00011c38-\U00011c40\U00011c50-\U00011c59\U00011c72-\U00011c8f\U00011c92-\U00011ca7\U00011ca9-\U00011cb6\U00011d00-\U00011d06\U00011d08-\U00011d09\U00011d0b-\U00011d36\U00011d3a\U00011d3c-\U00011d3d\U00011d3f-\U00011d47\U00011d50-\U00011d59\U00011d60-\U00011d65\U00011d67-\U00011d68\U00011d6a-\U00011d8e\U00011d90-\U00011d91\U00011d93-\U00011d98\U00011da0-\U00011da9\U00011ee0-\U00011ef6\U00012000-\U00012399\U00012400-\U0001246e\U00012480-\U00012543\U00013000-\U0001342e\U00014400-\U00014646\U00016800-\U00016a38\U00016a40-\U00016a5e\U00016a60-\U00016a69\U00016ad0-\U00016aed\U00016af0-\U00016af4\U00016b00-\U00016b36\U00016b40-\U00016b43\U00016b50-\U00016b59\U00016b63-\U00016b77\U00016b7d-\U00016b8f\U00016e40-\U00016e7f\U00016f00-\U00016f44\U00016f50-\U00016f7e\U00016f8f-\U00016f9f\U00016fe0-\U00016fe1\U00017000-\U000187f1\U00018800-\U00018af2\U0001b000-\U0001b11e\U0001b170-\U0001b2fb\U0001bc00-\U0001bc6a\U0001bc70-\U0001bc7c\U0001bc80-\U0001bc88\U0001bc90-\U0001bc99\U0001bc9d-\U0001bc9e\U0001d165-\U0001d169\U0001d16d-\U0001d172\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001d7ce-\U0001d7ff\U0001da00-\U0001da36\U0001da3b-\U0001da6c\U0001da75\U0001da84\U0001da9b-\U0001da9f\U0001daa1-\U0001daaf\U0001e000-\U0001e006\U0001e008-\U0001e018\U0001e01b-\U0001e021\U0001e023-\U0001e024\U0001e026-\U0001e02a\U0001e800-\U0001e8c4\U0001e8d0-\U0001e8d6\U0001e900-\U0001e94a\U0001e950-\U0001e959\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002b820-\U0002cea1\U0002ceb0-\U0002ebe0\U0002f800-\U0002fa1d\U000e0100-\U000e01ef' - xid_start += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U0001034a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011183-\U000111b2\U000111c1-\U000111c4\U00011680-\U000116aa\U00012000-\U0001236e\U00012400-\U00012462\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50\U00016f93-\U00016f9f\U0001b000-\U0001b001\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d' + xid_start += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031f\U0001032d-\U0001034a\U00010350-\U00010375\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U000104b0-\U000104d3\U000104d8-\U000104fb\U00010500-\U00010527\U00010530-\U00010563\U00010600-\U00010736\U00010740-\U00010755\U00010760-\U00010767\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010860-\U00010876\U00010880-\U0001089e\U000108e0-\U000108f2\U000108f4-\U000108f5\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a35\U00010a60-\U00010a7c\U00010a80-\U00010a9c\U00010ac0-\U00010ac7\U00010ac9-\U00010ae4\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010b80-\U00010b91\U00010c00-\U00010c48\U00010c80-\U00010cb2\U00010cc0-\U00010cf2\U00010d00-\U00010d23\U00010f00-\U00010f1c\U00010f27\U00010f30-\U00010f45\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011144\U00011150-\U00011172\U00011176\U00011183-\U000111b2\U000111c1-\U000111c4\U000111da\U000111dc\U00011200-\U00011211\U00011213-\U0001122b\U00011280-\U00011286\U00011288\U0001128a-\U0001128d\U0001128f-\U0001129d\U0001129f-\U000112a8\U000112b0-\U000112de\U00011305-\U0001130c\U0001130f-\U00011310\U00011313-\U00011328\U0001132a-\U00011330\U00011332-\U00011333\U00011335-\U00011339\U0001133d\U00011350\U0001135d-\U00011361\U00011400-\U00011434\U00011447-\U0001144a\U00011480-\U000114af\U000114c4-\U000114c5\U000114c7\U00011580-\U000115ae\U000115d8-\U000115db\U00011600-\U0001162f\U00011644\U00011680-\U000116aa\U00011700-\U0001171a\U00011800-\U0001182b\U000118a0-\U000118df\U000118ff\U00011a00\U00011a0b-\U00011a32\U00011a3a\U00011a50\U00011a5c-\U00011a83\U00011a86-\U00011a89\U00011a9d\U00011ac0-\U00011af8\U00011c00-\U00011c08\U00011c0a-\U00011c2e\U00011c40\U00011c72-\U00011c8f\U00011d00-\U00011d06\U00011d08-\U00011d09\U00011d0b-\U00011d30\U00011d46\U00011d60-\U00011d65\U00011d67-\U00011d68\U00011d6a-\U00011d89\U00011d98\U00011ee0-\U00011ef2\U00012000-\U00012399\U00012400-\U0001246e\U00012480-\U00012543\U00013000-\U0001342e\U00014400-\U00014646\U00016800-\U00016a38\U00016a40-\U00016a5e\U00016ad0-\U00016aed\U00016b00-\U00016b2f\U00016b40-\U00016b43\U00016b63-\U00016b77\U00016b7d-\U00016b8f\U00016e40-\U00016e7f\U00016f00-\U00016f44\U00016f50\U00016f93-\U00016f9f\U00016fe0-\U00016fe1\U00017000-\U000187f1\U00018800-\U00018af2\U0001b000-\U0001b11e\U0001b170-\U0001b2fb\U0001bc00-\U0001bc6a\U0001bc70-\U0001bc7c\U0001bc80-\U0001bc88\U0001bc90-\U0001bc99\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001e800-\U0001e8c4\U0001e900-\U0001e943\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002b820-\U0002cea1\U0002ceb0-\U0002ebe0\U0002f800-\U0002fa1d' cats = ['Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu', 'Mc', 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf', 'Pi', 'Po', 'Ps', 'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs'] -# Generated from unidata 6.3.0 +# Generated from unidata 11.0.0 def combine(*args): return u''.join(globals()[cat] for cat in args) diff --git a/scripts/release-checklist b/scripts/release-checklist new file mode 100644 index 00000000..efc1e1e8 --- /dev/null +++ b/scripts/release-checklist @@ -0,0 +1,24 @@ +Release checklist +================= + +* Check ``hg status`` +* ``make check`` +* LATER when configured properly: ``make pylint`` +* ``tox`` +* Update version info in ``setup.py/__init__.py`` +* Check setup.py metadata: long description, trove classifiers +* Update release date/code name in ``CHANGES`` +* ``hg commit`` +* ``make clean`` +* ``python2 setup.py release bdist_wheel`` +* ``python3 setup.py release bdist_wheel sdist`` +* ``twine upload dist/Pygments-$NEWVER*`` +* Check PyPI release page for obvious errors +* ``hg tag`` +* Merge default into stable if this was a ``x.y.0`` +* Update homepage (release info), regenerate docs (+printable!) +* Add new version/milestone to tracker categories +* Write announcement and send to mailing list/python-announce +* Update version info, add new ``CHANGES`` entry for next version +* ``hg commit`` +* ``hg push`` @@ -48,7 +48,7 @@ else: setup( name = 'Pygments', - version = '2.2.0', + version = '2.3.0', url = 'http://pygments.org/', license = 'BSD License', author = 'Georg Brandl', diff --git a/tests/examplefiles/99_bottles_of_beer.chpl b/tests/examplefiles/99_bottles_of_beer.chpl index cdc1e650..ff50b294 100644 --- a/tests/examplefiles/99_bottles_of_beer.chpl +++ b/tests/examplefiles/99_bottles_of_beer.chpl @@ -177,3 +177,30 @@ private module M3 { private var x: int; } +prototype module X { + + proc f() throws { + throw new Error(); + } + + proc g() { + try { + f(); + try! f(); + } catch e { + writeln("Caught ", e); + } + } + + proc int.add() { } + + g(); + + override proc test() throws { + var a = new borrowed IntPair(); + var b = new owned IntPair(); + var c = new shared IntPair(); + throw new unmanaged Error(); + } +} + diff --git a/tests/examplefiles/StdGeneric.icl b/tests/examplefiles/StdGeneric.icl index 2e6c3931..891b510a 100644 --- a/tests/examplefiles/StdGeneric.icl +++ b/tests/examplefiles/StdGeneric.icl @@ -1,5 +1,13 @@ implementation module StdGeneric +/** + * NOTE: this is a collection of different tricky parts of Clean modules (even + * though the file is simply called StdGeneric.icl). The code is taken from: + * + * - StdGeneric (StdEnv) + * - Graphics.Scalable.Image (Platform) + */ + import StdInt, StdMisc, StdClass, StdFunc generic bimap a b :: Bimap .a .b @@ -89,4 +97,38 @@ where = [ ConsLeft : doit i (n/2) ] | otherwise = [ ConsRight : doit (i - (n/2)) (n - (n/2)) ] -
\ No newline at end of file + +:: NoAttr m = NoAttr +:: DashAttr m = { dash :: ![Int] } +:: FillAttr m = { fill :: !SVGColor } +:: LineEndMarker m = { endmarker :: !Image m } +:: LineMidMarker m = { midmarker :: !Image m } +:: LineStartMarker m = { startmarker :: !Image m } +:: MaskAttr m = { mask :: !Image m } +:: OpacityAttr m = { opacity :: !Real } +:: StrokeAttr m = { stroke :: !SVGColor } +:: StrokeWidthAttr m = { strokewidth :: !Span } +:: XRadiusAttr m = { xradius :: !Span } +:: YRadiusAttr m = { yradius :: !Span } + + +instance tuneImage NoAttr where tuneImage image _ = image +instance tuneImage DashAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgDashAttr attr.DashAttr.dash)) image +instance tuneImage FillAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgFillAttr attr.FillAttr.fill)) image +instance tuneImage LineEndMarker where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineEndMarker.endmarker, markerPos = LineMarkerEnd}) image +instance tuneImage LineMidMarker where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineMidMarker.midmarker, markerPos = LineMarkerMid}) image +instance tuneImage LineStartMarker where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineStartMarker.startmarker, markerPos = LineMarkerStart}) image +instance tuneImage MaskAttr where tuneImage image attr = Attr` (MaskAttr` attr.MaskAttr.mask) image +instance tuneImage OpacityAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgFillOpacityAttr attr.OpacityAttr.opacity)) image +instance tuneImage StrokeAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgStrokeAttr attr.StrokeAttr.stroke)) image +instance tuneImage StrokeWidthAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgStrokeWidthAttr attr.StrokeWidthAttr.strokewidth)) image +instance tuneImage XRadiusAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgXRadiusAttr attr.XRadiusAttr.xradius)) image +instance tuneImage YRadiusAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgYRadiusAttr attr.YRadiusAttr.yradius)) image + +instance tuneImage DraggableAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerDraggableAttr attr)) image +instance tuneImage OnClickAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnClickAttr attr)) image +instance tuneImage OnMouseDownAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseDownAttr attr)) image +instance tuneImage OnMouseMoveAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseMoveAttr attr)) image +instance tuneImage OnMouseOutAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseOutAttr attr)) image +instance tuneImage OnMouseOverAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseOverAttr attr)) image +instance tuneImage OnMouseUpAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseUpAttr attr)) image diff --git a/tests/examplefiles/docker.docker b/tests/examplefiles/docker.docker index d65385b6..1ae3c3a1 100644 --- a/tests/examplefiles/docker.docker +++ b/tests/examplefiles/docker.docker @@ -1,5 +1,34 @@ -maintainer First O'Last +FROM alpine:3.5 +MAINTAINER First O'Last +# comment run echo \ 123 $bar -# comment +RUN apk --update add rsync dumb-init + +# Test env with both syntax +ENV FOO = "BAR" +ENV FOO \ + "BAR" + +COPY foo "bar" +COPY foo \ + "bar" + +HEALTHCHECK \ + --interval=5m --timeout=3s \ + CMD curl -f http://localhost/ || exit 1 + +# ONBUILD keyword, then with linebreak +ONBUILD ADD . /app/src +ONBUILD \ + RUN echo 123 $bar + +# Potential JSON array parsing, mixed with linebreaks +VOLUME \ + /foo +VOLUME \ + ["/bar"] +VOLUME ["/bar"] +VOLUME /foo +CMD ["foo", "bar"] diff --git a/tests/examplefiles/example.hlsl b/tests/examplefiles/example.hlsl new file mode 100644 index 00000000..21d0a672 --- /dev/null +++ b/tests/examplefiles/example.hlsl @@ -0,0 +1,168 @@ +// A few random snippets of HLSL shader code I gathered...
+
+[numthreads(256, 1, 1)]
+void cs_main(uint3 threadId : SV_DispatchThreadID)
+{
+ // Seed the PRNG using the thread ID
+ rng_state = threadId.x;
+
+ // Generate a few numbers...
+ uint r0 = rand_xorshift();
+ uint r1 = rand_xorshift();
+ // Do some stuff with them...
+
+ // Generate a random float in [0, 1)...
+ float f0 = float(rand_xorshift()) * (1.0 / 4294967296.0);
+
+ // ...etc.
+}
+
+// Constant buffer of parameters
+cbuffer IntegratorParams : register(b0)
+{
+ float2 specPow; // Spec powers in XY directions (equal for isotropic BRDFs)
+ float3 L; // Unit vector toward light
+ int2 cThread; // Total threads launched in XY dimensions
+ int2 xyOutput; // Where in the output buffer to store the result
+}
+
+static const float pi = 3.141592654;
+
+float AshikhminShirleyNDF(float3 H)
+{
+ float normFactor = sqrt((specPow.x + 2.0f) * (specPow.y + 2.0)) * (0.5f / pi);
+ float NdotH = H.z;
+ float2 Hxy = normalize(H.xy);
+ return normFactor * pow(NdotH, dot(specPow, Hxy * Hxy));
+}
+
+float BeckmannNDF(float3 H)
+{
+ float glossFactor = specPow.x * 0.5f + 1.0f; // This is 1/m^2 in the usual Beckmann formula
+ float normFactor = glossFactor * (1.0f / pi);
+ float NdotHSq = H.z * H.z;
+ return normFactor / (NdotHSq * NdotHSq) * exp(glossFactor * (1.0f - 1.0f / NdotHSq));
+}
+
+// Output buffer for compute shader (actually float, but must be declared as uint
+// for atomic operations to work)
+globallycoherent RWTexture2D<uint> o_data : register(u0);
+
+// Sum up the outputs of all threads and store to the output location
+static const uint threadGroupSize2D = 16;
+static const uint threadGroupSize1D = threadGroupSize2D * threadGroupSize2D;
+groupshared float g_partialSums[threadGroupSize1D];
+void SumAcrossThreadsAndStore(float value, uint iThreadInGroup)
+{
+ // First reduce within the threadgroup: partial sums of 2, 4, 8... elements
+ // are calculated by 1/2, 1/4, 1/8... of the threads, always keeping the
+ // active threads at the front of the group to minimize divergence.
+
+ // NOTE: there are faster ways of doing this...but this is simple to code
+ // and good enough.
+
+ g_partialSums[iThreadInGroup] = value;
+ GroupMemoryBarrierWithGroupSync();
+
+ [unroll] for (uint i = threadGroupSize1D / 2; i > 0; i /= 2)
+ {
+ if (iThreadInGroup < i)
+ {
+ g_partialSums[iThreadInGroup] += g_partialSums[iThreadInGroup + i];
+ }
+ GroupMemoryBarrierWithGroupSync();
+ }
+
+ // Then reduce across threadgroups: one thread from each group adds the group
+ // total to the final output location, using a software transactional memory
+ // style since D3D11 doesn't support atomic add on floats.
+ // (Assumes the output value has been cleared to zero beforehand.)
+
+ if (iThreadInGroup == 0)
+ {
+ float threadGroupSum = g_partialSums[0];
+ uint outputValueRead = o_data[xyOutput];
+ while (true)
+ {
+ uint newOutputValue = asuint(asfloat(outputValueRead) + threadGroupSum);
+ uint previousOutputValue;
+ InterlockedCompareExchange(
+ o_data[xyOutput], outputValueRead, newOutputValue, previousOutputValue);
+ if (previousOutputValue == outputValueRead)
+ break;
+ outputValueRead = previousOutputValue;
+ }
+ }
+}
+
+void main(
+ in Vertex i_vtx,
+ out Vertex o_vtx,
+ out float3 o_vecCamera : CAMERA,
+ out float4 o_uvzwShadow : UVZW_SHADOW,
+ out float4 o_posClip : SV_Position)
+{
+ o_vtx = i_vtx;
+ o_vecCamera = g_posCamera - i_vtx.m_pos;
+ o_uvzwShadow = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToUvzwShadow);
+ o_posClip = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToClip);
+}
+
+#pragma pack_matrix(row_major)
+
+struct Vertex
+{
+ float3 m_pos : POSITION;
+ float3 m_normal : NORMAL;
+ float2 m_uv : UV;
+};
+
+cbuffer CBFrame : CB_FRAME // matches struct CBFrame in test.cpp
+{
+ float4x4 g_matWorldToClip;
+ float4x4 g_matWorldToUvzwShadow;
+ float3x3 g_matWorldToUvzShadowNormal;
+ float3 g_posCamera;
+
+ float3 g_vecDirectionalLight;
+ float3 g_rgbDirectionalLight;
+
+ float2 g_dimsShadowMap;
+ float g_normalOffsetShadow;
+ float g_shadowSharpening;
+
+ float g_exposure; // Exposure multiplier
+}
+
+Texture2D<float3> g_texDiffuse : register(t0);
+SamplerState g_ss : register(s0);
+
+void main(
+ in Vertex i_vtx,
+ in float3 i_vecCamera : CAMERA,
+ in float4 i_uvzwShadow : UVZW_SHADOW,
+ out float3 o_rgb : SV_Target)
+{
+ float3 normal = normalize(i_vtx.m_normal);
+
+ // Sample shadow map
+ float shadow = EvaluateShadow(i_uvzwShadow, normal);
+
+ // Evaluate diffuse lighting
+ float3 diffuseColor = g_texDiffuse.Sample(g_ss, i_vtx.m_uv);
+ float3 diffuseLight = g_rgbDirectionalLight * (shadow * saturate(dot(normal, g_vecDirectionalLight)));
+ diffuseLight += SimpleAmbient(normal);
+
+ o_rgb = diffuseColor * diffuseLight;
+}
+
+[domain("quad")]
+void ds(
+ in float edgeFactors[4] : SV_TessFactor,
+ in float insideFactors[2] : SV_InsideTessFactor,
+ in OutputPatch<VData, 4> inp,
+ in float2 uv : SV_DomainLocation,
+ out float4 o_pos : SV_Position)
+{
+ o_pos = lerp(lerp(inp[0].pos, inp[1].pos, uv.x), lerp(inp[2].pos, inp[3].pos, uv.x), uv.y);
+}
diff --git a/tests/examplefiles/fennelview.fnl b/tests/examplefiles/fennelview.fnl new file mode 100644 index 00000000..fd0fc648 --- /dev/null +++ b/tests/examplefiles/fennelview.fnl @@ -0,0 +1,156 @@ +;; A pretty-printer that outputs tables in Fennel syntax. +;; Loosely based on inspect.lua: http://github.com/kikito/inspect.lua + +(local quote (fn [str] (.. '"' (: str :gsub '"' '\\"') '"'))) + +(local short-control-char-escapes + {"\a" "\\a" "\b" "\\b" "\f" "\\f" "\n" "\\n" + "\r" "\\r" "\t" "\\t" "\v" "\\v"}) + +(local long-control-char-esapes + (let [long {}] + (for [i 0 31] + (let [ch (string.char i)] + (when (not (. short-control-char-escapes ch)) + (tset short-control-char-escapes ch (.. "\\" i)) + (tset long ch (: "\\%03d" :format i))))) + long)) + +(fn escape [str] + (let [str (: str :gsub "\\" "\\\\") + str (: str :gsub "(%c)%f[0-9]" long-control-char-esapes)] + (: str :gsub "%c" short-control-char-escapes))) + +(fn sequence-key? [k len] + (and (= (type k) "number") + (<= 1 k) + (<= k len) + (= (math.floor k) k))) + +(local type-order {:number 1 :boolean 2 :string 3 :table 4 + :function 5 :userdata 6 :thread 7}) + +(fn sort-keys [a b] + (let [ta (type a) tb (type b)] + (if (and (= ta tb) (~= ta "boolean") + (or (= ta "string") (= ta "number"))) + (< a b) + (let [dta (. type-order a) + dtb (. type-order b)] + (if (and dta dtb) + (< dta dtb) + dta true + dtb false + :else (< ta tb)))))) + +(fn get-sequence-length [t] + (var len 1) + (each [i (ipairs t)] (set len i)) + len) + +(fn get-nonsequential-keys [t] + (let [keys {} + sequence-length (get-sequence-length t)] + (each [k (pairs t)] + (when (not (sequence-key? k sequence-length)) + (table.insert keys k))) + (table.sort keys sort-keys) + (values keys sequence-length))) + +(fn count-table-appearances [t appearances] + (if (= (type t) "table") + (when (not (. appearances t)) + (tset appearances t 1) + (each [k v (pairs t)] + (count-table-appearances k appearances) + (count-table-appearances v appearances))) + (when (and t (= t t)) ; no nans please + (tset appearances t (+ (or (. appearances t) 0) 1)))) + appearances) + + + +(var put-value nil) ; mutual recursion going on; defined below + +(fn puts [self ...] + (each [_ v (ipairs [...])] + (table.insert self.buffer v))) + +(fn tabify [self] (puts self "\n" (: self.indent :rep self.level))) + +(fn already-visited? [self v] (~= (. self.ids v) nil)) + +(fn get-id [self v] + (var id (. self.ids v)) + (when (not id) + (let [tv (type v)] + (set id (+ (or (. self.max-ids tv) 0) 1)) + (tset self.max-ids tv id) + (tset self.ids v id))) + (tostring id)) + +(fn put-sequential-table [self t length] + (puts self "[") + (set self.level (+ self.level 1)) + (for [i 1 length] + (puts self " ") + (put-value self (. t i))) + (set self.level (- self.level 1)) + (puts self " ]")) + +(fn put-key [self k] + (if (and (= (type k) "string") + (: k :find "^[-%w?\\^_`!#$%&*+./@~:|<=>]+$")) + (puts self ":" k) + (put-value self k))) + +(fn put-kv-table [self t] + (puts self "{") + (set self.level (+ self.level 1)) + (each [k v (pairs t)] + (tabify self) + (put-key self k) + (puts self " ") + (put-value self v)) + (set self.level (- self.level 1)) + (tabify self) + (puts self "}")) + +(fn put-table [self t] + (if (already-visited? self t) + (puts self "#<table " (get-id self t) ">") + (>= self.level self.depth) + (puts self "{...}") + :else + (let [(non-seq-keys length) (get-nonsequential-keys t) + id (get-id self t)] + (if (> (. self.appearances t) 1) + (puts self "#<" id ">") + (and (= (# non-seq-keys) 0) (= (# t) 0)) + (puts self "{}") + (= (# non-seq-keys) 0) + (put-sequential-table self t length) + :else + (put-kv-table self t))))) + +(set put-value (fn [self v] + (let [tv (type v)] + (if (= tv "string") + (puts self (quote (escape v))) + (or (= tv "number") (= tv "boolean") (= tv "nil")) + (puts self (tostring v)) + (= tv "table") + (put-table self v) + :else + (puts self "#<" (tostring v) ">"))))) + + + +(fn fennelview [root options] + (let [options (or options {}) + inspector {:appearances (count-table-appearances root {}) + :depth (or options.depth 128) + :level 0 :buffer {} :ids {} :max-ids {} + :indent (or options.indent " ")}] + (put-value inspector root) + (table.concat inspector.buffer))) diff --git a/tests/examplefiles/test.csd b/tests/examplefiles/test.csd index 9122309b..6512d99e 100644 --- a/tests/examplefiles/test.csd +++ b/tests/examplefiles/test.csd @@ -1,260 +1,18 @@ +/* + * comment + */ +; comment +// comment +/ <CsoundSynthesizer> <CsInstruments> -// This is a Csound orchestra file for testing a Pygments <http://pygments.org> -// lexer. Csound single-line comments can be preceded by a pair of forward -// slashes... -; ...or a semicolon. - -/* Block comments begin with /* and end with */ - -// Orchestras begin with a header of audio parameters. -nchnls = 1 -nchnls_i = 1 -sr = 44100 0dbfs = 1 -ksmps = 10 - -// The control rate kr = sr / ksmps can be omitted when the number of audio -// samples in a control period (ksmps) is set, but kr may appear in older -// orchestras. -kr = 4410 - -// Orchestras contain instruments. These begin with the keyword instr followed -// by a comma-separated list of numbers or names of the instrument. Instruments -// end at the endin keyword and cannot be nested. -instr 1, N_a_M_e_, +Name - // Instruments contain statements. Here is a typical statement: - aSignal oscil 0dbfs, 440, 1 - // Statements are terminated with a newline (possibly preceded by a comment). - // To write a statement on several lines, precede the newline with a - // backslash. - prints \ - "hello, world\n";comment - - // Csound 6 introduced function syntax for opcodes with one or zero outputs. - // The oscil statement above is the same as - aSignal = oscil(0dbfs, 440, 1) - - // Instruments can contain control structures. - kNote = p3 - if (kNote == 0) then - kFrequency = 220 - elseif kNote == 1 then // Parentheses around binary expressions are optional. - kFrequency = 440 - endif - - // Csound 6 introduced looping structures. - iIndex = 0 - while iIndex < 5 do - print iIndex - iIndex += 1 - od - iIndex = 0 - until iIndex >= 5 do - print iIndex - iIndex += 1 - enduntil - // Both kinds of loops can be terminated by either od or enduntil. - - // Single-line strings are enclosed in double-quotes. - prints "string\\\r\n\t\"" - // Multi-line strings are enclosed in pairs of curly braces. - prints {{ - hello, - - world - }} - - // Instruments often end with a statement containing an output opcode. - outc aSignal -endin - -// Orchestras can also contain user-defined opcodes (UDOs). Here is an -// oscillator with one audio-rate output and two control-rate inputs: -opcode anOscillator, a, kk - kAmplitude, kFrequency xin - aSignal vco2 kAmplitude, kFrequency - xout aSignal -endop -instr TestOscillator - outc(anOscillator(0dbfs, 110)) -endin - -// Python can be executed in Csound -// <http://www.csounds.com/manual/html/pyrun.html>. So can Lua -// <http://www.csounds.com/manual/html/lua.html>. -pyruni {{ -import random - -pool = [(1 + i / 10.0) ** 1.2 for i in range(100)] - -def get_number_from_pool(n, p): - if random.random() < p: - i = int(random.random() * len(pool)) - pool[i] = n - return random.choice(pool) -}} - -// The Csound preprocessor supports conditional compilation and including files. -#ifdef DEBUG -#undef DEBUG -#include "filename.orc" -#endif - -// The preprocessor also supports object- and function-like macros. This is an -// object-like macro that defines a number: -#define A_HZ #440# - -// This is a function-like macro: -#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# - -// Bodies of macros are enclosed in # and can contain newlines. The arguments of -// function-like macros are separated by single-quotes. Uses of macros are -// prefixed with a dollar sign. -instr TestMacro - aSignal $OSCIL_MACRO(1'$A_HZ'1) - // Not unlike PHP, macros expand in double-quoted strings. - prints "The frequency of the oscillator is $A_HZ Hz.\n" - out aSignal -endin - -// Here are other things to note about Csound. - -// There are two bitwise NOT operators, ~ and ¬ (U+00AC). The latter is common -// on keyboards in the United Kingdom -// <https://en.wikipedia.org/wiki/British_and_American_keyboards>. -instr TestBitwiseNOT - print ~42 - print ¬42 -endin - -// Csound uses # for bitwise XOR, which the Csound manual calls bitwise -// non-equivalence <http://www.csounds.com/manual/html/opnonequiv.html>. -instr TestBitwiseXOR - print 0 # 0 - print 0 # 1 - print 1 # 0 - print 1 # 1 -endin - -// Loops and if-then statements are relatively recent additions to Csound. There -// are many flow-control opcodes that involve goto and labels. -instr TestGoto - // This... - if p3 > 0 goto if_label - goto else_label -if_label: - prints "if branch\n" - goto endif_label -else_label: - prints "else branch\n" -endif_label: - - // ...is the same as this. - if p3 > 0 then - prints "if branch\n" - else - prints "else branch\n" - endif - - // This... - iIndex = 0 -loop_label: - print iIndex - iIndex += 1 - if iIndex < 10 goto loop_label - - // ...is the same as this... - iIndex = 0 -loop_lt_label: - print iIndex - loop_lt iIndex, 1, 10, loop_lt_label - - // ...and this. - iIndex = 0 - while iIndex < 10 do - print iIndex - iIndex += 1 - od -endin - -// The prints and printks opcodes -// <https://github.com/csound/csound/blob/develop/OOps/ugrw1.c#L831>, arguably -// the primary methods of logging output, treat certain sequences of characters -// different from printf in C. -instr TestPrints - // ^ prints an ESCAPE character (U+001B), not a CIRCUMFLEX ACCENT character - // (U+005E). ^^ prints a CIRCUMFLEX ACCENT. - prints "^^\n" - // ~ prints an ESCAPE character (U+001B) followed by a [, not a TILDE - // character (U+007E). ~~ prints a TILDE. - prints "~~\n" - // \A, \B, \N, \R, and \T correspond to the escaped lowercase characters (that - // is, BELL (U+0007), BACKSPACE (U+0008), new line (U+000A), CARRIAGE RETURN - // (U+000D), and tab (U+0009)). - prints "\T\R\N" - // %n, %r, and %t are the same as \n, \r, and \t, as are %N, %R, and %T. - prints "%t%r%n" - // %! prints a semicolon. This is a hold-over from old versions of Csound that - // allowed comments to begin in strings. - prints "; %!\n" -endin - -// The arguments of function-like macros can be separated by # instead of '. -// These two lines define the same macro. -#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# -#define OSCIL_MACRO(VOLUME#FREQUENCY#TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# - -// Uses of macros can optionally be suffixed with a period. -instr TestMacroPeriodSuffix - aSignal $OSCIL_MACRO.(1'$A_HZ'1) - prints "The frequency of the oscillator is $A_HZ.Hz.\n" - out aSignal -endin - -// Csound has @ and @@ operator-like macros that, when followed by a literal -// non-negative integer, expand to the next power of 2 and the next power of 2 -// plus 1: -// @x = 2^(ceil(log2(x + 1))), x >= 0 -// @@0 = 2 -// @@x = 2^(ceil(log2(x))) + 1, x > 0 -// These macros are in -// <https://github.com/csound/csound/blob/develop/Engine/csound_orc.l#L542> (and -// <https://github.com/csound/csound/blob/develop/Engine/csound_sco.lex#L202>) -// and are described at <http://www.csounds.com/manual/html/ScoreEval.html>. -instr TestAt - prints "%d %2d %2d\n", 0, @0, @@0 - prints "%d %2d %2d\n", 1, @1, @@1 - prints "%d %2d %2d\n", 2, @2, @@2 - prints "%d %2d %2d\n", 3, @3, @@3 - prints "%d %2d %2d\n", 4, @4, @@4 - prints "%d %2d %2d\n", 5, @5, @@5 - prints "%d %2d %2d\n", 6, @6, @@6 - prints "%d %2d %2d\n", 7, @7, @@7 - prints "%d %2d %2d\n", 8, @8, @@8 - prints "%d %2d %2d\n", 9, @9, @@9 -endin - -// Including newlines in macros can lead to confusing code, but it tests the -// lexer. -instr MacroAbuse - if 1 == 1 then - prints "on\n" -#define FOO# -BAR -#endif // This ends the if statement. It is not a preprocessor directive. -endin +prints "hello, world\n" </CsInstruments> <CsScore> -f 1 0 16384 10 1 -i "N_a_M_e_" 0 2 -i "TestOscillator" 2 2 -i "TestBitwiseNOT" 0 1 -i "TestBitwiseXOR" 0 1 -i "TestGoto" 0 1 -i "TestMacroPeriodSuffix" 4 1 -i "TestAt" 0 1 -i "MacroAbuse" 0 1 -e +i 1 0 0 </CsScore> +<html> +<!DOCTYPE html> +</html> </CsoundSynthesizer> diff --git a/tests/examplefiles/test.orc b/tests/examplefiles/test.orc index 36725342..d113303e 100644 --- a/tests/examplefiles/test.orc +++ b/tests/examplefiles/test.orc @@ -1,257 +1,81 @@ -// This is a Csound orchestra file for testing a Pygments <http://pygments.org> -// lexer. Csound single-line comments can be preceded by a pair of forward -// slashes... -; ...or a semicolon. - -/* Block comments begin with /* and end with */ - -// Orchestras begin with a header of audio parameters. -nchnls = 1 -nchnls_i = 1 -sr = 44100 -0dbfs = 1 -ksmps = 10 - -// The control rate kr = sr / ksmps can be omitted when the number of audio -// samples in a control period (ksmps) is set, but kr may appear in older -// orchestras. -kr = 4410 - -// Orchestras contain instruments. These begin with the keyword instr followed -// by a comma-separated list of numbers or names of the instrument. Instruments -// end at the endin keyword and cannot be nested. -instr 1, N_a_M_e_, +Name - // Instruments contain statements. Here is a typical statement: - aSignal oscil 0dbfs, 440, 1 - // Statements are terminated with a newline (possibly preceded by a comment). - // To write a statement on several lines, precede the newline with a - // backslash. - prints \ - "hello, world\n";comment - - // Csound 6 introduced function syntax for opcodes with one or zero outputs. - // The oscil statement above is the same as - aSignal = oscil(0dbfs, 440, 1) - - // Instruments can contain control structures. - kNote = p3 - if (kNote == 0) then - kFrequency = 220 - elseif kNote == 1 then // Parentheses around binary expressions are optional. - kFrequency = 440 - endif - - // Csound 6 introduced looping structures. - iIndex = 0 - while iIndex < 5 do - print iIndex - iIndex += 1 - od - iIndex = 0 - until iIndex >= 5 do - print iIndex - iIndex += 1 - enduntil - // Both kinds of loops can be terminated by either od or enduntil. - - // Single-line strings are enclosed in double-quotes. - prints "string\\\r\n\t\"" - // Multi-line strings are enclosed in pairs of curly braces. - prints {{ - hello, - - world - }} - - // Instruments often end with a statement containing an output opcode. - outc aSignal +/* + * comment + */ +; comment +// comment + +instr/**/1,/**/N_a_M_e_,/**/+Name/**/// + iDuration = p3 + outc:a(aSignal) endin -// Orchestras can also contain user-defined opcodes (UDOs). Here is an -// oscillator with one audio-rate output and two control-rate inputs: -opcode anOscillator, a, kk - kAmplitude, kFrequency xin - aSignal vco2 kAmplitude, kFrequency - xout aSignal +opcode/**/aUDO,/**/i[],/**/aik// + aUDO endop -instr TestOscillator - outc(anOscillator(0dbfs, 110)) -endin -// Python can be executed in Csound -// <http://www.csounds.com/manual/html/pyrun.html>. So can Lua -// <http://www.csounds.com/manual/html/lua.html>. -pyruni {{ -import random +123 0123456789 +0xabcdef0123456789 0XABCDEF +1e2 3e+4 5e-6 7E8 9E+0 1E-2 3. 4.56 .789 -pool = [(1 + i / 10.0) ** 1.2 for i in range(100)] +"characters$MACRO." +"\\\a\b\n\r\t\012\345\67\"" -def get_number_from_pool(n, p): - if random.random() < p: - i = int(random.random() * len(pool)) - pool[i] = n - return random.choice(pool) +{{ +characters$MACRO. }} +{{\\\a\b\n\r\t\"\012\345\67}} -// The Csound preprocessor supports conditional compilation and including files. -#ifdef DEBUG -#undef DEBUG -#include "filename.orc" -#endif - -// The preprocessor also supports object- and function-like macros. This is an -// object-like macro that defines a number: -#define A_HZ #440# - -// This is a function-like macro: -#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# - -// Bodies of macros are enclosed in # and can contain newlines. The arguments of -// function-like macros are separated by single-quotes. Uses of macros are -// prefixed with a dollar sign. -instr TestMacro - aSignal $OSCIL_MACRO(1'$A_HZ'1) - // Not unlike PHP, macros expand in double-quoted strings. - prints "The frequency of the oscillator is $A_HZ Hz.\n" - out aSignal -endin - -// Here are other things to note about Csound. ++ - ~ ¬ ! * / ^ % << >> < > <= >= == != & # | && || ? : += -= *= /= -// There are two bitwise NOT operators, ~ and ¬ (U+00AC). The latter is common -// on keyboards in the United Kingdom -// <https://en.wikipedia.org/wiki/British_and_American_keyboards>. -instr TestBitwiseNOT - print ~42 - print ¬42 -endin - -// Csound uses # for bitwise XOR, which the Csound manual calls bitwise -// non-equivalence <http://www.csounds.com/manual/html/opnonequiv.html>. -instr TestBitwiseXOR - print 0 # 0 - print 0 # 1 - print 1 # 0 - print 1 # 1 -endin +0dbfs A4 kr ksmps nchnls nchnls_i sr -// Loops and if-then statements are relatively recent additions to Csound. There -// are many flow-control opcodes that involve goto and labels. -instr TestGoto - // This... - if p3 > 0 goto if_label - goto else_label -if_label: - prints "if branch\n" - goto endif_label -else_label: - prints "else branch\n" -endif_label: +do else elseif endif enduntil fi if ithen kthen od then until while +return rireturn - // ...is the same as this. - if p3 > 0 then - prints "if branch\n" - else - prints "else branch\n" - endif +aLabel: + label2: - // This... - iIndex = 0 -loop_label: - print iIndex - iIndex += 1 - if iIndex < 10 goto loop_label +goto aLabel +reinit aLabel +cggoto 1==0, aLabel +timout 0, 0, aLabel +loop_ge 0, 0, 0, aLabel - // ...is the same as this... - iIndex = 0 -loop_lt_label: - print iIndex - loop_lt iIndex, 1, 10, loop_lt_label - - // ...and this. - iIndex = 0 - while iIndex < 10 do - print iIndex - iIndex += 1 - od -endin +prints "%! %% %n%N %r%R %t%T \\a\\A \\b\\B \\n\\N \\r\\R \\t\\T" +prints Soutput -// The prints and printks opcodes -// <https://github.com/csound/csound/blob/develop/OOps/ugrw1.c#L831>, arguably -// the primary methods of logging output, treat certain sequences of characters -// different from printf in C. -instr TestPrints - // ^ prints an ESCAPE character (U+001B), not a CIRCUMFLEX ACCENT character - // (U+005E). ^^ prints a CIRCUMFLEX ACCENT. - prints "^^\n" - // ~ prints an ESCAPE character (U+001B) followed by a [, not a TILDE - // character (U+007E). ~~ prints a TILDE. - prints "~~\n" - // \A, \B, \N, \R, and \T correspond to the escaped lowercase characters (that - // is, BELL (U+0007), BACKSPACE (U+0008), new line (U+000A), CARRIAGE RETURN - // (U+000D), and tab (U+0009)). - prints "\T\R\N" - // %n, %r, and %t are the same as \n, \r, and \t, as are %N, %R, and %T. - prints "%t%r%n" - // %! prints a semicolon. This is a hold-over from old versions of Csound that - // allowed comments to begin in strings. - prints "; %!\n" -endin - -// The arguments of function-like macros can be separated by # instead of '. -// These two lines define the same macro. -#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# -#define OSCIL_MACRO(VOLUME#FREQUENCY#TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# - -// Uses of macros can optionally be suffixed with a period. -instr TestMacroPeriodSuffix - aSignal $OSCIL_MACRO.(1'$A_HZ'1) - prints "The frequency of the oscillator is $A_HZ.Hz.\n" - out aSignal -endin - -// Csound has @ and @@ operator-like macros that, when followed by a literal -// non-negative integer, expand to the next power of 2 and the next power of 2 -// plus 1: -// @x = 2^(ceil(log2(x + 1))), x >= 0 -// @@0 = 2 -// @@x = 2^(ceil(log2(x))) + 1, x > 0 -// These macros are in -// <https://github.com/csound/csound/blob/develop/Engine/csound_orc.l#L542> (and -// <https://github.com/csound/csound/blob/develop/Engine/csound_sco.lex#L202>) -// and are described at <http://www.csounds.com/manual/html/ScoreEval.html>. -instr TestAt - prints "%d %2d %2d\n", 0, @0, @@0 - prints "%d %2d %2d\n", 1, @1, @@1 - prints "%d %2d %2d\n", 2, @2, @@2 - prints "%d %2d %2d\n", 3, @3, @@3 - prints "%d %2d %2d\n", 4, @4, @@4 - prints "%d %2d %2d\n", 5, @5, @@5 - prints "%d %2d %2d\n", 6, @6, @@6 - prints "%d %2d %2d\n", 7, @7, @@7 - prints "%d %2d %2d\n", 8, @8, @@8 - prints "%d %2d %2d\n", 9, @9, @@9 -endin +readscore {{ +i 1 0 0 +}} +pyrun {{ +# Python +}} +lua_exec {{ +-- Lua +}} -// Including newlines in macros can lead to confusing code, but it tests the -// lexer. -instr MacroAbuse - if 1 == 1 then - prints "on\n" -#define FOO# -BAR -#endif // This ends the if statement. It is not a preprocessor directive. -endin +#include/**/"file.udo" +#include/**/|file.udo| -scoreline_i {{ -f 1 0 16384 10 1 -i "N_a_M_e_" 0 2 -i "TestOscillator" 2 2 -i "TestBitwiseNOT" 0 1 -i "TestBitwiseXOR" 0 1 -i "TestGoto" 0 1 -i "TestMacroPeriodSuffix" 4 1 -i "TestAt" 0 1 -i "MacroAbuse" 0 1 -e -}} +#ifdef MACRO +#else +#ifndef MACRO +#endif +#undef MACRO + +# define MACRO#macro_body# +#define/**/ +MACRO/**/ +#\#macro +body\## + +#define MACRO(ARG1#ARG2) #macro_body# +#define/**/ +MACRO(ARG1'ARG2'ARG3)/**/ +#\#macro +body\## + +$MACRO $MACRO. +$MACRO(x) +@0 +@@ 1 diff --git a/tests/examplefiles/test.sco b/tests/examplefiles/test.sco index a0b39251..d997c1b3 100644 --- a/tests/examplefiles/test.sco +++ b/tests/examplefiles/test.sco @@ -1,10 +1,22 @@ -f 1 0 16384 10 1 -i "N_a_M_e_" 0 2 -i "TestOscillator" 2 2 -i "TestBitwiseNOT" 0 1 -i "TestBitwiseXOR" 0 1 -i "TestGoto" 0 1 -i "TestMacroPeriodSuffix" 4 1 -i "TestAt" 0 1 -i "MacroAbuse" 0 1 -e +/* + * comment + */ +; comment +// comment +a b C d e f i q s t v x y +z +np0 nP1 Np2 NP3 +m/**/label; +n label +123 0123456789 +0xabcdef0123456789 0XABCDEF +1e2 3e+4 5e-6 7E8 9E+0 1E-2 3. 4.56 .789 +"characters$MACRO." +{ 1 I + { 2 J + { 3 K + $I $J $K + } + } +} +#include "score.sco" diff --git a/tests/run.py b/tests/run.py index 07665b2a..2e962f2f 100644 --- a/tests/run.py +++ b/tests/run.py @@ -16,11 +16,19 @@ from __future__ import print_function import os import sys +import warnings # only find tests in this directory if os.path.dirname(__file__): os.chdir(os.path.dirname(__file__)) +# make FutureWarnings (coming from Regex syntax most likely) and +# DeprecationWarnings due to non-raw strings an error +warnings.filterwarnings("error", module=r"pygments\..*", + category=FutureWarning) +warnings.filterwarnings("error", module=r".*pygments.*", + category=DeprecationWarning) + try: import nose diff --git a/tests/test_csound.py b/tests/test_csound.py new file mode 100644 index 00000000..4d10c267 --- /dev/null +++ b/tests/test_csound.py @@ -0,0 +1,480 @@ +# -*- coding: utf-8 -*- +""" + Csound lexer tests + ~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest +from textwrap import dedent + +from pygments.token import Comment, Error, Keyword, Name, Number, Operator, Punctuation, \ + String, Text +from pygments.lexers import CsoundOrchestraLexer + + +class CsoundOrchestraTest(unittest.TestCase): + + def setUp(self): + self.lexer = CsoundOrchestraLexer() + self.maxDiff = None + + def testComments(self): + fragment = dedent('''\ + /* + * comment + */ + ; comment + // comment + ''') + tokens = [ + (Comment.Multiline, u'/*\n * comment\n */'), + (Text, u'\n'), + (Comment.Single, u'; comment'), + (Text, u'\n'), + (Comment.Single, u'// comment'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testInstrumentBlocks(self): + fragment = dedent('''\ + instr/**/1,/**/N_a_M_e_,/**/+Name/**/// + iDuration = p3 + outc:a(aSignal) + endin + ''') + tokens = [ + (Keyword.Declaration, u'instr'), + (Comment.Multiline, u'/**/'), + (Name.Function, u'1'), + (Punctuation, u','), + (Comment.Multiline, u'/**/'), + (Name.Function, u'N_a_M_e_'), + (Punctuation, u','), + (Comment.Multiline, u'/**/'), + (Punctuation, u'+'), + (Name.Function, u'Name'), + (Comment.Multiline, u'/**/'), + (Comment.Single, u'//'), + (Text, u'\n'), + (Text, u' '), + (Keyword.Type, u'i'), + (Name, u'Duration'), + (Text, u' '), + (Operator, u'='), + (Text, u' '), + (Name.Variable.Instance, u'p3'), + (Text, u'\n'), + (Text, u' '), + (Name.Builtin, u'outc'), + (Punctuation, u':'), + (Keyword.Type, u'a'), + (Punctuation, u'('), + (Keyword.Type, u'a'), + (Name, u'Signal'), + (Punctuation, u')'), + (Text, u'\n'), + (Keyword.Declaration, u'endin'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testUserDefinedOpcodes(self): + fragment = dedent('''\ + opcode/**/aUDO,/**/i[],/**/aik// + aUDO + endop + ''') + tokens = [ + (Keyword.Declaration, u'opcode'), + (Comment.Multiline, u'/**/'), + (Name.Function, u'aUDO'), + (Punctuation, u','), + (Comment.Multiline, u'/**/'), + (Keyword.Type, u'i[]'), + (Punctuation, u','), + (Comment.Multiline, u'/**/'), + (Keyword.Type, u'aik'), + (Comment.Single, u'//'), + (Text, u'\n'), + (Text, u' '), + (Name.Function, u'aUDO'), + (Text, u'\n'), + (Keyword.Declaration, u'endop'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testNumbers(self): + fragment = '123 0123456789' + tokens = [ + (Number.Integer, u'123'), + (Text, u' '), + (Number.Integer, u'0123456789'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + fragment = '0xabcdef0123456789 0XABCDEF' + tokens = [ + (Keyword.Type, u'0x'), + (Number.Hex, u'abcdef0123456789'), + (Text, u' '), + (Keyword.Type, u'0X'), + (Number.Hex, u'ABCDEF'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + fragments = ['1e2', '3e+4', '5e-6', '7E8', '9E+0', '1E-2', '3.', '4.56', '.789'] + for fragment in fragments: + tokens = [ + (Number.Float, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testQuotedStrings(self): + fragment = '"characters$MACRO."' + tokens = [ + (String, u'"'), + (String, u'characters'), + (Comment.Preproc, u'$MACRO.'), + (String, u'"'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testBracedStrings(self): + fragment = dedent('''\ + {{ + characters$MACRO. + }} + ''') + tokens = [ + (String, u'{{'), + (String, u'\ncharacters$MACRO.\n'), + (String, u'}}'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testEscapeSequences(self): + for character in ['\\', 'a', 'b', 'n', 'r', 't', '"', '012', '345', '67']: + escapedCharacter = '\\' + character + fragment = '"' + escapedCharacter + '"' + tokens = [ + (String, u'"'), + (String.Escape, escapedCharacter), + (String, u'"'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + fragment = '{{' + escapedCharacter + '}}' + tokens = [ + (String, u'{{'), + (String.Escape, escapedCharacter), + (String, u'}}'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testOperators(self): + fragments = ['+', '-', '~', u'¬', '!', '*', '/', '^', '%', '<<', '>>', '<', '>', + '<=', '>=', '==', '!=', '&', '#', '|', '&&', '||', '?', ':', '+=', + '-=', '*=', '/='] + for fragment in fragments: + tokens = [ + (Operator, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testGlobalValueIdentifiers(self): + for fragment in ['0dbfs', 'A4', 'kr', 'ksmps', 'nchnls', 'nchnls_i', 'sr']: + tokens = [ + (Name.Variable.Global, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testKeywords(self): + fragments = ['do', 'else', 'elseif', 'endif', 'enduntil', 'fi', 'if', 'ithen', + 'kthen', 'od', 'then', 'until', 'while'] + for fragment in fragments: + tokens = [ + (Keyword, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + for fragment in ['return', 'rireturn']: + tokens = [ + (Keyword.Pseudo, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testLabels(self): + fragment = dedent('''\ + aLabel: + label2: + ''') + tokens = [ + (Name.Label, u'aLabel'), + (Punctuation, u':'), + (Text, u'\n'), + (Text, u' '), + (Name.Label, u'label2'), + (Punctuation, u':'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testPrintksAndPrintsEscapeSequences(self): + escapedCharacters = ['%!', '%%', '%n', '%N', '%r', '%R', '%t', '%T', '\\\\a', + '\\\\A', '\\\\b', '\\\\B', '\\\\n', '\\\\N', '\\\\r', + '\\\\R', '\\\\t', '\\\\T'] + for opcode in ['printks', 'prints']: + for escapedCharacter in escapedCharacters: + fragment = opcode + ' "' + escapedCharacter + '"' + tokens = [ + (Name.Builtin, opcode), + (Text, u' '), + (String, u'"'), + (String.Escape, escapedCharacter), + (String, u'"'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testGotoStatements(self): + for keyword in ['goto', 'igoto', 'kgoto']: + fragment = keyword + ' aLabel' + tokens = [ + (Keyword, keyword), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + for opcode in ['reinit', 'rigoto', 'tigoto']: + fragment = opcode + ' aLabel' + tokens = [ + (Keyword.Pseudo, opcode), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + for opcode in ['cggoto', 'cigoto', 'cingoto', 'ckgoto', 'cngoto', 'cnkgoto']: + fragment = opcode + ' 1==0, aLabel' + tokens = [ + (Keyword.Pseudo, opcode), + (Text, u' '), + (Number.Integer, u'1'), + (Operator, u'=='), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + fragment = 'timout 0, 0, aLabel' + tokens = [ + (Keyword.Pseudo, 'timout'), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + for opcode in ['loop_ge', 'loop_gt', 'loop_le', 'loop_lt']: + fragment = opcode + ' 0, 0, 0, aLabel' + tokens = [ + (Keyword.Pseudo, opcode), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testIncludeDirectives(self): + for character in ['"', '|']: + fragment = '#include/**/' + character + 'file.udo' + character + tokens = [ + (Comment.Preproc, u'#include'), + (Comment.Multiline, u'/**/'), + (String, character + u'file.udo' + character), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testObjectLikeMacroDefinitions(self): + fragment = dedent('''\ + # \tdefine MACRO#macro_body# + #define/**/ + MACRO/**/ + #\\#macro + body\\## + ''') + tokens = [ + (Comment.Preproc, u'# \tdefine'), + (Text, u' '), + (Comment.Preproc, u'MACRO'), + (Punctuation, u'#'), + (Comment.Preproc, u'macro_body'), + (Punctuation, u'#'), + (Text, u'\n'), + (Comment.Preproc, u'#define'), + (Comment.Multiline, u'/**/'), + (Text, u'\n'), + (Comment.Preproc, u'MACRO'), + (Comment.Multiline, u'/**/'), + (Text, u'\n'), + (Punctuation, u'#'), + (Comment.Preproc, u'\\#'), + (Comment.Preproc, u'macro\nbody'), + (Comment.Preproc, u'\\#'), + (Punctuation, u'#'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testFunctionLikeMacroDefinitions(self): + fragment = dedent('''\ + #define MACRO(ARG1#ARG2) #macro_body# + #define/**/ + MACRO(ARG1'ARG2' ARG3)/**/ + #\\#macro + body\\## + ''') + tokens = [ + (Comment.Preproc, u'#define'), + (Text, u' '), + (Comment.Preproc, u'MACRO'), + (Punctuation, u'('), + (Comment.Preproc, u'ARG1'), + (Punctuation, u'#'), + (Comment.Preproc, u'ARG2'), + (Punctuation, u')'), + (Text, u' '), + (Punctuation, u'#'), + (Comment.Preproc, u'macro_body'), + (Punctuation, u'#'), + (Text, u'\n'), + (Comment.Preproc, u'#define'), + (Comment.Multiline, u'/**/'), + (Text, u'\n'), + (Comment.Preproc, u'MACRO'), + (Punctuation, u'('), + (Comment.Preproc, u'ARG1'), + (Punctuation, u"'"), + (Comment.Preproc, u'ARG2'), + (Punctuation, u"'"), + (Text, u' '), + (Comment.Preproc, u'ARG3'), + (Punctuation, u')'), + (Comment.Multiline, u'/**/'), + (Text, u'\n'), + (Punctuation, u'#'), + (Comment.Preproc, u'\\#'), + (Comment.Preproc, u'macro\nbody'), + (Comment.Preproc, u'\\#'), + (Punctuation, u'#'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testMacroPreprocessorDirectives(self): + for directive in ['#ifdef', '#ifndef', '#undef']: + fragment = directive + ' MACRO' + tokens = [ + (Comment.Preproc, directive), + (Text, u' '), + (Comment.Preproc, u'MACRO'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testOtherPreprocessorDirectives(self): + fragment = dedent('''\ + #else + #end + #endif + ### + @ \t12345 + @@ \t67890 + ''') + tokens = [ + (Comment.Preproc, u'#else'), + (Text, u'\n'), + (Comment.Preproc, u'#end'), + (Text, u'\n'), + (Comment.Preproc, u'#endif'), + (Text, u'\n'), + (Comment.Preproc, u'###'), + (Text, u'\n'), + (Comment.Preproc, u'@ \t12345'), + (Text, u'\n'), + (Comment.Preproc, u'@@ \t67890'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testFunctionLikeMacros(self): + fragment = "$MACRO.(((x#y\\)))' \"(#'x)\\)x\\))\"# {{x\\))x)\\)(#'}});" + tokens = [ + (Comment.Preproc, u'$MACRO.'), + (Punctuation, u'('), + (Comment.Preproc, u'('), + (Comment.Preproc, u'('), + (Comment.Preproc, u'x#y\\)'), + (Comment.Preproc, u')'), + (Comment.Preproc, u')'), + (Punctuation, u"'"), + (Comment.Preproc, u' '), + (String, u'"'), + (Error, u'('), + (Error, u'#'), + (Error, u"'"), + (String, u'x'), + (Error, u')'), + (Comment.Preproc, u'\\)'), + (String, u'x'), + (Comment.Preproc, u'\\)'), + (Error, u')'), + (String, u'"'), + (Punctuation, u'#'), + (Comment.Preproc, u' '), + (String, u'{{'), + (String, u'x'), + (Comment.Preproc, u'\\)'), + (Error, u')'), + (String, u'x'), + (Error, u')'), + (Comment.Preproc, u'\\)'), + (Error, u'('), + (Error, u'#'), + (Error, u"'"), + (String, u'}}'), + (Punctuation, u')'), + (Comment.Single, u';'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_html_formatter.py b/tests/test_html_formatter.py index 79990edd..10450c56 100644 --- a/tests/test_html_formatter.py +++ b/tests/test_html_formatter.py @@ -100,7 +100,7 @@ class HtmlFormatterTest(unittest.TestCase): fmt = HtmlFormatter(**optdict) fmt.format(tokensource, outfile) html = outfile.getvalue() - self.assertTrue(re.search("<pre>\s+1\s+2\s+3", html)) + self.assertTrue(re.search(r"<pre>\s+1\s+2\s+3", html)) def test_linenos_with_startnum(self): optdict = dict(linenos=True, linenostart=5) @@ -108,7 +108,7 @@ class HtmlFormatterTest(unittest.TestCase): fmt = HtmlFormatter(**optdict) fmt.format(tokensource, outfile) html = outfile.getvalue() - self.assertTrue(re.search("<pre>\s+5\s+6\s+7", html)) + self.assertTrue(re.search(r"<pre>\s+5\s+6\s+7", html)) def test_lineanchors(self): optdict = dict(lineanchors="foo") diff --git a/tests/test_markdown_lexer.py b/tests/test_markdown_lexer.py new file mode 100644 index 00000000..16d1f28d --- /dev/null +++ b/tests/test_markdown_lexer.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" + Pygments regex lexer tests + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import unittest + +from pygments.lexers.markup import MarkdownLexer + + +class SameTextTests(unittest.TestCase): + + lexer = MarkdownLexer() + + def assert_same_text(self, text): + """Show that lexed markdown does not remove any content. """ + tokens = list(self.lexer.get_tokens_unprocessed(text)) + output = ''.join(t[2] for t in tokens) + self.assertEqual(text, output) + + def test_code_fence(self): + self.assert_same_text(r'```\nfoo\n```\n') + + def test_code_fence_gsm(self): + self.assert_same_text(r'```markdown\nfoo\n```\n') + + def test_code_fence_gsm_with_no_lexer(self): + self.assert_same_text(r'```invalid-lexer\nfoo\n```\n') diff --git a/tests/test_python.py b/tests/test_python.py index e99687a6..6445022c 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -111,3 +111,23 @@ class Python3Test(unittest.TestCase): (Token.Text, u'\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def test_pep_515(self): + """ + Tests that the lexer can parse numeric literals with underscores + """ + fragments = ( + (Token.Literal.Number.Integer, u'1_000_000'), + (Token.Literal.Number.Float, u'1_000.000_001'), + (Token.Literal.Number.Float, u'1_000e1_000j'), + (Token.Literal.Number.Hex, u'0xCAFE_F00D'), + (Token.Literal.Number.Bin, u'0b_0011_1111_0100_1110'), + (Token.Literal.Number.Oct, u'0o_777_123'), + ) + + for token, fragment in fragments: + tokens = [ + (token, fragment), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_r.py b/tests/test_r.py new file mode 100644 index 00000000..70148e53 --- /dev/null +++ b/tests/test_r.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +""" + R Tests + ~~~~~~~~~ + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest + +from pygments.lexers import SLexer +from pygments.token import Token, Name, Punctuation + + +class RTest(unittest.TestCase): + def setUp(self): + self.lexer = SLexer() + + def testCall(self): + fragment = u'f(1, a)\n' + tokens = [ + (Name.Function, u'f'), + (Punctuation, u'('), + (Token.Literal.Number, u'1'), + (Punctuation, u','), + (Token.Text, u' '), + (Token.Name, u'a'), + (Punctuation, u')'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testName1(self): + fragment = u'._a_2.c' + tokens = [ + (Name, u'._a_2.c'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testName2(self): + # Invalid names are valid if backticks are used + fragment = u'`.1 blah`' + tokens = [ + (Name, u'`.1 blah`'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testName3(self): + # Internal backticks can be escaped + fragment = u'`.1 \\` blah`' + tokens = [ + (Name, u'`.1 \\` blah`'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testCustomOperator(self): + fragment = u'7 % and % 8' + tokens = [ + (Token.Literal.Number, u'7'), + (Token.Text, u' '), + (Token.Operator, u'% and %'), + (Token.Text, u' '), + (Token.Literal.Number, u'8'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_rtf_formatter.py b/tests/test_rtf_formatter.py index 756c03a9..44da5768 100644 --- a/tests/test_rtf_formatter.py +++ b/tests/test_rtf_formatter.py @@ -80,7 +80,7 @@ class RtfFormatterTest(StringTests, unittest.TestCase): self.assertEndsWith(result, expected+self.foot, msg) def test_escape_characters(self): - t = u'\ {{' + t = u'\\ {{' result = self.format_rtf(t) expected = (r'\\ \{\{') if not result.endswith(self.foot): @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py33, py34 +envlist = py27, py35, py36, py37 [testenv] deps = nose |