diff options
310 files changed, 10654 insertions, 3797 deletions
@@ -8,6 +8,7 @@ syntax: glob .project .tags .tox +.cache/ Pygments.egg-info/* TAGS build/* @@ -7,7 +7,8 @@ Other contributors, listed alphabetically, are: * Sam Aaron -- Ioke lexer * Ali Afshar -- image formatter -* Thomas Aglassinger -- Easytrieve, JCL, Rexx and Transact-SQL lexers +* Thomas Aglassinger -- Easytrieve, JCL, Rexx, Transact-SQL and VBScript + lexers * Muthiah Annamalai -- Ezhil lexer * Kumar Appaiah -- Debian control lexer * Andreas Amann -- AppleScript lexer @@ -31,6 +32,7 @@ Other contributors, listed alphabetically, are: * Sébastien Bigaret -- QVT Operational lexer * Jarrett Billingsley -- MiniD lexer * Adam Blinkinsop -- Haskell, Redcode lexers +* Stéphane Blondon -- SGF lexer * Frits van Bommel -- assembler lexers * Pierre Bourdon -- bugfixes * Matthias Bussonnier -- ANSI style handling for terminal-256 formatter @@ -72,7 +74,7 @@ Other contributors, listed alphabetically, are: * Alex Gosse -- TrafficScript lexer * Patrick Gotthardt -- PHP namespaces support * Olivier Guibe -- Asymptote lexer -* Jordi Gutiérrez Hermoso -- Octave lexer +* Phil Hagelberg -- Fennel lexer * Florian Hahn -- Boogie lexer * Martin Harriman -- SNOBOL lexer * Matthew Harrison -- SVG formatter @@ -81,6 +83,7 @@ Other contributors, listed alphabetically, are: * Aslak Hellesøy -- Gherkin lexer * Greg Hendershott -- Racket lexer * Justin Hendrick -- ParaSail lexer +* Jordi Gutiérrez Hermoso -- Octave lexer * David Hess, Fish Software, Inc. -- Objective-J lexer * Varun Hiremath -- Debian control lexer * Rob Hoelz -- Perl 6 lexer @@ -129,6 +132,7 @@ Other contributors, listed alphabetically, are: * Stephen McKamey -- Duel/JBST lexer * Brian McKenna -- F# lexer * Charles McLaughlin -- Puppet lexer +* Kurt McKee -- Tera Term macro lexer * Lukas Meuser -- BBCode formatter, Lua lexer * Cat Miller -- Pig lexer * Paul Miller -- LiveScript lexer @@ -160,6 +164,7 @@ Other contributors, listed alphabetically, are: * Elias Rabel -- Fortran fixed form lexer * raichoo -- Idris lexer * Kashif Rasul -- CUDA lexer +* Nathan Reed -- HLSL lexer * Justin Reidy -- MXML lexer * Norman Richards -- JSON lexer * Corey Richardson -- Rust lexer updates @@ -185,7 +190,7 @@ Other contributors, listed alphabetically, are: * Alexander Smishlajev -- Visual FoxPro lexer * Steve Spigarelli -- XQuery lexer * Jerome St-Louis -- eC lexer -* Camil Staps -- Clean and NuSMV lexers +* Camil Staps -- Clean and NuSMV lexers; Solarized style * James Strachan -- Kotlin lexer * Tom Stuart -- Treetop lexer * Colin Sullivan -- SuperCollider lexer @@ -216,5 +221,8 @@ Other contributors, listed alphabetically, are: * Alex Zimin -- Nemerle lexer * Rob Zimmerman -- Kal lexer * Vincent Zurczak -- Roboconf lexer +* Rostyslav Golda -- FloScript lexer +* GitHub, Inc -- DASM16, Augeas, TOML, and Slash lexers +* Simon Garnotel -- FreeFem++ lexer Many thanks for all contributions! @@ -6,9 +6,138 @@ Issue numbers refer to the tracker at pull request numbers to the requests at <https://bitbucket.org/birkenfeld/pygments-main/pull-requests/merged>. +Version 2.4.2 +------------- +(released May 28, 2019) + +- Updated lexers: + + * Erlang, Elxir shells (PR#823, #1521) + * YAML (#1528) + +- Fix encoding error when guessing lexer with given ``encoding`` option + (#1438) +- Bump ``NasmLexer`` priority over ``TasmLexer`` for ``.asm`` files + (fixes #1326) + + +Version 2.4.1 +------------- +(released May 24, 2019) + +- Updated lexers: + + * Coq (#1430) + * MSDOS Session (PR#734) + * NASM (#1517) + * Objective-C (PR#813, #1508) + * Prolog (#1511) + * TypeScript (#1515) + +- Support CSS variables in stylesheets (PR#814, #1356) +- Fix F# lexer name (PR#709) +- Fix ``TerminalFormatter`` using bold for bright text (#1480) + + +Version 2.4.0 +------------- +(released May 8, 2019) + +- Added lexers: + + * Augeas (PR#807) + * BBC Basic (PR#806) + * Boa (PR#756) + * Charm++ CI (PR#788) + * DASM16 (PR#807) + * FloScript (PR#750) + * FreeFem++ (PR#785) + * Hspec (PR#790) + * Pony (PR#627) + * SGF (PR#780) + * Slash (PR#807) + * Slurm (PR#760) + * Tera Term Language (PR#749) + * TOML (PR#807) + * Unicon (PR#731) + * VBScript (PR#673) + +- Updated lexers: + + * Apache2 (PR#766) + * Cypher (PR#746) + * LLVM (PR#792) + * Makefiles (PR#766) + * PHP (#1482) + * Rust + * SQL (PR#672) + * Stan (PR#774) + * Stata (PR#800) + * Terraform (PR#787) + * YAML + +- Add solarized style (PR#708) +- Add support for Markdown reference-style links (PR#753) +- Add license information to generated HTML/CSS files (#1496) +- Change ANSI color names (PR#777) +- Fix catastrophic backtracking in the bash lexer (#1494) +- Fix documentation failing to build using Sphinx 2.0 (#1501) +- Fix incorrect links in the Lisp and R lexer documentation (PR#775) +- Fix rare unicode errors on Python 2.7 (PR#798, #1492) +- Fix lexers popping from an empty stack (#1506) +- TypoScript uses ``.typoscript`` now (#1498) +- Updated Trove classifiers and ``pip`` requirements (PR#799) + + +Version 2.3.1 +------------- +(released Dec 16, 2018) + +- Updated lexers: + + * ASM (PR#784) + * Chapel (PR#735) + * Clean (PR#621) + * CSound (PR#684) + * Elm (PR#744) + * Fortran (PR#747) + * GLSL (PR#740) + * Haskell (PR#745) + * Hy (PR#754) + * Igor Pro (PR#764) + * PowerShell (PR#705) + * Python (PR#720, #1299, PR#715) + * SLexer (PR#680) + * YAML (PR#762, PR#724) + +- Fix invalid string escape sequences +- Fix `FutureWarning` introduced by regex changes in Python 3.7 + + +Version 2.3.0 +------------- +(released Nov 25, 2018) + +- Added lexers: + + * Fennel (PR#783) + * HLSL (PR#675) + +- Updated lexers: + + * Dockerfile (PR#714) + +- Minimum Python versions changed to 2.7 and 3.5 +- Added support for Python 3.7 generator changes (PR#772) +- Fix incorrect token type in SCSS for single-quote strings (#1322) +- Use `terminal256` formatter if `TERM` contains `256` (PR#666) +- Fix incorrect handling of GitHub style fences in Markdown (PR#741, #1389) +- Fix `%a` not being highlighted in Python3 strings (PR#727) + + Version 2.2.0 ------------- -(release Jan 22, 2017) +(released Jan 22, 2017) - Added lexers: @@ -1,4 +1,4 @@ -Copyright (c) 2006-2017 by the respective authors (see AUTHORS file). +Copyright (c) 2006-2019 by the respective authors (see AUTHORS file). All rights reserved. Redistribution and use in source and binary forms, with or without @@ -4,7 +4,7 @@ # # Combines scripts for common tasks. # -# :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. +# :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. # :license: BSD, see LICENSE for details. # @@ -63,3 +63,9 @@ tox-test: tox-test-coverage: @tox -- --with-coverage --cover-package=pygments --cover-erase $(TEST) + +RLMODULES = pygments.lexers + +regexlint: + @if [ -z "$(REGEXLINT)" ]; then echo "Please set REGEXLINT=checkout path"; exit 1; fi + PYTHONPATH=`pwd`:$(REGEXLINT) $(REGEXLINT)/regexlint/cmdline.py $(RLMODULES) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml new file mode 100644 index 00000000..4a9f1b6d --- /dev/null +++ b/bitbucket-pipelines.yml @@ -0,0 +1,34 @@ +pipelines: + default: + - step: + name: Test on Python 2.7 + image: python:2.7 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py27 + - step: + name: Test on Python 3.5 + image: python:3.5 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py35 + - step: + name: Test on Python 3.6 + image: python:3.6 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py36 + - step: + name: Test on Python 3.7 + image: python:3.7 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py37 diff --git a/doc/_themes/pygments14/layout.html b/doc/_themes/pygments14/layout.html index e8860827..53f8f37f 100644 --- a/doc/_themes/pygments14/layout.html +++ b/doc/_themes/pygments14/layout.html @@ -82,7 +82,7 @@ {% block footer %} <div class="footer" role="contentinfo"> - © Copyright 2006-2017, Georg Brandl and Pygments contributors. + © Copyright 2006-2019, Georg Brandl and Pygments contributors. Created using <a href="http://sphinx-doc.org/">Sphinx</a> {{ sphinx_version }}. <br/> Pygments logo created by <a href="http://joelunger.com">Joel Unger</a>. diff --git a/doc/_themes/pygments14/static/pygments14.css_t b/doc/_themes/pygments14/static/pygments14.css_t index 7f09f623..72ca942e 100644 --- a/doc/_themes/pygments14/static/pygments14.css_t +++ b/doc/_themes/pygments14/static/pygments14.css_t @@ -4,7 +4,7 @@ * * Sphinx stylesheet -- pygments14 theme. Heavily copied from sphinx13. * - * :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + * :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ diff --git a/doc/conf.py b/doc/conf.py index 51a91617..00db7d9b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -125,8 +125,8 @@ html_static_path = ['_static'] #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -html_sidebars = {'index': 'indexsidebar.html', - 'docs/*': 'docssidebar.html'} +html_sidebars = {'index': ['indexsidebar.html'], + 'docs/*': ['docssidebar.html']} # Additional templates that should be rendered to pages, maps page names to # template names. diff --git a/doc/docs/quickstart.rst b/doc/docs/quickstart.rst index dba7698a..3a823e7f 100644 --- a/doc/docs/quickstart.rst +++ b/doc/docs/quickstart.rst @@ -39,7 +39,7 @@ Here is a small example for highlighting Python code: from pygments.formatters import HtmlFormatter code = 'print "Hello World"' - print highlight(code, PythonLexer(), HtmlFormatter()) + print(highlight(code, PythonLexer(), HtmlFormatter())) which prints something like this: @@ -56,7 +56,7 @@ can be produced by: .. sourcecode:: python - print HtmlFormatter().get_style_defs('.highlight') + print(HtmlFormatter().get_style_defs('.highlight')) The argument to :func:`get_style_defs` is used as an additional CSS selector: the output may look like this: diff --git a/doc/docs/styles.rst b/doc/docs/styles.rst index 1094a270..570293a5 100644 --- a/doc/docs/styles.rst +++ b/doc/docs/styles.rst @@ -153,17 +153,17 @@ Terminal Styles .. versionadded:: 2.2 Custom styles used with the 256-color terminal formatter can also map colors to -use the 8 default ANSI colors. To do so, use ``#ansigreen``, ``#ansired`` or +use the 8 default ANSI colors. To do so, use ``ansigreen``, ``ansibrightred`` or any other colors defined in :attr:`pygments.style.ansicolors`. Foreground ANSI colors will be mapped to the corresponding `escape codes 30 to 37 <https://en.wikipedia.org/wiki/ANSI_escape_code#Colors>`_ thus respecting any custom color mapping and themes provided by many terminal emulators. Light variants are treated as foreground color with and an added bold flag. -``bg:#ansi<color>`` will also be respected, except the light variant will be the +``bg:ansi<color>`` will also be respected, except the light variant will be the same shade as their dark variant. See the following example where the color of the string ``"hello world"`` is -governed by the escape sequence ``\x1b[34;01m`` (Ansi Blue, Bold, 41 being red +governed by the escape sequence ``\x1b[34;01m`` (Ansi bright blue, Bold, 41 being red background) instead of an extended foreground & background color. .. sourcecode:: pycon @@ -176,7 +176,7 @@ background) instead of an extended foreground & background color. >>> class MyStyle(Style): styles = { - Token.String: '#ansiblue bg:#ansired', + Token.String: 'ansibrightblue bg:ansibrightred', } >>> code = 'print("Hello World")' @@ -184,18 +184,49 @@ background) instead of an extended foreground & background color. >>> print(result.encode()) b'\x1b[34;41;01m"\x1b[39;49;00m\x1b[34;41;01mHello World\x1b[39;49;00m\x1b[34;41;01m"\x1b[39;49;00m' -Colors specified using ``#ansi*`` are converted to a default set of RGB colors +Colors specified using ``ansi*`` are converted to a default set of RGB colors when used with formatters other than the terminal-256 formatter. By definition of ANSI, the following colors are considered "light" colors, and will be rendered by most terminals as bold: -- "darkgray", "red", "green", "yellow", "blue", "fuchsia", "turquoise", "white" +- "brightblack" (darkgrey), "brightred", "brightgreen", "brightyellow", "brightblue", + "brightmagenta", "brightcyan", "white" The following are considered "dark" colors and will be rendered as non-bold: -- "black", "darkred", "darkgreen", "brown", "darkblue", "purple", "teal", - "lightgray" +- "black", "red", "green", "yellow", "blue", "magenta", "cyan", + "gray" Exact behavior might depends on the terminal emulator you are using, and its settings. + +.. _new-ansi-color-names: + +.. versionchanged:: 2.4 + +The definition of the ANSI color names has changed. +New names are easier to understand and align to the colors used in other projects. + +===================== ==================== +New names Pygments up to 2.3 +===================== ==================== +``ansiblack`` ``#ansiblack`` +``ansired`` ``#ansidarkred`` +``ansigreen`` ``#ansidarkgreen`` +``ansiyellow`` ``#ansibrown`` +``ansiblue`` ``#ansidarkblue`` +``ansimagenta`` ``#ansipurple`` +``ansicyan`` ``#ansiteal`` +``ansigray`` ``#ansilightgray`` +``ansibrightblack`` ``#ansidarkgray`` +``ansibrightred`` ``#ansired`` +``ansibrightgreen`` ``#ansigreen`` +``ansibrightyellow`` ``#ansiyellow`` +``ansibrightblue`` ``#ansiblue`` +``ansibrightmagenta`` ``#ansifuchsia`` +``ansibrightcyan`` ``#ansiturquoise`` +``ansiwhite`` ``#ansiwhite`` +===================== ==================== + +Old ANSI color names are deprecated but will still work. diff --git a/doc/faq.rst b/doc/faq.rst index f375828b..172929e0 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -35,8 +35,8 @@ and in this case, source code! What are the system requirements? --------------------------------- -Pygments only needs a standard Python install, version 2.6 or higher or version -3.3 or higher for Python 3. No additional libraries are needed. +Pygments only needs a standard Python install, version 2.7 or higher or version +3.5 or higher for Python 3. No additional libraries are needed. How can I use Pygments? ----------------------- diff --git a/doc/languages.rst b/doc/languages.rst index 7fa8eb2f..b06ccc55 100644 --- a/doc/languages.rst +++ b/doc/languages.rst @@ -14,12 +14,16 @@ Programming languages * AppleScript * Assembly (various) * Asymptote +* `Augeas <http://augeas.net>`_ * Awk +* BBC Basic * Befunge +* `Boa <http://boa.cs.iastate.edu/docs/index.php>`_ * Boo * BrainFuck * C, C++ * C# +* `Charm++ CI <http://charmplusplus.org/>`_ * Clojure * CoffeeScript * ColdFusion @@ -30,6 +34,7 @@ Programming languages * `Cython <http://cython.org>`_ * `D <http://dlang.org>`_ * Dart +* DCPU-16 * Delphi * Dylan * `Elm <http://elm-lang.org/>`_ @@ -37,13 +42,18 @@ Programming languages * `Ezhil <http://ezhillang.org>`_ Ezhil - A Tamil programming language * Factor * Fancy +* `Fennel <https://fennel-lang.org/>`_ +* `FloScript <http://ioflo.com/>`_ * Fortran +* `FreeFEM++ <https://freefem.org/>`_ * F# * GAP * Gherkin (Cucumber) * GL shaders * Groovy * `Haskell <http://www.haskell.org>`_ (incl. Literate Haskell) +* HLSL +* `HSpec <http://hackage.haskell.org/package/hspec>`_ * IDL * Io * Java @@ -64,7 +74,8 @@ Programming languages * Octave * OCaml * PHP -* `Perl <http://perl.org>`_ +* `Perl 5 <http://perl.org>`_ and `Perl 6 <https://perl6.org>`_ +* `Pony <https://www.ponylang.io/>`_ * PovRay * PostScript * PowerShell @@ -79,9 +90,14 @@ Programming languages * Scala * Scheme * Scilab +* `SGF <https://www.red-bean.com/sgf/>`_ +* `Slash <https://github.com/arturadib/Slash-A>`_ +* `Slurm <https://slurm.schedmd.com/overview.html>`_ * Smalltalk * SNOBOL * Tcl +* `Tera Term language <https://ttssh2.osdn.jp/>`_ +* `TOML <https://github.com/toml-lang/toml>`_ * Vala * Verilog * VHDL diff --git a/external/markdown-processor.py b/external/markdown-processor.py index cb8d793d..3a7bca4c 100644 --- a/external/markdown-processor.py +++ b/external/markdown-processor.py @@ -22,7 +22,7 @@ .. _Markdown: https://pypi.python.org/pypi/Markdown - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/external/moin-parser.py b/external/moin-parser.py index 03a7c3c3..4e74447f 100644 --- a/external/moin-parser.py +++ b/external/moin-parser.py @@ -31,7 +31,7 @@ If you do not want to do that and are willing to accept larger HTML output, you can set the INLINESTYLES option below to True. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/external/rst-directive.py b/external/rst-directive.py index de26dd03..0b7831f7 100644 --- a/external/rst-directive.py +++ b/external/rst-directive.py @@ -31,7 +31,7 @@ .. _directive documentation: http://docutils.sourceforge.net/docs/howto/rst-directives.html - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/__init__.py b/pygments/__init__.py index 394a85f2..15c226a3 100644 --- a/pygments/__init__.py +++ b/pygments/__init__.py @@ -22,14 +22,14 @@ .. _Pygments tip: http://bitbucket.org/birkenfeld/pygments-main/get/tip.zip#egg=Pygments-dev - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import sys from pygments.util import StringIO, BytesIO -__version__ = '2.2.0' +__version__ = '2.4.2' __docformat__ = 'restructuredtext' __all__ = ['lex', 'format', 'highlight'] diff --git a/pygments/cmdline.py b/pygments/cmdline.py index 2f6c1401..292cb870 100644 --- a/pygments/cmdline.py +++ b/pygments/cmdline.py @@ -5,7 +5,7 @@ Command line interface. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -233,7 +233,7 @@ def main_inner(popts, args, usage): return 0 if opts.pop('-V', None) is not None: - print('Pygments version %s, (c) 2006-2017 by Georg Brandl.' % __version__) + print('Pygments version %s, (c) 2006-2019 by Georg Brandl.' % __version__) return 0 # handle ``pygmentize -L`` diff --git a/pygments/console.py b/pygments/console.py index 31b6839d..a05b256e 100644 --- a/pygments/console.py +++ b/pygments/console.py @@ -5,7 +5,7 @@ Format colored console output. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -22,22 +22,19 @@ codes["underline"] = esc + "04m" codes["blink"] = esc + "05m" codes["overline"] = esc + "06m" -dark_colors = ["black", "darkred", "darkgreen", "brown", "darkblue", - "purple", "teal", "lightgray"] -light_colors = ["darkgray", "red", "green", "yellow", "blue", - "fuchsia", "turquoise", "white"] +dark_colors = ["black", "red", "green", "yellow", "blue", + "magenta", "cyan", "gray"] +light_colors = ["brightblack", "brightred", "brightgreen", "brightyellow", "brightblue", + "brightmagenta", "brightcyan", "white"] x = 30 for d, l in zip(dark_colors, light_colors): codes[d] = esc + "%im" % x - codes[l] = esc + "%i;01m" % x + codes[l] = esc + "%im" % (60 + x) x += 1 del d, l, x -codes["darkteal"] = codes["turquoise"] -codes["darkyellow"] = codes["brown"] -codes["fuscia"] = codes["fuchsia"] codes["white"] = codes["bold"] diff --git a/pygments/filter.py b/pygments/filter.py index 68be7ad7..7f81920b 100644 --- a/pygments/filter.py +++ b/pygments/filter.py @@ -5,7 +5,7 @@ Module that implements the default filter. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/filters/__init__.py b/pygments/filters/__init__.py index 45f9608c..3fe6caa7 100644 --- a/pygments/filters/__init__.py +++ b/pygments/filters/__init__.py @@ -6,7 +6,7 @@ Module containing filter lookup functions and default filters. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatter.py b/pygments/formatter.py index c0780f62..f09f6e3c 100644 --- a/pygments/formatter.py +++ b/pygments/formatter.py @@ -5,7 +5,7 @@ Base formatter class. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/__init__.py b/pygments/formatters/__init__.py index 965c5f1a..4fa4178e 100644 --- a/pygments/formatters/__init__.py +++ b/pygments/formatters/__init__.py @@ -5,7 +5,7 @@ Pygments formatters. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -98,7 +98,8 @@ def load_formatter_from_file(filename, formattername="CustomFormatter", try: # This empty dict will contain the namespace for the exec'd file custom_namespace = {} - exec(open(filename, 'rb').read(), custom_namespace) + with open(filename, 'rb') as f: + exec(f.read(), custom_namespace) # Retrieve the class `formattername` from that namespace if formattername not in custom_namespace: raise ClassNotFound('no valid %s class found in %s' % diff --git a/pygments/formatters/_mapping.py b/pygments/formatters/_mapping.py index 7bb3e71c..5086e519 100755 --- a/pygments/formatters/_mapping.py +++ b/pygments/formatters/_mapping.py @@ -9,7 +9,7 @@ Do not alter the FORMATTERS dictionary by hand. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/bbcode.py b/pygments/formatters/bbcode.py index 9fc9476d..784aee3a 100644 --- a/pygments/formatters/bbcode.py +++ b/pygments/formatters/bbcode.py @@ -5,7 +5,7 @@ BBcode formatter. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py index 2969d502..d65c09ce 100644 --- a/pygments/formatters/html.py +++ b/pygments/formatters/html.py @@ -5,7 +5,7 @@ Formatter for HTML output. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -41,6 +41,11 @@ def escape_html(text, table=_escape_html_table): """Escape &, <, > as well as single and double quotes for HTML.""" return text.translate(table) +def webify(color): + if color.startswith('calc') or color.startswith('var'): + return color + else: + return '#' + color def _get_ttype_class(ttype): fname = STANDARD_TYPES.get(ttype) @@ -55,6 +60,11 @@ def _get_ttype_class(ttype): CSSFILE_TEMPLATE = '''\ +/* +generated by Pygments <http://pygments.org> +Copyright 2006-2019 by the Pygments team. +Licensed under the BSD license, see LICENSE for details. +*/ td.linenos { background-color: #f0f0f0; padding-right: 10px; } span.lineno { background-color: #f0f0f0; padding: 0 5px 0 5px; } pre { line-height: 125%%; } @@ -64,7 +74,11 @@ pre { line-height: 125%%; } DOC_HEADER = '''\ <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> - +<!-- +generated by Pygments <http://pygments.org> +Copyright 2006-2019 by the Pygments team. +Licensed under the BSD license, see LICENSE for details. +--> <html> <head> <title>%(title)s</title> @@ -322,11 +336,17 @@ class HtmlFormatter(Formatter): .. versionadded:: 1.6 `filename` - A string used to generate a filename when rendering <pre> blocks, + A string used to generate a filename when rendering ``<pre>`` blocks, for example if displaying source code. .. versionadded:: 2.1 + `wrapcode` + Wrap the code inside ``<pre>`` blocks using ``<code>``, as recommended + by the HTML5 specification. + + .. versionadded:: 2.4 + **Subclassing the HTML formatter** @@ -395,6 +415,7 @@ class HtmlFormatter(Formatter): self.tagsfile = self._decodeifneeded(options.get('tagsfile', '')) self.tagurlformat = self._decodeifneeded(options.get('tagurlformat', '')) self.filename = self._decodeifneeded(options.get('filename', '')) + self.wrapcode = get_bool_opt(options, 'wrapcode', False) if self.tagsfile: if not ctags: @@ -451,7 +472,7 @@ class HtmlFormatter(Formatter): name = self._get_css_class(ttype) style = '' if ndef['color']: - style += 'color: #%s; ' % ndef['color'] + style += 'color: %s; ' % webify(ndef['color']) if ndef['bold']: style += 'font-weight: bold; ' if ndef['italic']: @@ -459,9 +480,9 @@ class HtmlFormatter(Formatter): if ndef['underline']: style += 'text-decoration: underline; ' if ndef['bgcolor']: - style += 'background-color: #%s; ' % ndef['bgcolor'] + style += 'background-color: %s; ' % webify(ndef['bgcolor']) if ndef['border']: - style += 'border: 1px solid #%s; ' % ndef['border'] + style += 'border: 1px solid %s; ' % webify(ndef['border']) if style: t2c[ttype] = name # save len(ttype) to enable ordering the styles by @@ -535,10 +556,9 @@ class HtmlFormatter(Formatter): # write CSS file only if noclobber_cssfile isn't given as an option. try: if not os.path.exists(cssfilename) or not self.noclobber_cssfile: - cf = open(cssfilename, "w") - cf.write(CSSFILE_TEMPLATE % - {'styledefs': self.get_style_defs('body')}) - cf.close() + with open(cssfilename, "w") as cf: + cf.write(CSSFILE_TEMPLATE % + {'styledefs': self.get_style_defs('body')}) except IOError as err: err.strerror = 'Error writing CSS file: ' + err.strerror raise @@ -709,6 +729,12 @@ class HtmlFormatter(Formatter): yield tup yield 0, '</pre>' + def _wrap_code(self, inner): + yield 0, '<code>' + for tup in inner: + yield tup + yield 0, '</code>' + def _format_lines(self, tokensource): """ Just format the tokens, without any wrapping tags. @@ -815,7 +841,10 @@ class HtmlFormatter(Formatter): individual lines, in custom generators. See docstring for `format`. Can be overridden. """ - return self._wrap_div(self._wrap_pre(source)) + if self.wrapcode: + return self._wrap_div(self._wrap_pre(self._wrap_code(source))) + else: + return self._wrap_div(self._wrap_pre(source)) def format_unencoded(self, tokensource, outfile): """ diff --git a/pygments/formatters/img.py b/pygments/formatters/img.py index 2fb0dea5..de0ea0a4 100644 --- a/pygments/formatters/img.py +++ b/pygments/formatters/img.py @@ -5,7 +5,7 @@ Formatter for Pixmap output. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -237,7 +237,7 @@ class ImageFormatter(Formatter): bold and italic fonts will be generated. This really should be a monospace font to look sane. - Default: "Bitstream Vera Sans Mono" on Windows, Courier New on \*nix + Default: "Bitstream Vera Sans Mono" on Windows, Courier New on \\*nix `font_size` The font size in points to be used. diff --git a/pygments/formatters/irc.py b/pygments/formatters/irc.py index eb744d74..0650492a 100644 --- a/pygments/formatters/irc.py +++ b/pygments/formatters/irc.py @@ -5,7 +5,7 @@ Formatter for IRC output - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -25,55 +25,55 @@ __all__ = ['IRCFormatter'] IRC_COLORS = { Token: ('', ''), - Whitespace: ('lightgray', 'darkgray'), - Comment: ('lightgray', 'darkgray'), - Comment.Preproc: ('teal', 'turquoise'), - Keyword: ('darkblue', 'blue'), - Keyword.Type: ('teal', 'turquoise'), - Operator.Word: ('purple', 'fuchsia'), - Name.Builtin: ('teal', 'turquoise'), - Name.Function: ('darkgreen', 'green'), - Name.Namespace: ('_teal_', '_turquoise_'), - Name.Class: ('_darkgreen_', '_green_'), - Name.Exception: ('teal', 'turquoise'), - Name.Decorator: ('darkgray', 'lightgray'), - Name.Variable: ('darkred', 'red'), - Name.Constant: ('darkred', 'red'), - Name.Attribute: ('teal', 'turquoise'), - Name.Tag: ('blue', 'blue'), - String: ('brown', 'brown'), - Number: ('darkblue', 'blue'), - - Generic.Deleted: ('red', 'red'), - Generic.Inserted: ('darkgreen', 'green'), + Whitespace: ('gray', 'brightblack'), + Comment: ('gray', 'brightblack'), + Comment.Preproc: ('cyan', 'brightcyan'), + Keyword: ('blue', 'brightblue'), + Keyword.Type: ('cyan', 'brightcyan'), + Operator.Word: ('magenta', 'brightcyan'), + Name.Builtin: ('cyan', 'brightcyan'), + Name.Function: ('green', 'brightgreen'), + Name.Namespace: ('_cyan_', '_brightcyan_'), + Name.Class: ('_green_', '_brightgreen_'), + Name.Exception: ('cyan', 'brightcyan'), + Name.Decorator: ('brightblack', 'gray'), + Name.Variable: ('red', 'brightred'), + Name.Constant: ('red', 'brightred'), + Name.Attribute: ('cyan', 'brightcyan'), + Name.Tag: ('brightblue', 'brightblue'), + String: ('yellow', 'yellow'), + Number: ('blue', 'brightblue'), + + Generic.Deleted: ('brightred', 'brightred'), + Generic.Inserted: ('green', 'brightgreen'), Generic.Heading: ('**', '**'), - Generic.Subheading: ('*purple*', '*fuchsia*'), - Generic.Error: ('red', 'red'), + Generic.Subheading: ('*magenta*', '*brightmagenta*'), + Generic.Error: ('brightred', 'brightred'), - Error: ('_red_', '_red_'), + Error: ('_brightred_', '_brightred_'), } IRC_COLOR_MAP = { 'white': 0, 'black': 1, - 'darkblue': 2, - 'green': 3, - 'red': 4, - 'brown': 5, - 'purple': 6, + 'blue': 2, + 'brightgreen': 3, + 'brightred': 4, + 'yellow': 5, + 'magenta': 6, 'orange': 7, - 'darkgreen': 7, #compat w/ ansi - 'yellow': 8, + 'green': 7, #compat w/ ansi + 'brightyellow': 8, 'lightgreen': 9, - 'turquoise': 9, # compat w/ ansi - 'teal': 10, + 'brightcyan': 9, # compat w/ ansi + 'cyan': 10, 'lightblue': 11, - 'darkred': 11, # compat w/ ansi - 'blue': 12, - 'fuchsia': 13, - 'darkgray': 14, - 'lightgray': 15, + 'red': 11, # compat w/ ansi + 'brightblue': 12, + 'brightmagenta': 13, + 'brightblack': 14, + 'gray': 15, } def ircformat(color, text): diff --git a/pygments/formatters/latex.py b/pygments/formatters/latex.py index 336b59de..7f6aa9e3 100644 --- a/pygments/formatters/latex.py +++ b/pygments/formatters/latex.py @@ -5,7 +5,7 @@ Formatter for LaTeX fancyvrb output. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/other.py b/pygments/formatters/other.py index d6bfcacf..a2eca91a 100644 --- a/pygments/formatters/other.py +++ b/pygments/formatters/other.py @@ -5,7 +5,7 @@ Other formatters: NullFormatter, RawTokenFormatter. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/rtf.py b/pygments/formatters/rtf.py index c6353c12..0dae6aab 100644 --- a/pygments/formatters/rtf.py +++ b/pygments/formatters/rtf.py @@ -5,7 +5,7 @@ A formatter that generates RTF files. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/svg.py b/pygments/formatters/svg.py index 944b25e0..ccfd2b3f 100644 --- a/pygments/formatters/svg.py +++ b/pygments/formatters/svg.py @@ -5,7 +5,7 @@ Formatter for SVG output. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/terminal.py b/pygments/formatters/terminal.py index b8fec52e..e60bde91 100644 --- a/pygments/formatters/terminal.py +++ b/pygments/formatters/terminal.py @@ -5,7 +5,7 @@ Formatter for terminal output with ANSI sequences. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -26,33 +26,33 @@ __all__ = ['TerminalFormatter'] TERMINAL_COLORS = { Token: ('', ''), - Whitespace: ('lightgray', 'darkgray'), - Comment: ('lightgray', 'darkgray'), - Comment.Preproc: ('teal', 'turquoise'), - Keyword: ('darkblue', 'blue'), - Keyword.Type: ('teal', 'turquoise'), - Operator.Word: ('purple', 'fuchsia'), - Name.Builtin: ('teal', 'turquoise'), - Name.Function: ('darkgreen', 'green'), - Name.Namespace: ('_teal_', '_turquoise_'), - Name.Class: ('_darkgreen_', '_green_'), - Name.Exception: ('teal', 'turquoise'), - Name.Decorator: ('darkgray', 'lightgray'), - Name.Variable: ('darkred', 'red'), - Name.Constant: ('darkred', 'red'), - Name.Attribute: ('teal', 'turquoise'), - Name.Tag: ('blue', 'blue'), - String: ('brown', 'brown'), - Number: ('darkblue', 'blue'), - - Generic.Deleted: ('red', 'red'), - Generic.Inserted: ('darkgreen', 'green'), + Whitespace: ('gray', 'brightblack'), + Comment: ('gray', 'brightblack'), + Comment.Preproc: ('cyan', 'brightcyan'), + Keyword: ('blue', 'brightblue'), + Keyword.Type: ('cyan', 'brightcyan'), + Operator.Word: ('magenta', 'brightmagenta'), + Name.Builtin: ('cyan', 'brightcyan'), + Name.Function: ('green', 'brightgreen'), + Name.Namespace: ('_cyan_', '_brightcyan_'), + Name.Class: ('_green_', '_brightgreen_'), + Name.Exception: ('cyan', 'brightcyan'), + Name.Decorator: ('brightblack', 'gray'), + Name.Variable: ('red', 'brightred'), + Name.Constant: ('red', 'brightred'), + Name.Attribute: ('cyan', 'brightcyan'), + Name.Tag: ('brightblue', 'brightblue'), + String: ('yellow', 'yellow'), + Number: ('blue', 'brightblue'), + + Generic.Deleted: ('brightred', 'brightred'), + Generic.Inserted: ('green', 'brightgreen'), Generic.Heading: ('**', '**'), - Generic.Subheading: ('*purple*', '*fuchsia*'), + Generic.Subheading: ('*magenta*', '*brightmagenta*'), Generic.Prompt: ('**', '**'), - Generic.Error: ('red', 'red'), + Generic.Error: ('brightred', 'brightred'), - Error: ('_red_', '_red_'), + Error: ('_brightred_', '_brightred_'), } diff --git a/pygments/formatters/terminal256.py b/pygments/formatters/terminal256.py index b80dc7dd..43ec01c2 100644 --- a/pygments/formatters/terminal256.py +++ b/pygments/formatters/terminal256.py @@ -11,7 +11,7 @@ Formatter version 1. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -50,7 +50,7 @@ class EscapeSequence: attrs = [] if self.fg is not None: if self.fg in ansicolors: - esc = codes[self.fg[5:]] + esc = codes[self.fg.replace('ansi','')] if ';01m' in esc: self.bold = True # extract fg color code. @@ -59,7 +59,7 @@ class EscapeSequence: attrs.extend(("38", "5", "%i" % self.fg)) if self.bg is not None: if self.bg in ansicolors: - esc = codes[self.bg[5:]] + esc = codes[self.bg.replace('ansi','')] # extract fg color code, add 10 for bg. attrs.append(str(int(esc[2:4])+10)) else: @@ -110,6 +110,12 @@ class Terminal256Formatter(Formatter): `Terminal256Formatter` will map these to non extended foreground color. See :ref:`AnsiTerminalStyle` for more information. + .. versionchanged:: 2.4 + The ANSI color names have been updated with names that are easier to + understand and align with colornames of other projects and terminals. + See :ref:`this table <new-ansi-color-names>` for more information. + + Options accepted: `style` @@ -189,7 +195,7 @@ class Terminal256Formatter(Formatter): def _color_index(self, color): index = self.best_match.get(color, None) if color in ansicolors: - # strip the `#ansi` part and look up code + # strip the `ansi/#ansi` part and look up code index = color self.best_match[color] = index if index is None: diff --git a/pygments/lexer.py b/pygments/lexer.py index 90905ba5..56a7e1e8 100644 --- a/pygments/lexer.py +++ b/pygments/lexer.py @@ -5,7 +5,7 @@ Base lexer classes. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -639,14 +639,20 @@ class RegexLexer(Lexer): if isinstance(new_state, tuple): for state in new_state: if state == '#pop': - statestack.pop() + if len(statestack) > 1: + statestack.pop() elif state == '#push': statestack.append(statestack[-1]) else: statestack.append(state) elif isinstance(new_state, int): - # pop - del statestack[new_state:] + # pop, but keep at least one state on the stack + # (random code leading to unexpected pops should + # not allow exceptions) + if abs(new_state) >= len(statestack): + del statestack[1:] + else: + del statestack[new_state:] elif new_state == '#push': statestack.append(statestack[-1]) else: @@ -724,14 +730,18 @@ class ExtendedRegexLexer(RegexLexer): if isinstance(new_state, tuple): for state in new_state: if state == '#pop': - ctx.stack.pop() + if len(ctx.stack) > 1: + ctx.stack.pop() elif state == '#push': ctx.stack.append(ctx.stack[-1]) else: ctx.stack.append(state) elif isinstance(new_state, int): - # pop - del ctx.stack[new_state:] + # see RegexLexer for why this check is made + if abs(new_state) >= len(ctx.stack): + del ctx.state[1:] + else: + del ctx.stack[new_state:] elif new_state == '#push': ctx.stack.append(ctx.stack[-1]) else: diff --git a/pygments/lexers/__init__.py b/pygments/lexers/__init__.py index 328e072c..1baf93b2 100644 --- a/pygments/lexers/__init__.py +++ b/pygments/lexers/__init__.py @@ -5,7 +5,7 @@ Pygments lexers. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -18,7 +18,7 @@ from os.path import basename from pygments.lexers._mapping import LEXERS from pygments.modeline import get_filetype_from_buffer from pygments.plugin import find_plugin_lexers -from pygments.util import ClassNotFound, itervalues, guess_decode +from pygments.util import ClassNotFound, itervalues, guess_decode, text_type __all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class', @@ -133,7 +133,8 @@ def load_lexer_from_file(filename, lexername="CustomLexer", **options): try: # This empty dict will contain the namespace for the exec'd file custom_namespace = {} - exec(open(filename, 'rb').read(), custom_namespace) + with open(filename, 'rb') as f: + exec(f.read(), custom_namespace) # Retrieve the class `lexername` from that namespace if lexername not in custom_namespace: raise ClassNotFound('no valid %s class found in %s' % @@ -288,6 +289,13 @@ def guess_lexer_for_filename(_fn, _text, **options): def guess_lexer(_text, **options): """Guess a lexer by strong distinctions in the text (eg, shebang).""" + if not isinstance(_text, text_type): + inencoding = options.get('inencoding', options.get('encoding')) + if inencoding: + _text = _text.decode(inencoding or 'utf8') + else: + _text, _ = guess_decode(_text) + # try to get a vim modeline first ft = get_filetype_from_buffer(_text) diff --git a/pygments/lexers/_asy_builtins.py b/pygments/lexers/_asy_builtins.py index 1f831cdb..6ac79ad7 100644 --- a/pygments/lexers/_asy_builtins.py +++ b/pygments/lexers/_asy_builtins.py @@ -10,7 +10,7 @@ TODO: perl/python script in Asymptote SVN similar to asy-list.pl but only for function and variable names. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_cl_builtins.py b/pygments/lexers/_cl_builtins.py index ce5ad48e..d0306fab 100644 --- a/pygments/lexers/_cl_builtins.py +++ b/pygments/lexers/_cl_builtins.py @@ -5,7 +5,7 @@ ANSI Common Lisp builtins. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_cocoa_builtins.py b/pygments/lexers/_cocoa_builtins.py index 064167ff..f7c55c61 100644 --- a/pygments/lexers/_cocoa_builtins.py +++ b/pygments/lexers/_cocoa_builtins.py @@ -8,7 +8,7 @@ File may be also used as standalone generator for aboves. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -40,24 +40,25 @@ if __name__ == '__main__': # pragma: no cover continue headerFilePath = frameworkHeadersDir + f - content = open(headerFilePath).read() - res = re.findall('(?<=@interface )\w+', content) + with open(headerFilePath) as f: + content = f.read() + res = re.findall(r'(?<=@interface )\w+', content) for r in res: all_interfaces.add(r) - res = re.findall('(?<=@protocol )\w+', content) + res = re.findall(r'(?<=@protocol )\w+', content) for r in res: all_protocols.add(r) - res = re.findall('(?<=typedef enum )\w+', content) + res = re.findall(r'(?<=typedef enum )\w+', content) for r in res: all_primitives.add(r) - res = re.findall('(?<=typedef struct )\w+', content) + res = re.findall(r'(?<=typedef struct )\w+', content) for r in res: all_primitives.add(r) - res = re.findall('(?<=typedef const struct )\w+', content) + res = re.findall(r'(?<=typedef const struct )\w+', content) for r in res: all_primitives.add(r) diff --git a/pygments/lexers/_csound_builtins.py b/pygments/lexers/_csound_builtins.py index e5a9aaf7..16a48396 100644 --- a/pygments/lexers/_csound_builtins.py +++ b/pygments/lexers/_csound_builtins.py @@ -3,1344 +3,1658 @@ pygments.lexers._csound_builtins ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -# Opcodes in Csound 6.05 from -# csound --list-opcodes -# except -# cggoto <http://www.csounds.com/manual/html/cggoto.html> -# cigoto <http://www.csounds.com/manual/html/cigoto.html> -# cingoto (undocumented) -# ckgoto <http://www.csounds.com/manual/html/ckgoto.html> -# cngoto <http://www.csounds.com/manual/html/cngoto.html> -# endin <http://www.csounds.com/manual/html/endin.html -# endop <http://www.csounds.com/manual/html/endop.html -# goto <http://www.csounds.com/manual/html/goto.html> -# igoto <http://www.csounds.com/manual/html/igoto.html> -# instr <http://www.csounds.com/manual/html/instr.html> -# kgoto <http://www.csounds.com/manual/html/kgoto.html> -# loop_ge <http://www.csounds.com/manual/html/loop_ge.html> -# loop_gt <http://www.csounds.com/manual/html/loop_gt.html> -# loop_le <http://www.csounds.com/manual/html/loop_le.html> -# loop_lt <http://www.csounds.com/manual/html/loop_lt.html> -# opcode <http://www.csounds.com/manual/html/opcode.html> -# return <http://www.csounds.com/manual/html/return.html> -# rigoto <http://www.csounds.com/manual/html/rigoto.html> -# tigoto <http://www.csounds.com/manual/html/tigoto.html> -# timout <http://www.csounds.com/manual/html/timout.html> -# which are treated as keywords; the scoreline opcodes -# scoreline <http://www.csounds.com/manual/html/scoreline.html> -# scoreline_i <http://www.csounds.com/manual/html/scoreline_i.html> -# which allow Csound Score highlighting; the pyrun opcodes -# <http://www.csounds.com/manual/html/pyrun.html> -# pylrun -# pylruni -# pylrunt -# pyrun -# pyruni -# pyrunt -# which allow Python highlighting; and the Lua opcodes -# lua_exec <http://www.csounds.com/manual/html/lua_exec.html> -# lua_opdef <http://www.csounds.com/manual/html/lua_opdef.html> -# which allow Lua highlighting. -OPCODES = set(( - 'ATSadd', - 'ATSaddnz', - 'ATSbufread', - 'ATScross', - 'ATSinfo', - 'ATSinterpread', - 'ATSpartialtap', - 'ATSread', - 'ATSreadnz', - 'ATSsinnoi', - 'FLbox', - 'FLbutBank', - 'FLbutton', - 'FLcloseButton', - 'FLcolor', - 'FLcolor2', - 'FLcount', - 'FLexecButton', - 'FLgetsnap', - 'FLgroup', - 'FLgroupEnd', - 'FLgroup_end', - 'FLhide', - 'FLhvsBox', - 'FLhvsBoxSetValue', - 'FLjoy', - 'FLkeyIn', - 'FLknob', - 'FLlabel', - 'FLloadsnap', - 'FLmouse', - 'FLpack', - 'FLpackEnd', - 'FLpack_end', - 'FLpanel', - 'FLpanelEnd', - 'FLpanel_end', - 'FLprintk', - 'FLprintk2', - 'FLroller', - 'FLrun', - 'FLsavesnap', - 'FLscroll', - 'FLscrollEnd', - 'FLscroll_end', - 'FLsetAlign', - 'FLsetBox', - 'FLsetColor', - 'FLsetColor2', - 'FLsetFont', - 'FLsetPosition', - 'FLsetSize', - 'FLsetSnapGroup', - 'FLsetText', - 'FLsetTextColor', - 'FLsetTextSize', - 'FLsetTextType', - 'FLsetVal', - 'FLsetVal_i', - 'FLsetVali', - 'FLsetsnap', - 'FLshow', - 'FLslidBnk', - 'FLslidBnk2', - 'FLslidBnk2Set', - 'FLslidBnk2Setk', - 'FLslidBnkGetHandle', - 'FLslidBnkSet', - 'FLslidBnkSetk', - 'FLslider', - 'FLtabs', - 'FLtabsEnd', - 'FLtabs_end', - 'FLtext', - 'FLupdate', - 'FLvalue', - 'FLvkeybd', - 'FLvslidBnk', - 'FLvslidBnk2', - 'FLxyin', - 'MixerClear', - 'MixerGetLevel', - 'MixerReceive', - 'MixerSend', - 'MixerSetLevel', - 'MixerSetLevel_i', - 'OSCinit', - 'OSClisten', - 'OSCsend', - 'a', - 'abs', - 'active', - 'adsr', - 'adsyn', - 'adsynt', - 'adsynt2', - 'aftouch', - 'alpass', - 'alwayson', - 'ampdb', - 'ampdbfs', - 'ampmidi', - 'ampmidid', - 'areson', - 'aresonk', - 'array', - 'atone', - 'atonek', - 'atonex', - 'babo', - 'balance', - 'bamboo', - 'barmodel', - 'bbcutm', - 'bbcuts', - 'betarand', - 'bexprnd', - 'bformdec', - 'bformdec1', - 'bformenc', - 'bformenc1', - 'binit', - 'biquad', - 'biquada', - 'birnd', - 'bqrez', - 'buchla', - 'butbp', - 'butbr', - 'buthp', - 'butlp', - 'butterbp', - 'butterbr', - 'butterhp', - 'butterlp', - 'button', - 'buzz', - 'c2r', - 'cabasa', - 'cauchy', - 'cauchyi', - 'ceil', - 'cell', - 'cent', - 'centroid', - 'ceps', - #'cggoto', - 'chanctrl', - 'changed', - 'chani', - 'chano', - 'chebyshevpoly', - 'checkbox', - 'chn_S', - 'chn_a', - 'chn_k', - 'chnclear', - 'chnexport', - 'chnget', - 'chnmix', - 'chnparams', - 'chnset', - 'chuap', - #'cigoto', - #'cingoto', - #'ckgoto', - 'clear', - 'clfilt', - 'clip', - 'clockoff', - 'clockon', - 'cmplxprod', - #'cngoto', - 'comb', - 'combinv', - 'compilecsd', - 'compileorc', - 'compilestr', - 'compress', - 'connect', - 'control', - 'convle', - 'convolve', - 'copy2ftab', - 'copy2ttab', - 'copya2ftab', - 'copyf2array', - 'cos', - 'cosh', - 'cosinv', - 'cosseg', - 'cossegb', - 'cossegr', - 'cps2pch', - 'cpsmidi', - 'cpsmidib', - 'cpsmidinn', - 'cpsoct', - 'cpspch', - 'cpstmid', - 'cpstun', - 'cpstuni', - 'cpsxpch', - 'cpuprc', - 'cross2', - 'crossfm', - 'crossfmi', - 'crossfmpm', - 'crossfmpmi', - 'crosspm', - 'crosspmi', - 'crunch', - 'ctlchn', - 'ctrl14', - 'ctrl21', - 'ctrl7', - 'ctrlinit', - 'cuserrnd', - 'dam', - 'date', - 'dates', - 'db', - 'dbamp', - 'dbfsamp', - 'dcblock', - 'dcblock2', - 'dconv', - 'delay', - 'delay1', - 'delayk', - 'delayr', - 'delayw', - 'deltap', - 'deltap3', - 'deltapi', - 'deltapn', - 'deltapx', - 'deltapxw', - 'denorm', - 'diff', - 'diskgrain', - 'diskin', - 'diskin2', - 'dispfft', - 'display', - 'distort', - 'distort1', - 'divz', - 'doppler', - 'downsamp', - 'dripwater', - 'dumpk', - 'dumpk2', - 'dumpk3', - 'dumpk4', - 'duserrnd', - 'dust', - 'dust2', - #'endin', - #'endop', - 'envlpx', - 'envlpxr', - 'ephasor', - 'eqfil', - 'evalstr', - 'event', - 'event_i', - 'exciter', - 'exitnow', - 'exp', - 'expcurve', - 'expon', - 'exprand', - 'exprandi', - 'expseg', - 'expsega', - 'expsegb', - 'expsegba', - 'expsegr', - 'fareylen', - 'fareyleni', - 'faustaudio', - 'faustcompile', - 'faustctl', - 'faustgen', - 'fft', - 'fftinv', - 'ficlose', - 'filebit', - 'filelen', - 'filenchnls', - 'filepeak', - 'filesr', - 'filevalid', - 'fillarray', - 'filter2', - 'fin', - 'fini', - 'fink', - 'fiopen', - 'flanger', - 'flashtxt', - 'flooper', - 'flooper2', - 'floor', - 'fluidAllOut', - 'fluidCCi', - 'fluidCCk', - 'fluidControl', - 'fluidEngine', - 'fluidLoad', - 'fluidNote', - 'fluidOut', - 'fluidProgramSelect', - 'fluidSetInterpMethod', - 'fmb3', - 'fmbell', - 'fmmetal', - 'fmpercfl', - 'fmrhode', - 'fmvoice', - 'fmwurlie', - 'fof', - 'fof2', - 'fofilter', - 'fog', - 'fold', - 'follow', - 'follow2', - 'foscil', - 'foscili', - 'fout', - 'fouti', - 'foutir', - 'foutk', - 'fprintks', - 'fprints', - 'frac', - 'fractalnoise', - 'freeverb', - 'ftchnls', - 'ftconv', - 'ftcps', - 'ftfree', - 'ftgen', - 'ftgenonce', - 'ftgentmp', - 'ftlen', - 'ftload', - 'ftloadk', - 'ftlptim', - 'ftmorf', - 'ftresize', - 'ftresizei', - 'ftsave', - 'ftsavek', - 'ftsr', - 'gain', - 'gainslider', - 'gauss', - 'gaussi', - 'gausstrig', - 'gbuzz', - 'genarray', - 'genarray_i', - 'gendy', - 'gendyc', - 'gendyx', - 'getcfg', - 'getcol', - 'getrow', - 'gogobel', - #'goto', - 'grain', - 'grain2', - 'grain3', - 'granule', - 'guiro', - 'harmon', - 'harmon2', - 'harmon3', - 'harmon4', - 'hdf5read', - 'hdf5write', - 'hilbert', - 'hrtfearly', - 'hrtfer', - 'hrtfmove', - 'hrtfmove2', - 'hrtfreverb', - 'hrtfstat', - 'hsboscil', - 'hvs1', - 'hvs2', - 'hvs3', - 'i', - 'iceps', - #'igoto', - 'ihold', - 'imagecreate', - 'imagefree', - 'imagegetpixel', - 'imageload', - 'imagesave', - 'imagesetpixel', - 'imagesize', - 'in', - 'in32', - 'inch', - 'inh', - 'init', - 'initc14', - 'initc21', - 'initc7', - 'inleta', - 'inletf', - 'inletk', - 'inletkid', - 'inletv', - 'ino', - 'inq', - 'inrg', - 'ins', - 'insglobal', - 'insremot', - #'instr', - 'int', - 'integ', - 'interp', - 'invalue', - 'inx', - 'inz', - 'jitter', - 'jitter2', - 'jspline', - 'k', - #'kgoto', - 'ktableseg', - 'lenarray', - 'lentab', - 'lfo', - 'limit', - 'line', - 'linen', - 'linenr', - 'lineto', - 'linrand', - 'linseg', - 'linsegb', - 'linsegr', - 'locsend', - 'locsig', - 'log', - 'log10', - 'log2', - 'logbtwo', - 'logcurve', - #'loop_ge', - #'loop_gt', - #'loop_le', - #'loop_lt', - 'loopseg', - 'loopsegp', - 'looptseg', - 'loopxseg', - 'lorenz', - 'loscil', - 'loscil3', - 'loscilx', - 'lowpass2', - 'lowres', - 'lowresx', - 'lpf18', - 'lpform', - 'lpfreson', - 'lphasor', - 'lpinterp', - 'lposcil', - 'lposcil3', - 'lposcila', - 'lposcilsa', - 'lposcilsa2', - 'lpread', - 'lpreson', - 'lpshold', - 'lpsholdp', - 'lpslot', - #'lua_exec', - 'lua_ikopcall', - #'lua_opdef', - 'mac', - 'maca', - 'madsr', - 'mags', - 'mandel', - 'mandol', - 'maparray', - 'maparray_i', - 'marimba', - 'massign', - 'max', - 'max_k', - 'maxabs', - 'maxabsaccum', - 'maxaccum', - 'maxalloc', - 'maxarray', - 'maxtab', - 'mclock', - 'mdelay', - 'median', - 'mediank', - 'metro', - 'midglobal', - 'midic14', - 'midic21', - 'midic7', - 'midichannelaftertouch', - 'midichn', - 'midicontrolchange', - 'midictrl', - 'mididefault', - 'midifilestatus', - 'midiin', - 'midinoteoff', - 'midinoteoncps', - 'midinoteonkey', - 'midinoteonoct', - 'midinoteonpch', - 'midion', - 'midion2', - 'midiout', - 'midipgm', - 'midipitchbend', - 'midipolyaftertouch', - 'midiprogramchange', - 'miditempo', - 'midremot', - 'min', - 'minabs', - 'minabsaccum', - 'minaccum', - 'minarray', - 'mincer', - 'mintab', - 'mirror', - 'mode', - 'modmatrix', - 'monitor', - 'moog', - 'moogladder', - 'moogvcf', - 'moogvcf2', - 'moscil', - 'mp3bitrate', - 'mp3in', - 'mp3len', - 'mp3nchnls', - 'mp3sr', - 'mpulse', - 'mrtmsg', - 'multitap', - 'mute', - 'mxadsr', - 'nestedap', - 'nlalp', - 'nlfilt', - 'nlfilt2', - 'noise', - 'noteoff', - 'noteon', - 'noteondur', - 'noteondur2', - 'notnum', - 'nreverb', - 'nrpn', - 'nsamp', - 'nstance', - 'nstrnum', - 'ntrpol', - 'octave', - 'octcps', - 'octmidi', - 'octmidib', - 'octmidinn', - 'octpch', - #'opcode', - 'oscbnk', - 'oscil', - 'oscil1', - 'oscil1i', - 'oscil3', - 'oscili', - 'oscilikt', - 'osciliktp', - 'oscilikts', - 'osciln', - 'oscils', - 'oscilx', - 'out', - 'out32', - 'outc', - 'outch', - 'outh', - 'outiat', - 'outic', - 'outic14', - 'outipat', - 'outipb', - 'outipc', - 'outkat', - 'outkc', - 'outkc14', - 'outkpat', - 'outkpb', - 'outkpc', - 'outleta', - 'outletf', - 'outletk', - 'outletkid', - 'outletv', - 'outo', - 'outq', - 'outq1', - 'outq2', - 'outq3', - 'outq4', - 'outrg', - 'outs', - 'outs1', - 'outs2', - 'outvalue', - 'outx', - 'outz', - 'p', - 'pan', - 'pan2', - 'pareq', - 'partials', - 'partikkel', - 'partikkelget', - 'partikkelset', - 'partikkelsync', - 'passign', - 'pcauchy', - 'pchbend', - 'pchmidi', - 'pchmidib', - 'pchmidinn', - 'pchoct', - 'pconvolve', - 'pcount', - 'pdclip', - 'pdhalf', - 'pdhalfy', - 'peak', - 'pgmassign', - 'pgmchn', - 'phaser1', - 'phaser2', - 'phasor', - 'phasorbnk', - 'phs', - 'pindex', - 'pinker', - 'pinkish', - 'pitch', - 'pitchac', - 'pitchamdf', - 'planet', - 'platerev', - 'plltrack', - 'pluck', - 'poisson', - 'pol2rect', - 'polyaft', - 'polynomial', - 'pop', - 'pop_f', - 'port', - 'portk', - 'poscil', - 'poscil3', - 'pow', - 'powershape', - 'powoftwo', - 'prealloc', - 'prepiano', - 'print', - 'print_type', - 'printf', - 'printf_i', - 'printk', - 'printk2', - 'printks', - 'printks2', - 'prints', - 'product', - 'pset', - 'ptable', - 'ptable3', - 'ptablei', - 'ptableiw', - 'ptablew', - 'ptrack', - 'push', - 'push_f', - 'puts', - 'pvadd', - 'pvbufread', - 'pvcross', - 'pvinterp', - 'pvoc', - 'pvread', - 'pvs2array', - 'pvs2tab', - 'pvsadsyn', - 'pvsanal', - 'pvsarp', - 'pvsbandp', - 'pvsbandr', - 'pvsbin', - 'pvsblur', - 'pvsbuffer', - 'pvsbufread', - 'pvsbufread2', - 'pvscale', - 'pvscent', - 'pvsceps', - 'pvscross', - 'pvsdemix', - 'pvsdiskin', - 'pvsdisp', - 'pvsenvftw', - 'pvsfilter', - 'pvsfread', - 'pvsfreeze', - 'pvsfromarray', - 'pvsftr', - 'pvsftw', - 'pvsfwrite', - 'pvsgain', - 'pvsgendy', - 'pvshift', - 'pvsifd', - 'pvsin', - 'pvsinfo', - 'pvsinit', - 'pvslock', - 'pvsmaska', - 'pvsmix', - 'pvsmooth', - 'pvsmorph', - 'pvsosc', - 'pvsout', - 'pvspitch', - 'pvstanal', - 'pvstencil', - 'pvsvoc', - 'pvswarp', - 'pvsynth', - 'pwd', - 'pyassign', - 'pyassigni', - 'pyassignt', - 'pycall', - 'pycall1', - 'pycall1i', - 'pycall1t', - 'pycall2', - 'pycall2i', - 'pycall2t', - 'pycall3', - 'pycall3i', - 'pycall3t', - 'pycall4', - 'pycall4i', - 'pycall4t', - 'pycall5', - 'pycall5i', - 'pycall5t', - 'pycall6', - 'pycall6i', - 'pycall6t', - 'pycall7', - 'pycall7i', - 'pycall7t', - 'pycall8', - 'pycall8i', - 'pycall8t', - 'pycalli', - 'pycalln', - 'pycallni', - 'pycallt', - 'pyeval', - 'pyevali', - 'pyevalt', - 'pyexec', - 'pyexeci', - 'pyexect', - 'pyinit', - 'pylassign', - 'pylassigni', - 'pylassignt', - 'pylcall', - 'pylcall1', - 'pylcall1i', - 'pylcall1t', - 'pylcall2', - 'pylcall2i', - 'pylcall2t', - 'pylcall3', - 'pylcall3i', - 'pylcall3t', - 'pylcall4', - 'pylcall4i', - 'pylcall4t', - 'pylcall5', - 'pylcall5i', - 'pylcall5t', - 'pylcall6', - 'pylcall6i', - 'pylcall6t', - 'pylcall7', - 'pylcall7i', - 'pylcall7t', - 'pylcall8', - 'pylcall8i', - 'pylcall8t', - 'pylcalli', - 'pylcalln', - 'pylcallni', - 'pylcallt', - 'pyleval', - 'pylevali', - 'pylevalt', - 'pylexec', - 'pylexeci', - 'pylexect', - #'pylrun', - #'pylruni', - #'pylrunt', - #'pyrun', - #'pyruni', - #'pyrunt', - 'qinf', - 'qnan', - 'r2c', - 'rand', - 'randh', - 'randi', - 'random', - 'randomh', - 'randomi', - 'rbjeq', - 'readclock', - 'readf', - 'readfi', - 'readk', - 'readk2', - 'readk3', - 'readk4', - 'readks', - 'readscore', - 'readscratch', - 'rect2pol', - 'reinit', - 'release', - 'remoteport', - 'remove', - 'repluck', - 'reson', - 'resonk', - 'resonr', - 'resonx', - 'resonxk', - 'resony', - 'resonz', - 'resyn', - #'return', - 'reverb', - 'reverb2', - 'reverbsc', - 'rewindscore', - 'rezzy', - 'rfft', - 'rifft', - #'rigoto', - 'rireturn', - 'rms', - 'rnd', - 'rnd31', - 'round', - 'rspline', - 'rtclock', - 's16b14', - 's32b14', - 'samphold', - 'sandpaper', - 'scale', - 'scalearray', - 'scalet', - 'scanhammer', - 'scans', - 'scantable', - 'scanu', - 'schedkwhen', - 'schedkwhennamed', - 'schedule', - 'schedwhen', - #'scoreline', - #'scoreline_i', - 'seed', - 'sekere', - 'semitone', - 'sense', - 'sensekey', - 'seqtime', - 'seqtime2', - 'serialBegin', - 'serialEnd', - 'serialFlush', - 'serialPrint', - 'serialRead', - 'serialWrite', - 'serialWrite_i', - 'setcol', - 'setctrl', - 'setksmps', - 'setrow', - 'setscorepos', - 'sfilist', - 'sfinstr', - 'sfinstr3', - 'sfinstr3m', - 'sfinstrm', - 'sfload', - 'sflooper', - 'sfpassign', - 'sfplay', - 'sfplay3', - 'sfplay3m', - 'sfplaym', - 'sfplist', - 'sfpreset', - 'shaker', - 'shiftin', - 'shiftout', - 'signalflowgraph', - 'signum', - 'sin', - 'sinh', - 'sininv', - 'sinsyn', - 'sleighbells', - 'slicearray', - 'slider16', - 'slider16f', - 'slider16table', - 'slider16tablef', - 'slider32', - 'slider32f', - 'slider32table', - 'slider32tablef', - 'slider64', - 'slider64f', - 'slider64table', - 'slider64tablef', - 'slider8', - 'slider8f', - 'slider8table', - 'slider8tablef', - 'sliderKawai', - 'sndload', - 'sndloop', - 'sndwarp', - 'sndwarpst', - 'sockrecv', - 'sockrecvs', - 'socksend', - 'socksends', - 'soundin', - 'soundout', - 'soundouts', - 'space', - 'spat3d', - 'spat3di', - 'spat3dt', - 'spdist', - 'specaddm', - 'specdiff', - 'specdisp', - 'specfilt', - 'spechist', - 'specptrk', - 'specscal', - 'specsum', - 'spectrum', - 'splitrig', - 'sprintf', - 'sprintfk', - 'spsend', - 'sqrt', - 'stack', - 'statevar', - 'stix', - 'strcat', - 'strcatk', - 'strchar', - 'strchark', - 'strcmp', - 'strcmpk', - 'strcpy', - 'strcpyk', - 'strecv', - 'streson', - 'strfromurl', - 'strget', - 'strindex', - 'strindexk', - 'strlen', - 'strlenk', - 'strlower', - 'strlowerk', - 'strrindex', - 'strrindexk', - 'strset', - 'strsub', - 'strsubk', - 'strtod', - 'strtodk', - 'strtol', - 'strtolk', - 'strupper', - 'strupperk', - 'stsend', - 'subinstr', - 'subinstrinit', - 'sum', - 'sumarray', - 'sumtab', - 'svfilter', - 'syncgrain', - 'syncloop', - 'syncphasor', - 'system', - 'system_i', - 'tab', - 'tab2pvs', - 'tab_i', - 'tabgen', - 'table', - 'table3', - 'table3kt', - 'tablecopy', - 'tablefilter', - 'tablefilteri', - 'tablegpw', - 'tablei', - 'tableicopy', - 'tableigpw', - 'tableikt', - 'tableimix', - 'tableiw', - 'tablekt', - 'tablemix', - 'tableng', - 'tablera', - 'tableseg', - 'tableshuffle', - 'tableshufflei', - 'tablew', - 'tablewa', - 'tablewkt', - 'tablexkt', - 'tablexseg', - 'tabmap', - 'tabmap_i', - 'tabmorph', - 'tabmorpha', - 'tabmorphak', - 'tabmorphi', - 'tabplay', - 'tabrec', - 'tabslice', - 'tabsum', - 'tabw', - 'tabw_i', - 'tambourine', - 'tan', - 'tanh', - 'taninv', - 'taninv2', - 'tb0', - 'tb0_init', - 'tb1', - 'tb10', - 'tb10_init', - 'tb11', - 'tb11_init', - 'tb12', - 'tb12_init', - 'tb13', - 'tb13_init', - 'tb14', - 'tb14_init', - 'tb15', - 'tb15_init', - 'tb1_init', - 'tb2', - 'tb2_init', - 'tb3', - 'tb3_init', - 'tb4', - 'tb4_init', - 'tb5', - 'tb5_init', - 'tb6', - 'tb6_init', - 'tb7', - 'tb7_init', - 'tb8', - 'tb8_init', - 'tb9', - 'tb9_init', - 'tbvcf', - 'tempest', - 'tempo', - 'temposcal', - 'tempoval', - #'tigoto', - 'timedseq', - 'timeinstk', - 'timeinsts', - 'timek', - 'times', - #'timout', - 'tival', - 'tlineto', - 'tone', - 'tonek', - 'tonex', - 'tradsyn', - 'trandom', - 'transeg', - 'transegb', - 'transegr', - 'trcross', - 'trfilter', - 'trhighest', - 'trigger', - 'trigseq', - 'trirand', - 'trlowest', - 'trmix', - 'trscale', - 'trshift', - 'trsplit', - 'turnoff', - 'turnoff2', - 'turnon', - 'unirand', - 'unwrap', - 'upsamp', - 'urd', - 'vactrol', - 'vadd', - 'vadd_i', - 'vaddv', - 'vaddv_i', - 'vaget', - 'valpass', - 'vaset', - 'vbap', - 'vbap16', - 'vbap4', - 'vbap4move', - 'vbap8', - 'vbap8move', - 'vbapg', - 'vbapgmove', - 'vbaplsinit', - 'vbapmove', - 'vbapz', - 'vbapzmove', - 'vcella', - 'vco', - 'vco2', - 'vco2ft', - 'vco2ift', - 'vco2init', - 'vcomb', - 'vcopy', - 'vcopy_i', - 'vdel_k', - 'vdelay', - 'vdelay3', - 'vdelayk', - 'vdelayx', - 'vdelayxq', - 'vdelayxs', - 'vdelayxw', - 'vdelayxwq', - 'vdelayxws', - 'vdivv', - 'vdivv_i', - 'vecdelay', - 'veloc', - 'vexp', - 'vexp_i', - 'vexpseg', - 'vexpv', - 'vexpv_i', - 'vibes', - 'vibr', - 'vibrato', - 'vincr', - 'vlimit', - 'vlinseg', - 'vlowres', - 'vmap', - 'vmirror', - 'vmult', - 'vmult_i', - 'vmultv', - 'vmultv_i', - 'voice', - 'vosim', - 'vphaseseg', - 'vport', - 'vpow', - 'vpow_i', - 'vpowv', - 'vpowv_i', - 'vpvoc', - 'vrandh', - 'vrandi', - 'vsubv', - 'vsubv_i', - 'vtaba', - 'vtabi', - 'vtabk', - 'vtable1k', - 'vtablea', - 'vtablei', - 'vtablek', - 'vtablewa', - 'vtablewi', - 'vtablewk', - 'vtabwa', - 'vtabwi', - 'vtabwk', - 'vwrap', - 'waveset', - 'weibull', - 'wgbow', - 'wgbowedbar', - 'wgbrass', - 'wgclar', - 'wgflute', - 'wgpluck', - 'wgpluck2', - 'wguide1', - 'wguide2', - 'wiiconnect', - 'wiidata', - 'wiirange', - 'wiisend', - 'window', - 'wrap', - 'writescratch', - 'wterrain', - 'xadsr', - 'xin', - 'xout', - 'xscanmap', - 'xscans', - 'xscansmap', - 'xscanu', - 'xtratim', - 'xyin', - 'zacl', - 'zakinit', - 'zamod', - 'zar', - 'zarg', - 'zaw', - 'zawm', - 'zfilter2', - 'zir', - 'ziw', - 'ziwm', - 'zkcl', - 'zkmod', - 'zkr', - 'zkw', - 'zkwm' -)) +# Opcodes in Csound 6.12.0 at commit 6ca322bd31f1ca907c008616b40a5f237ff449db using +# python -c " +# import re, subprocess +# output = subprocess.Popen(['csound', '--list-opcodes0'], stderr=subprocess.PIPE).communicate()[1] +# opcodes = output[re.search(r'^$', output, re.M).end():re.search(r'^\d+ opcodes$', output, re.M).start()].split() +# output = subprocess.Popen(['csound', '--list-opcodes2'], stderr=subprocess.PIPE).communicate()[1] +# all_opcodes = output[re.search(r'^$', output, re.M).end():re.search(r'^\d+ opcodes$', output, re.M).start()].split() +# deprecated_opcodes = [opcode for opcode in all_opcodes if opcode not in opcodes] +# print '''OPCODES = set(\''' +# {} +# \'''.split()) +# +# DEPRECATED_OPCODES = set(\''' +# {} +# \'''.split()) +# '''.format('\n'.join(opcodes), '\n'.join(deprecated_opcodes)) +# " +# except for +# cggoto csound.com/docs/manual/cggoto.html +# cigoto csound.com/docs/manual/cigoto.html +# cingoto (undocumented) +# ckgoto csound.com/docs/manual/ckgoto.html +# cngoto csound.com/docs/manual/cngoto.html +# cnkgoto (undocumented) +# endin csound.com/docs/manual/endin.html +# endop csound.com/docs/manual/endop.html +# goto csound.com/docs/manual/goto.html +# igoto csound.com/docs/manual/igoto.html +# instr csound.com/docs/manual/instr.html +# kgoto csound.com/docs/manual/kgoto.html +# loop_ge csound.com/docs/manual/loop_ge.html +# loop_gt csound.com/docs/manual/loop_gt.html +# loop_le csound.com/docs/manual/loop_le.html +# loop_lt csound.com/docs/manual/loop_lt.html +# opcode csound.com/docs/manual/opcode.html +# reinit csound.com/docs/manual/reinit.html +# return csound.com/docs/manual/return.html +# rireturn csound.com/docs/manual/rireturn.html +# rigoto csound.com/docs/manual/rigoto.html +# tigoto csound.com/docs/manual/tigoto.html +# timout csound.com/docs/manual/timout.html +# which are treated as keywords in csound.py. + +OPCODES = set(''' +ATSadd +ATSaddnz +ATSbufread +ATScross +ATSinfo +ATSinterpread +ATSpartialtap +ATSread +ATSreadnz +ATSsinnoi +FLbox +FLbutBank +FLbutton +FLcloseButton +FLcolor +FLcolor2 +FLcount +FLexecButton +FLgetsnap +FLgroup +FLgroupEnd +FLgroup_end +FLhide +FLhvsBox +FLhvsBoxSetValue +FLjoy +FLkeyIn +FLknob +FLlabel +FLloadsnap +FLmouse +FLpack +FLpackEnd +FLpack_end +FLpanel +FLpanelEnd +FLpanel_end +FLprintk +FLprintk2 +FLroller +FLrun +FLsavesnap +FLscroll +FLscrollEnd +FLscroll_end +FLsetAlign +FLsetBox +FLsetColor +FLsetColor2 +FLsetFont +FLsetPosition +FLsetSize +FLsetSnapGroup +FLsetText +FLsetTextColor +FLsetTextSize +FLsetTextType +FLsetVal +FLsetVal_i +FLsetVali +FLsetsnap +FLshow +FLslidBnk +FLslidBnk2 +FLslidBnk2Set +FLslidBnk2Setk +FLslidBnkGetHandle +FLslidBnkSet +FLslidBnkSetk +FLslider +FLtabs +FLtabsEnd +FLtabs_end +FLtext +FLupdate +FLvalue +FLvkeybd +FLvslidBnk +FLvslidBnk2 +FLxyin +JackoAudioIn +JackoAudioInConnect +JackoAudioOut +JackoAudioOutConnect +JackoFreewheel +JackoInfo +JackoInit +JackoMidiInConnect +JackoMidiOut +JackoMidiOutConnect +JackoNoteOut +JackoOn +JackoTransport +K35_hpf +K35_lpf +MixerClear +MixerGetLevel +MixerReceive +MixerSend +MixerSetLevel +MixerSetLevel_i +OSCbundle +OSCcount +OSCinit +OSCinitM +OSClisten +OSCraw +OSCsend +OSCsend_lo +S +STKBandedWG +STKBeeThree +STKBlowBotl +STKBlowHole +STKBowed +STKBrass +STKClarinet +STKDrummer +STKFlute +STKFMVoices +STKHevyMetl +STKMandolin +STKModalBar +STKMoog +STKPercFlut +STKPlucked +STKResonate +STKRhodey +STKSaxofony +STKShakers +STKSimple +STKSitar +STKStifKarp +STKTubeBell +STKVoicForm +STKWhistle +STKWurley +a +abs +active +adsr +adsyn +adsynt +adsynt2 +aftouch +alpass +alwayson +ampdb +ampdbfs +ampmidi +ampmidid +areson +aresonk +atone +atonek +atonex +babo +balance +balance2 +bamboo +barmodel +bbcutm +bbcuts +beadsynt +beosc +betarand +bexprnd +bformdec1 +bformenc1 +binit +biquad +biquada +birnd +bpf +bpfcos +bqrez +butbp +butbr +buthp +butlp +butterbp +butterbr +butterhp +butterlp +button +buzz +c2r +cabasa +cauchy +cauchyi +cbrt +ceil +cell +cent +centroid +ceps +cepsinv +chanctrl +changed +changed2 +chani +chano +chebyshevpoly +checkbox +chn_S +chn_a +chn_k +chnclear +chnexport +chnget +chngetks +chnmix +chnparams +chnset +chnsetks +chuap +clear +clfilt +clip +clockoff +clockon +cmp +cmplxprod +comb +combinv +compilecsd +compileorc +compilestr +compress +compress2 +connect +control +convle +convolve +copya2ftab +copyf2array +cos +cosh +cosinv +cosseg +cossegb +cossegr +cps2pch +cpsmidi +cpsmidib +cpsmidinn +cpsoct +cpspch +cpstmid +cpstun +cpstuni +cpsxpch +cpumeter +cpuprc +cross2 +crossfm +crossfmi +crossfmpm +crossfmpmi +crosspm +crosspmi +crunch +ctlchn +ctrl14 +ctrl21 +ctrl7 +ctrlinit +cuserrnd +dam +date +dates +db +dbamp +dbfsamp +dcblock +dcblock2 +dconv +dct +dctinv +deinterleave +delay +delay1 +delayk +delayr +delayw +deltap +deltap3 +deltapi +deltapn +deltapx +deltapxw +denorm +diff +diode_ladder +directory +diskgrain +diskin +diskin2 +dispfft +display +distort +distort1 +divz +doppler +dot +downsamp +dripwater +dssiactivate +dssiaudio +dssictls +dssiinit +dssilist +dumpk +dumpk2 +dumpk3 +dumpk4 +duserrnd +dust +dust2 +envlpx +envlpxr +ephasor +eqfil +evalstr +event +event_i +exciter +exitnow +exp +expcurve +expon +exprand +exprandi +expseg +expsega +expsegb +expsegba +expsegr +fareylen +fareyleni +faustaudio +faustcompile +faustctl +faustdsp +faustgen +faustplay +fft +fftinv +ficlose +filebit +filelen +filenchnls +filepeak +filescal +filesr +filevalid +fillarray +filter2 +fin +fini +fink +fiopen +flanger +flashtxt +flooper +flooper2 +floor +fmanal +fmax +fmb3 +fmbell +fmin +fmmetal +fmod +fmpercfl +fmrhode +fmvoice +fmwurlie +fof +fof2 +fofilter +fog +fold +follow +follow2 +foscil +foscili +fout +fouti +foutir +foutk +fprintks +fprints +frac +fractalnoise +framebuffer +freeverb +ftaudio +ftchnls +ftconv +ftcps +ftfree +ftgen +ftgenonce +ftgentmp +ftlen +ftload +ftloadk +ftlptim +ftmorf +ftom +ftprint +ftresize +ftresizei +ftsamplebank +ftsave +ftsavek +ftslice +ftsr +gain +gainslider +gauss +gaussi +gausstrig +gbuzz +genarray +genarray_i +gendy +gendyc +gendyx +getcfg +getcol +getftargs +getrow +getrowlin +getseed +gogobel +grain +grain2 +grain3 +granule +guiro +harmon +harmon2 +harmon3 +harmon4 +hdf5read +hdf5write +hilbert +hilbert2 +hrtfearly +hrtfmove +hrtfmove2 +hrtfreverb +hrtfstat +hsboscil +hvs1 +hvs2 +hvs3 +hypot +i +ihold +imagecreate +imagefree +imagegetpixel +imageload +imagesave +imagesetpixel +imagesize +in +in32 +inch +inh +init +initc14 +initc21 +initc7 +inleta +inletf +inletk +inletkid +inletv +ino +inq +inrg +ins +insglobal +insremot +int +integ +interleave +interp +invalue +inx +inz +jacktransport +jitter +jitter2 +joystick +jspline +k +la_i_add_mc +la_i_add_mr +la_i_add_vc +la_i_add_vr +la_i_assign_mc +la_i_assign_mr +la_i_assign_t +la_i_assign_vc +la_i_assign_vr +la_i_conjugate_mc +la_i_conjugate_mr +la_i_conjugate_vc +la_i_conjugate_vr +la_i_distance_vc +la_i_distance_vr +la_i_divide_mc +la_i_divide_mr +la_i_divide_vc +la_i_divide_vr +la_i_dot_mc +la_i_dot_mc_vc +la_i_dot_mr +la_i_dot_mr_vr +la_i_dot_vc +la_i_dot_vr +la_i_get_mc +la_i_get_mr +la_i_get_vc +la_i_get_vr +la_i_invert_mc +la_i_invert_mr +la_i_lower_solve_mc +la_i_lower_solve_mr +la_i_lu_det_mc +la_i_lu_det_mr +la_i_lu_factor_mc +la_i_lu_factor_mr +la_i_lu_solve_mc +la_i_lu_solve_mr +la_i_mc_create +la_i_mc_set +la_i_mr_create +la_i_mr_set +la_i_multiply_mc +la_i_multiply_mr +la_i_multiply_vc +la_i_multiply_vr +la_i_norm_euclid_mc +la_i_norm_euclid_mr +la_i_norm_euclid_vc +la_i_norm_euclid_vr +la_i_norm_inf_mc +la_i_norm_inf_mr +la_i_norm_inf_vc +la_i_norm_inf_vr +la_i_norm_max_mc +la_i_norm_max_mr +la_i_norm1_mc +la_i_norm1_mr +la_i_norm1_vc +la_i_norm1_vr +la_i_print_mc +la_i_print_mr +la_i_print_vc +la_i_print_vr +la_i_qr_eigen_mc +la_i_qr_eigen_mr +la_i_qr_factor_mc +la_i_qr_factor_mr +la_i_qr_sym_eigen_mc +la_i_qr_sym_eigen_mr +la_i_random_mc +la_i_random_mr +la_i_random_vc +la_i_random_vr +la_i_size_mc +la_i_size_mr +la_i_size_vc +la_i_size_vr +la_i_subtract_mc +la_i_subtract_mr +la_i_subtract_vc +la_i_subtract_vr +la_i_t_assign +la_i_trace_mc +la_i_trace_mr +la_i_transpose_mc +la_i_transpose_mr +la_i_upper_solve_mc +la_i_upper_solve_mr +la_i_vc_create +la_i_vc_set +la_i_vr_create +la_i_vr_set +la_k_a_assign +la_k_add_mc +la_k_add_mr +la_k_add_vc +la_k_add_vr +la_k_assign_a +la_k_assign_f +la_k_assign_mc +la_k_assign_mr +la_k_assign_t +la_k_assign_vc +la_k_assign_vr +la_k_conjugate_mc +la_k_conjugate_mr +la_k_conjugate_vc +la_k_conjugate_vr +la_k_current_f +la_k_current_vr +la_k_distance_vc +la_k_distance_vr +la_k_divide_mc +la_k_divide_mr +la_k_divide_vc +la_k_divide_vr +la_k_dot_mc +la_k_dot_mc_vc +la_k_dot_mr +la_k_dot_mr_vr +la_k_dot_vc +la_k_dot_vr +la_k_f_assign +la_k_get_mc +la_k_get_mr +la_k_get_vc +la_k_get_vr +la_k_invert_mc +la_k_invert_mr +la_k_lower_solve_mc +la_k_lower_solve_mr +la_k_lu_det_mc +la_k_lu_det_mr +la_k_lu_factor_mc +la_k_lu_factor_mr +la_k_lu_solve_mc +la_k_lu_solve_mr +la_k_mc_set +la_k_mr_set +la_k_multiply_mc +la_k_multiply_mr +la_k_multiply_vc +la_k_multiply_vr +la_k_norm_euclid_mc +la_k_norm_euclid_mr +la_k_norm_euclid_vc +la_k_norm_euclid_vr +la_k_norm_inf_mc +la_k_norm_inf_mr +la_k_norm_inf_vc +la_k_norm_inf_vr +la_k_norm_max_mc +la_k_norm_max_mr +la_k_norm1_mc +la_k_norm1_mr +la_k_norm1_vc +la_k_norm1_vr +la_k_qr_eigen_mc +la_k_qr_eigen_mr +la_k_qr_factor_mc +la_k_qr_factor_mr +la_k_qr_sym_eigen_mc +la_k_qr_sym_eigen_mr +la_k_random_mc +la_k_random_mr +la_k_random_vc +la_k_random_vr +la_k_subtract_mc +la_k_subtract_mr +la_k_subtract_vc +la_k_subtract_vr +la_k_t_assign +la_k_trace_mc +la_k_trace_mr +la_k_upper_solve_mc +la_k_upper_solve_mr +la_k_vc_set +la_k_vr_set +lenarray +lfo +limit +limit1 +lincos +line +linen +linenr +lineto +link_beat_force +link_beat_get +link_beat_request +link_create +link_enable +link_is_enabled +link_metro +link_peers +link_tempo_get +link_tempo_set +linlin +linrand +linseg +linsegb +linsegr +liveconv +locsend +locsig +log +log10 +log2 +logbtwo +logcurve +loopseg +loopsegp +looptseg +loopxseg +lorenz +loscil +loscil3 +loscil3phs +loscilphs +loscilx +lowpass2 +lowres +lowresx +lpf18 +lpform +lpfreson +lphasor +lpinterp +lposcil +lposcil3 +lposcila +lposcilsa +lposcilsa2 +lpread +lpreson +lpshold +lpsholdp +lpslot +lua_exec +lua_iaopcall +lua_iaopcall_off +lua_ikopcall +lua_ikopcall_off +lua_iopcall +lua_iopcall_off +lua_opdef +mac +maca +madsr +mags +mandel +mandol +maparray +maparray_i +marimba +massign +max +max_k +maxabs +maxabsaccum +maxaccum +maxalloc +maxarray +mclock +mdelay +median +mediank +metro +mfb +midglobal +midiarp +midic14 +midic21 +midic7 +midichannelaftertouch +midichn +midicontrolchange +midictrl +mididefault +midifilestatus +midiin +midinoteoff +midinoteoncps +midinoteonkey +midinoteonoct +midinoteonpch +midion +midion2 +midiout +midiout_i +midipgm +midipitchbend +midipolyaftertouch +midiprogramchange +miditempo +midremot +min +minabs +minabsaccum +minaccum +minarray +mincer +mirror +mode +modmatrix +monitor +moog +moogladder +moogladder2 +moogvcf +moogvcf2 +moscil +mp3bitrate +mp3in +mp3len +mp3nchnls +mp3scal +mp3sr +mpulse +mrtmsg +mtof +mton +multitap +mute +mvchpf +mvclpf1 +mvclpf2 +mvclpf3 +mvclpf4 +mxadsr +nchnls_hw +nestedap +nlalp +nlfilt +nlfilt2 +noise +noteoff +noteon +noteondur +noteondur2 +notnum +nreverb +nrpn +nsamp +nstance +nstrnum +ntom +ntrpol +nxtpow2 +octave +octcps +octmidi +octmidib +octmidinn +octpch +olabuffer +oscbnk +oscil +oscil1 +oscil1i +oscil3 +oscili +oscilikt +osciliktp +oscilikts +osciln +oscils +oscilx +out +out32 +outc +outch +outh +outiat +outic +outic14 +outipat +outipb +outipc +outkat +outkc +outkc14 +outkpat +outkpb +outkpc +outleta +outletf +outletk +outletkid +outletv +outo +outq +outq1 +outq2 +outq3 +outq4 +outrg +outs +outs1 +outs2 +outvalue +outx +outz +p +p5gconnect +p5gdata +pan +pan2 +pareq +part2txt +partials +partikkel +partikkelget +partikkelset +partikkelsync +passign +paulstretch +pcauchy +pchbend +pchmidi +pchmidib +pchmidinn +pchoct +pchtom +pconvolve +pcount +pdclip +pdhalf +pdhalfy +peak +pgmassign +pgmchn +phaser1 +phaser2 +phasor +phasorbnk +phs +pindex +pinker +pinkish +pitch +pitchac +pitchamdf +planet +platerev +plltrack +pluck +poisson +pol2rect +polyaft +polynomial +port +portk +poscil +poscil3 +pow +powershape +powoftwo +pows +prealloc +prepiano +print +print_type +printarray +printf +printf_i +printk +printk2 +printks +printks2 +prints +product +pset +ptable +ptable3 +ptablei +ptableiw +ptablew +ptrack +puts +pvadd +pvbufread +pvcross +pvinterp +pvoc +pvread +pvs2array +pvs2tab +pvsadsyn +pvsanal +pvsarp +pvsbandp +pvsbandr +pvsbin +pvsblur +pvsbuffer +pvsbufread +pvsbufread2 +pvscale +pvscent +pvsceps +pvscross +pvsdemix +pvsdiskin +pvsdisp +pvsenvftw +pvsfilter +pvsfread +pvsfreeze +pvsfromarray +pvsftr +pvsftw +pvsfwrite +pvsgain +pvshift +pvsifd +pvsin +pvsinfo +pvsinit +pvslock +pvsmaska +pvsmix +pvsmooth +pvsmorph +pvsosc +pvsout +pvspitch +pvstanal +pvstencil +pvstrace +pvsvoc +pvswarp +pvsynth +pwd +pyassign +pyassigni +pyassignt +pycall +pycall1 +pycall1i +pycall1t +pycall2 +pycall2i +pycall2t +pycall3 +pycall3i +pycall3t +pycall4 +pycall4i +pycall4t +pycall5 +pycall5i +pycall5t +pycall6 +pycall6i +pycall6t +pycall7 +pycall7i +pycall7t +pycall8 +pycall8i +pycall8t +pycalli +pycalln +pycallni +pycallt +pyeval +pyevali +pyevalt +pyexec +pyexeci +pyexect +pyinit +pylassign +pylassigni +pylassignt +pylcall +pylcall1 +pylcall1i +pylcall1t +pylcall2 +pylcall2i +pylcall2t +pylcall3 +pylcall3i +pylcall3t +pylcall4 +pylcall4i +pylcall4t +pylcall5 +pylcall5i +pylcall5t +pylcall6 +pylcall6i +pylcall6t +pylcall7 +pylcall7i +pylcall7t +pylcall8 +pylcall8i +pylcall8t +pylcalli +pylcalln +pylcallni +pylcallt +pyleval +pylevali +pylevalt +pylexec +pylexeci +pylexect +pylrun +pylruni +pylrunt +pyrun +pyruni +pyrunt +qinf +qnan +r2c +rand +randh +randi +random +randomh +randomi +rbjeq +readclock +readf +readfi +readk +readk2 +readk3 +readk4 +readks +readscore +readscratch +rect2pol +release +remoteport +remove +repluck +reshapearray +reson +resonk +resonr +resonx +resonxk +resony +resonz +resyn +reverb +reverb2 +reverbsc +rewindscore +rezzy +rfft +rifft +rms +rnd +rnd31 +round +rspline +rtclock +s16b14 +s32b14 +samphold +sandpaper +sc_lag +sc_lagud +sc_phasor +sc_trig +scale +scalearray +scanhammer +scans +scantable +scanu +schedkwhen +schedkwhennamed +schedule +schedwhen +scoreline +scoreline_i +seed +sekere +select +semitone +sense +sensekey +seqtime +seqtime2 +serialBegin +serialEnd +serialFlush +serialPrint +serialRead +serialWrite +serialWrite_i +setcol +setctrl +setksmps +setrow +setscorepos +sfilist +sfinstr +sfinstr3 +sfinstr3m +sfinstrm +sfload +sflooper +sfpassign +sfplay +sfplay3 +sfplay3m +sfplaym +sfplist +sfpreset +shaker +shiftin +shiftout +signum +sin +sinh +sininv +sinsyn +sleighbells +slicearray +slicearray_i +slider16 +slider16f +slider16table +slider16tablef +slider32 +slider32f +slider32table +slider32tablef +slider64 +slider64f +slider64table +slider64tablef +slider8 +slider8f +slider8table +slider8tablef +sliderKawai +sndloop +sndwarp +sndwarpst +sockrecv +sockrecvs +socksend +socksends +sorta +sortd +soundin +space +spat3d +spat3di +spat3dt +spdist +splitrig +sprintf +sprintfk +spsend +sqrt +squinewave +statevar +stix +strcat +strcatk +strchar +strchark +strcmp +strcmpk +strcpy +strcpyk +strecv +streson +strfromurl +strget +strindex +strindexk +strlen +strlenk +strlower +strlowerk +strrindex +strrindexk +strset +strsub +strsubk +strtod +strtodk +strtol +strtolk +strupper +strupperk +stsend +subinstr +subinstrinit +sum +sumarray +svfilter +syncgrain +syncloop +syncphasor +system +system_i +tab +tab2array +tab2pvs +tab_i +tabifd +table +table3 +table3kt +tablecopy +tablefilter +tablefilteri +tablegpw +tablei +tableicopy +tableigpw +tableikt +tableimix +tableiw +tablekt +tablemix +tableng +tablera +tableseg +tableshuffle +tableshufflei +tablew +tablewa +tablewkt +tablexkt +tablexseg +tabmorph +tabmorpha +tabmorphak +tabmorphi +tabplay +tabrec +tabrowlin +tabsum +tabw +tabw_i +tambourine +tan +tanh +taninv +taninv2 +tbvcf +tempest +tempo +temposcal +tempoval +timedseq +timeinstk +timeinsts +timek +times +tival +tlineto +tone +tonek +tonex +tradsyn +trandom +transeg +transegb +transegr +trcross +trfilter +trhighest +trigger +trigseq +trim +trim_i +trirand +trlowest +trmix +trscale +trshift +trsplit +turnoff +turnoff2 +turnon +tvconv +unirand +unwrap +upsamp +urandom +urd +vactrol +vadd +vadd_i +vaddv +vaddv_i +vaget +valpass +vaset +vbap +vbapg +vbapgmove +vbaplsinit +vbapmove +vbapz +vbapzmove +vcella +vco +vco2 +vco2ft +vco2ift +vco2init +vcomb +vcopy +vcopy_i +vdel_k +vdelay +vdelay3 +vdelayk +vdelayx +vdelayxq +vdelayxs +vdelayxw +vdelayxwq +vdelayxws +vdivv +vdivv_i +vecdelay +veloc +vexp +vexp_i +vexpseg +vexpv +vexpv_i +vibes +vibr +vibrato +vincr +vlimit +vlinseg +vlowres +vmap +vmirror +vmult +vmult_i +vmultv +vmultv_i +voice +vosim +vphaseseg +vport +vpow +vpow_i +vpowv +vpowv_i +vpvoc +vrandh +vrandi +vsubv +vsubv_i +vtaba +vtabi +vtabk +vtable1k +vtablea +vtablei +vtablek +vtablewa +vtablewi +vtablewk +vtabwa +vtabwi +vtabwk +vwrap +waveset +websocket +weibull +wgbow +wgbowedbar +wgbrass +wgclar +wgflute +wgpluck +wgpluck2 +wguide1 +wguide2 +wiiconnect +wiidata +wiirange +wiisend +window +wrap +writescratch +wterrain +xadsr +xin +xout +xscanmap +xscans +xscansmap +xscanu +xtratim +xyscale +zacl +zakinit +zamod +zar +zarg +zaw +zawm +zdf_1pole +zdf_1pole_mode +zdf_2pole +zdf_2pole_mode +zdf_ladder +zfilter2 +zir +ziw +ziwm +zkcl +zkmod +zkr +zkw +zkwm +'''.split()) + +DEPRECATED_OPCODES = set(''' +array +bformdec +bformenc +copy2ftab +copy2ttab +hrtfer +ktableseg +lentab +maxtab +mintab +pop +pop_f +push +push_f +scalet +sndload +soundout +soundouts +specaddm +specdiff +specdisp +specfilt +spechist +specptrk +specscal +specsum +spectrum +stack +sumtab +tabgen +tabmap +tabmap_i +tabslice +tb0 +tb0_init +tb1 +tb10 +tb10_init +tb11 +tb11_init +tb12 +tb12_init +tb13 +tb13_init +tb14 +tb14_init +tb15 +tb15_init +tb1_init +tb2 +tb2_init +tb3 +tb3_init +tb4 +tb4_init +tb5 +tb5_init +tb6 +tb6_init +tb7 +tb7_init +tb8 +tb8_init +tb9 +tb9_init +vbap16 +vbap4 +vbap4move +vbap8 +vbap8move +xyin +'''.split()) diff --git a/pygments/lexers/_lasso_builtins.py b/pygments/lexers/_lasso_builtins.py index d950cbe8..1d2719da 100644 --- a/pygments/lexers/_lasso_builtins.py +++ b/pygments/lexers/_lasso_builtins.py @@ -5,7 +5,7 @@ Built-in Lasso types, traits, methods, and members. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_lua_builtins.py b/pygments/lexers/_lua_builtins.py index c60bf5a2..b2b46a6f 100644 --- a/pygments/lexers/_lua_builtins.py +++ b/pygments/lexers/_lua_builtins.py @@ -9,7 +9,7 @@ Do not edit the MODULES dict by hand. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -288,7 +288,7 @@ if __name__ == '__main__': # pragma: no cover print('>> %s' % full_function_name) m = get_function_module(full_function_name) modules.setdefault(m, []).append(full_function_name) - modules = {k: tuple(v) for k, v in modules.iteritems()} + modules = dict((k, tuple(v)) for k, v in modules.iteritems()) regenerate(__file__, modules) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index b48ee1d1..adb1ec83 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -44,14 +44,16 @@ LEXERS = { 'ArduinoLexer': ('pygments.lexers.c_like', 'Arduino', ('arduino',), ('*.ino',), ('text/x-arduino',)), 'AspectJLexer': ('pygments.lexers.jvm', 'AspectJ', ('aspectj',), ('*.aj',), ('text/x-aspectj',)), 'AsymptoteLexer': ('pygments.lexers.graphics', 'Asymptote', ('asy', 'asymptote'), ('*.asy',), ('text/x-asymptote',)), + 'AugeasLexer': ('pygments.lexers.configs', 'Augeas', ('augeas',), ('*.aug',), ()), 'AutoItLexer': ('pygments.lexers.automation', 'AutoIt', ('autoit',), ('*.au3',), ('text/x-autoit',)), 'AutohotkeyLexer': ('pygments.lexers.automation', 'autohotkey', ('ahk', 'autohotkey'), ('*.ahk', '*.ahkl'), ('text/x-autohotkey',)), 'AwkLexer': ('pygments.lexers.textedit', 'Awk', ('awk', 'gawk', 'mawk', 'nawk'), ('*.awk',), ('application/x-awk',)), + 'BBCBasicLexer': ('pygments.lexers.basic', 'BBC Basic', ('bbcbasic',), ('*.bbc',), ()), 'BBCodeLexer': ('pygments.lexers.markup', 'BBCode', ('bbcode',), (), ('text/x-bbcode',)), 'BCLexer': ('pygments.lexers.algebra', 'BC', ('bc',), ('*.bc',), ()), 'BSTLexer': ('pygments.lexers.bibtex', 'BST', ('bst', 'bst-pybtex'), ('*.bst',), ()), 'BaseMakefileLexer': ('pygments.lexers.make', 'Base Makefile', ('basemake',), (), ()), - 'BashLexer': ('pygments.lexers.shell', 'Bash', ('bash', 'sh', 'ksh', 'zsh', 'shell'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', '*.exheres-0', '*.exlib', '*.zsh', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', 'PKGBUILD'), ('application/x-sh', 'application/x-shellscript')), + 'BashLexer': ('pygments.lexers.shell', 'Bash', ('bash', 'sh', 'ksh', 'zsh', 'shell'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', '*.exheres-0', '*.exlib', '*.zsh', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', 'PKGBUILD'), ('application/x-sh', 'application/x-shellscript', 'text/x-shellscript')), 'BashSessionLexer': ('pygments.lexers.shell', 'Bash Session', ('console', 'shell-session'), ('*.sh-session', '*.shell-session'), ('application/x-shell-session', 'application/x-sh-session')), 'BatchLexer': ('pygments.lexers.shell', 'Batchfile', ('bat', 'batch', 'dosbatch', 'winbatch'), ('*.bat', '*.cmd'), ('application/x-dos-batch',)), 'BefungeLexer': ('pygments.lexers.esoteric', 'Befunge', ('befunge',), ('*.befunge',), ('application/x-befunge',)), @@ -59,6 +61,7 @@ LEXERS = { 'BlitzBasicLexer': ('pygments.lexers.basic', 'BlitzBasic', ('blitzbasic', 'b3d', 'bplus'), ('*.bb', '*.decls'), ('text/x-bb',)), 'BlitzMaxLexer': ('pygments.lexers.basic', 'BlitzMax', ('blitzmax', 'bmax'), ('*.bmx',), ('text/x-bmx',)), 'BnfLexer': ('pygments.lexers.grammar_notation', 'BNF', ('bnf',), ('*.bnf',), ('text/x-bnf',)), + 'BoaLexer': ('pygments.lexers.boa', 'Boa', ('boa',), ('*.boa',), ()), 'BooLexer': ('pygments.lexers.dotnet', 'Boo', ('boo',), ('*.boo',), ('text/x-boo',)), 'BoogieLexer': ('pygments.lexers.verification', 'Boogie', ('boogie',), ('*.bpl',), ()), 'BrainfuckLexer': ('pygments.lexers.esoteric', 'Brainfuck', ('brainfuck', 'bf'), ('*.bf', '*.b'), ('application/x-brainfuck',)), @@ -80,6 +83,7 @@ LEXERS = { 'Cfengine3Lexer': ('pygments.lexers.configs', 'CFEngine3', ('cfengine3', 'cf3'), ('*.cf',), ()), 'ChaiscriptLexer': ('pygments.lexers.scripting', 'ChaiScript', ('chai', 'chaiscript'), ('*.chai',), ('text/x-chaiscript', 'application/x-chaiscript')), 'ChapelLexer': ('pygments.lexers.chapel', 'Chapel', ('chapel', 'chpl'), ('*.chpl',), ()), + 'CharmciLexer': ('pygments.lexers.c_like', 'Charmci', ('charmci',), ('*.ci',), ()), 'CheetahHtmlLexer': ('pygments.lexers.templates', 'HTML+Cheetah', ('html+cheetah', 'html+spitfire', 'htmlcheetah'), (), ('text/html+cheetah', 'text/html+spitfire')), 'CheetahJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Cheetah', ('js+cheetah', 'javascript+cheetah', 'js+spitfire', 'javascript+spitfire'), (), ('application/x-javascript+cheetah', 'text/x-javascript+cheetah', 'text/javascript+cheetah', 'application/x-javascript+spitfire', 'text/x-javascript+spitfire', 'text/javascript+spitfire')), 'CheetahLexer': ('pygments.lexers.templates', 'Cheetah', ('cheetah', 'spitfire'), ('*.tmpl', '*.spt'), ('application/x-cheetah', 'application/x-spitfire')), @@ -105,7 +109,7 @@ LEXERS = { 'CryptolLexer': ('pygments.lexers.haskell', 'Cryptol', ('cryptol', 'cry'), ('*.cry',), ('text/x-cryptol',)), 'CrystalLexer': ('pygments.lexers.crystal', 'Crystal', ('cr', 'crystal'), ('*.cr',), ('text/x-crystal',)), 'CsoundDocumentLexer': ('pygments.lexers.csound', 'Csound Document', ('csound-document', 'csound-csd'), ('*.csd',), ()), - 'CsoundOrchestraLexer': ('pygments.lexers.csound', 'Csound Orchestra', ('csound', 'csound-orc'), ('*.orc',), ()), + 'CsoundOrchestraLexer': ('pygments.lexers.csound', 'Csound Orchestra', ('csound', 'csound-orc'), ('*.orc', '*.udo'), ()), 'CsoundScoreLexer': ('pygments.lexers.csound', 'Csound Score', ('csound-score', 'csound-sco'), ('*.sco',), ()), 'CssDjangoLexer': ('pygments.lexers.templates', 'CSS+Django/Jinja', ('css+django', 'css+jinja'), (), ('text/css+django', 'text/css+jinja')), 'CssErbLexer': ('pygments.lexers.templates', 'CSS+Ruby', ('css+erb', 'css+ruby'), (), ('text/css+ruby',)), @@ -120,6 +124,7 @@ LEXERS = { 'DObjdumpLexer': ('pygments.lexers.asm', 'd-objdump', ('d-objdump',), ('*.d-objdump',), ('text/x-d-objdump',)), 'DarcsPatchLexer': ('pygments.lexers.diff', 'Darcs Patch', ('dpatch',), ('*.dpatch', '*.darcspatch'), ()), 'DartLexer': ('pygments.lexers.javascript', 'Dart', ('dart',), ('*.dart',), ('text/x-dart',)), + 'Dasm16Lexer': ('pygments.lexers.asm', 'DASM16', ('dasm16',), ('*.dasm16', '*.dasm'), ('text/x-dasm16',)), 'DebianControlLexer': ('pygments.lexers.installers', 'Debian Control file', ('control', 'debcontrol'), ('control',), ()), 'DelphiLexer': ('pygments.lexers.pascal', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas', '*.dpr'), ('text/x-pascal',)), 'DgLexer': ('pygments.lexers.python', 'dg', ('dg',), ('*.dg',), ('text/x-dg',)), @@ -148,17 +153,20 @@ LEXERS = { 'EvoqueLexer': ('pygments.lexers.templates', 'Evoque', ('evoque',), ('*.evoque',), ('application/x-evoque',)), 'EvoqueXmlLexer': ('pygments.lexers.templates', 'XML+Evoque', ('xml+evoque',), ('*.xml',), ('application/xml+evoque',)), 'EzhilLexer': ('pygments.lexers.ezhil', 'Ezhil', ('ezhil',), ('*.n',), ('text/x-ezhil',)), - 'FSharpLexer': ('pygments.lexers.dotnet', 'FSharp', ('fsharp',), ('*.fs', '*.fsi'), ('text/x-fsharp',)), + 'FSharpLexer': ('pygments.lexers.dotnet', 'F#', ('fsharp', 'f#'), ('*.fs', '*.fsi'), ('text/x-fsharp',)), 'FactorLexer': ('pygments.lexers.factor', 'Factor', ('factor',), ('*.factor',), ('text/x-factor',)), 'FancyLexer': ('pygments.lexers.ruby', 'Fancy', ('fancy', 'fy'), ('*.fy', '*.fancypack'), ('text/x-fancysrc',)), 'FantomLexer': ('pygments.lexers.fantom', 'Fantom', ('fan',), ('*.fan',), ('application/x-fantom',)), 'FelixLexer': ('pygments.lexers.felix', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)), + 'FennelLexer': ('pygments.lexers.lisp', 'Fennel', ('fennel', 'fnl'), ('*.fnl',), ()), 'FishShellLexer': ('pygments.lexers.shell', 'Fish', ('fish', 'fishshell'), ('*.fish', '*.load'), ('application/x-fish',)), 'FlatlineLexer': ('pygments.lexers.dsls', 'Flatline', ('flatline',), (), ('text/x-flatline',)), + 'FloScriptLexer': ('pygments.lexers.floscript', 'FloScript', ('floscript', 'flo'), ('*.flo',), ()), 'ForthLexer': ('pygments.lexers.forth', 'Forth', ('forth',), ('*.frt', '*.fs'), ('application/x-forth',)), 'FortranFixedLexer': ('pygments.lexers.fortran', 'FortranFixed', ('fortranfixed',), ('*.f', '*.F'), ()), 'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f03', '*.f90', '*.F03', '*.F90'), ('text/x-fortran',)), 'FoxProLexer': ('pygments.lexers.foxpro', 'FoxPro', ('foxpro', 'vfp', 'clipper', 'xbase'), ('*.PRG', '*.prg'), ()), + 'FreeFemLexer': ('pygments.lexers.freefem', 'Freefem', ('freefem',), ('*.edp',), ('text/x-freefem',)), 'GAPLexer': ('pygments.lexers.algebra', 'GAP', ('gap',), ('*.g', '*.gd', '*.gi', '*.gap'), ()), 'GLShaderLexer': ('pygments.lexers.graphics', 'GLSL', ('glsl',), ('*.vert', '*.frag', '*.geo'), ('text/x-glslsrc',)), 'GasLexer': ('pygments.lexers.asm', 'GAS', ('gas', 'asm'), ('*.s', '*.S'), ('text/x-gas',)), @@ -174,6 +182,7 @@ LEXERS = { 'GosuTemplateLexer': ('pygments.lexers.jvm', 'Gosu Template', ('gst',), ('*.gst',), ('text/x-gosu-template',)), 'GroffLexer': ('pygments.lexers.markup', 'Groff', ('groff', 'nroff', 'man'), ('*.[1234567]', '*.man'), ('application/x-troff', 'text/troff')), 'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy', '*.gradle'), ('text/x-groovy',)), + 'HLSLShaderLexer': ('pygments.lexers.graphics', 'HLSL', ('hlsl',), ('*.hlsl', '*.hlsli'), ('text/x-hlsl',)), 'HamlLexer': ('pygments.lexers.html', 'Haml', ('haml',), ('*.haml',), ('text/x-haml',)), 'HandlebarsHtmlLexer': ('pygments.lexers.templates', 'HTML+Handlebars', ('html+handlebars',), ('*.handlebars', '*.hbs'), ('text/html+handlebars', 'text/x-handlebars-template')), 'HandlebarsLexer': ('pygments.lexers.templates', 'Handlebars', ('handlebars',), (), ()), @@ -181,6 +190,7 @@ LEXERS = { 'HaxeLexer': ('pygments.lexers.haxe', 'Haxe', ('hx', 'haxe', 'hxsl'), ('*.hx', '*.hxsl'), ('text/haxe', 'text/x-haxe', 'text/x-hx')), 'HexdumpLexer': ('pygments.lexers.hexdump', 'Hexdump', ('hexdump',), (), ()), 'HsailLexer': ('pygments.lexers.asm', 'HSAIL', ('hsail', 'hsa'), ('*.hsail',), ('text/x-hsail',)), + 'HspecLexer': ('pygments.lexers.haskell', 'Hspec', ('hspec',), (), ()), 'HtmlDjangoLexer': ('pygments.lexers.templates', 'HTML+Django/Jinja', ('html+django', 'html+jinja', 'htmldjango'), (), ('text/html+django', 'text/html+jinja')), 'HtmlGenshiLexer': ('pygments.lexers.templates', 'HTML+Genshi', ('html+genshi', 'html+kid'), (), ('text/html+genshi',)), 'HtmlLexer': ('pygments.lexers.html', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml', '*.xslt'), ('text/html', 'application/xhtml+xml')), @@ -191,6 +201,7 @@ LEXERS = { 'HyLexer': ('pygments.lexers.lisp', 'Hy', ('hylang',), ('*.hy',), ('text/x-hy', 'application/x-hy')), 'HybrisLexer': ('pygments.lexers.scripting', 'Hybris', ('hybris', 'hy'), ('*.hy', '*.hyb'), ('text/x-hybris', 'application/x-hybris')), 'IDLLexer': ('pygments.lexers.idl', 'IDL', ('idl',), ('*.pro',), ('text/idl',)), + 'IconLexer': ('pygments.lexers.unicon', 'Icon', ('icon',), ('*.icon', '*.ICON'), ()), 'IdrisLexer': ('pygments.lexers.haskell', 'Idris', ('idris', 'idr'), ('*.idr',), ('text/x-idris',)), 'IgorLexer': ('pygments.lexers.igor', 'Igor', ('igor', 'igorpro'), ('*.ipf',), ('text/ipf',)), 'Inform6Lexer': ('pygments.lexers.int_fiction', 'Inform 6', ('inform6', 'i6'), ('*.inf',), ()), @@ -316,6 +327,7 @@ LEXERS = { 'PikeLexer': ('pygments.lexers.c_like', 'Pike', ('pike',), ('*.pike', '*.pmod'), ('text/x-pike',)), 'PkgConfigLexer': ('pygments.lexers.configs', 'PkgConfig', ('pkgconfig',), ('*.pc',), ()), 'PlPgsqlLexer': ('pygments.lexers.sql', 'PL/pgSQL', ('plpgsql',), (), ('text/x-plpgsql',)), + 'PonyLexer': ('pygments.lexers.pony', 'Pony', ('pony',), ('*.pony',), ()), 'PostScriptLexer': ('pygments.lexers.graphics', 'PostScript', ('postscript', 'postscr'), ('*.ps', '*.eps'), ('application/postscript',)), 'PostgresConsoleLexer': ('pygments.lexers.sql', 'PostgreSQL console (psql)', ('psql', 'postgresql-console', 'postgres-console'), (), ('text/x-postgresql-psql',)), 'PostgresLexer': ('pygments.lexers.sql', 'PostgreSQL SQL dialect', ('postgresql', 'postgres'), (), ('text/x-postgresql',)), @@ -367,10 +379,11 @@ LEXERS = { 'RtsLexer': ('pygments.lexers.trafficscript', 'TrafficScript', ('rts', 'trafficscript'), ('*.rts',), ()), 'RubyConsoleLexer': ('pygments.lexers.ruby', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)), 'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')), - 'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs', '*.rs.in'), ('text/rust',)), + 'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust', 'rs'), ('*.rs', '*.rs.in'), ('text/rust',)), 'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')), 'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')), 'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')), + 'SarlLexer': ('pygments.lexers.jvm', 'SARL', ('sarl',), ('*.sarl',), ('text/x-sarl',)), 'SassLexer': ('pygments.lexers.css', 'Sass', ('sass',), ('*.sass',), ('text/x-sass',)), 'ScalaLexer': ('pygments.lexers.jvm', 'Scala', ('scala',), ('*.scala',), ('text/x-scala',)), 'ScamlLexer': ('pygments.lexers.html', 'Scaml', ('scaml',), ('*.scaml',), ('text/x-scaml',)), @@ -379,9 +392,12 @@ LEXERS = { 'ScssLexer': ('pygments.lexers.css', 'SCSS', ('scss',), ('*.scss',), ('text/x-scss',)), 'ShenLexer': ('pygments.lexers.lisp', 'Shen', ('shen',), ('*.shen',), ('text/x-shen', 'application/x-shen')), 'SilverLexer': ('pygments.lexers.verification', 'Silver', ('silver',), ('*.sil', '*.vpr'), ()), + 'SlashLexer': ('pygments.lexers.slash', 'Slash', ('slash',), ('*.sl',), ()), 'SlimLexer': ('pygments.lexers.webmisc', 'Slim', ('slim',), ('*.slim',), ('text/x-slim',)), + 'SlurmBashLexer': ('pygments.lexers.shell', 'Slurm', ('slurm', 'sbatch'), ('*.sl',), ()), 'SmaliLexer': ('pygments.lexers.dalvik', 'Smali', ('smali',), ('*.smali',), ('text/smali',)), 'SmalltalkLexer': ('pygments.lexers.smalltalk', 'Smalltalk', ('smalltalk', 'squeak', 'st'), ('*.st',), ('text/x-smalltalk',)), + 'SmartGameFormatLexer': ('pygments.lexers.sgf', 'SmartGameFormat', ('sgf',), ('*.sgf',), ()), 'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)), 'SnobolLexer': ('pygments.lexers.snobol', 'Snobol', ('snobol',), ('*.snobol',), ('text/x-snobol',)), 'SnowballLexer': ('pygments.lexers.dsls', 'Snowball', ('snowball',), ('*.sbl',), ()), @@ -399,12 +415,14 @@ LEXERS = { 'SwigLexer': ('pygments.lexers.c_like', 'SWIG', ('swig',), ('*.swg', '*.i'), ('text/swig',)), 'SystemVerilogLexer': ('pygments.lexers.hdl', 'systemverilog', ('systemverilog', 'sv'), ('*.sv', '*.svh'), ('text/x-systemverilog',)), 'TAPLexer': ('pygments.lexers.testing', 'TAP', ('tap',), ('*.tap',), ()), + 'TOMLLexer': ('pygments.lexers.configs', 'TOML', ('toml',), ('*.toml',), ()), 'Tads3Lexer': ('pygments.lexers.int_fiction', 'TADS 3', ('tads3',), ('*.t',), ()), 'TasmLexer': ('pygments.lexers.asm', 'TASM', ('tasm',), ('*.asm', '*.ASM', '*.tasm'), ('text/x-tasm',)), 'TclLexer': ('pygments.lexers.tcl', 'Tcl', ('tcl',), ('*.tcl', '*.rvt'), ('text/x-tcl', 'text/x-script.tcl', 'application/x-tcl')), 'TcshLexer': ('pygments.lexers.shell', 'Tcsh', ('tcsh', 'csh'), ('*.tcsh', '*.csh'), ('application/x-csh',)), 'TcshSessionLexer': ('pygments.lexers.shell', 'Tcsh Session', ('tcshcon',), (), ()), 'TeaTemplateLexer': ('pygments.lexers.templates', 'Tea', ('tea',), ('*.tea',), ('text/x-tea',)), + 'TeraTermLexer': ('pygments.lexers.teraterm', 'Tera Term macro', ('ttl', 'teraterm', 'teratermmacro'), ('*.ttl',), ('text/x-teratermmacro',)), 'TermcapLexer': ('pygments.lexers.configs', 'Termcap', ('termcap',), ('termcap', 'termcap.src'), ()), 'TerminfoLexer': ('pygments.lexers.configs', 'Terminfo', ('terminfo',), ('terminfo', 'terminfo.src'), ()), 'TerraformLexer': ('pygments.lexers.configs', 'Terraform', ('terraform', 'tf'), ('*.tf',), ('application/x-tf', 'application/x-terraform')), @@ -420,8 +438,11 @@ LEXERS = { 'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts', 'typescript'), ('*.ts', '*.tsx'), ('text/x-typescript',)), 'TypoScriptCssDataLexer': ('pygments.lexers.typoscript', 'TypoScriptCssData', ('typoscriptcssdata',), (), ()), 'TypoScriptHtmlDataLexer': ('pygments.lexers.typoscript', 'TypoScriptHtmlData', ('typoscripthtmldata',), (), ()), - 'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.ts', '*.txt'), ('text/x-typoscript',)), + 'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.typoscript',), ('text/x-typoscript',)), + 'UcodeLexer': ('pygments.lexers.unicon', 'ucode', ('ucode',), ('*.u', '*.u1', '*.u2'), ()), + 'UniconLexer': ('pygments.lexers.unicon', 'Unicon', ('unicon',), ('*.icn',), ('text/unicon',)), 'UrbiscriptLexer': ('pygments.lexers.urbi', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)), + 'VBScriptLexer': ('pygments.lexers.basic', 'VBScript', ('vbscript',), ('*.vbs', '*.VBS'), ()), 'VCLLexer': ('pygments.lexers.varnish', 'VCL', ('vcl',), ('*.vcl',), ('text/x-vclsrc',)), 'VCLSnippetLexer': ('pygments.lexers.varnish', 'VCLSnippets', ('vclsnippets', 'vclsnippet'), (), ('text/x-vclsnippet',)), 'VCTreeStatusLexer': ('pygments.lexers.console', 'VCTreeStatus', ('vctreestatus',), (), ()), @@ -451,6 +472,7 @@ LEXERS = { 'YamlJinjaLexer': ('pygments.lexers.templates', 'YAML+Jinja', ('yaml+jinja', 'salt', 'sls'), ('*.sls',), ('text/x-yaml+jinja', 'text/x-sls')), 'YamlLexer': ('pygments.lexers.data', 'YAML', ('yaml',), ('*.yaml', '*.yml'), ('text/x-yaml',)), 'ZephirLexer': ('pygments.lexers.php', 'Zephir', ('zephir',), ('*.zep',), ()), + 'ZigLexer': ('pygments.lexers.zig', 'Zig', ('zig',), ('*.zig',), ('text/zig',)), } if __name__ == '__main__': # pragma: no cover diff --git a/pygments/lexers/_mql_builtins.py b/pygments/lexers/_mql_builtins.py index 6eb600c4..8c803791 100644 --- a/pygments/lexers/_mql_builtins.py +++ b/pygments/lexers/_mql_builtins.py @@ -5,7 +5,7 @@ Builtins for the MqlLexer. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ types = ( diff --git a/pygments/lexers/_openedge_builtins.py b/pygments/lexers/_openedge_builtins.py index 0fa7d1b2..09587e00 100644 --- a/pygments/lexers/_openedge_builtins.py +++ b/pygments/lexers/_openedge_builtins.py @@ -5,7 +5,7 @@ Builtin list for the OpenEdgeLexer. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_php_builtins.py b/pygments/lexers/_php_builtins.py index fec3286a..44ef2053 100644 --- a/pygments/lexers/_php_builtins.py +++ b/pygments/lexers/_php_builtins.py @@ -12,7 +12,7 @@ internet connection. don't run that at home, use a server ;-) - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -4688,7 +4688,7 @@ if __name__ == '__main__': # pragma: no cover PHP_MANUAL_URL = 'http://us3.php.net/distributions/manual/php_manual_en.tar.gz' PHP_MANUAL_DIR = './php-chunked-xhtml/' PHP_REFERENCE_GLOB = 'ref.*' - PHP_FUNCTION_RE = '<a href="function\..*?\.html">(.*?)</a>' + PHP_FUNCTION_RE = r'<a href="function\..*?\.html">(.*?)</a>' PHP_MODULE_RE = '<title>(.*?) Functions</title>' def get_php_functions(): @@ -4698,18 +4698,19 @@ if __name__ == '__main__': # pragma: no cover for file in get_php_references(): module = '' - for line in open(file): - if not module: - search = module_re.search(line) - if search: - module = search.group(1) - modules[module] = [] + with open(file) as f: + for line in f: + if not module: + search = module_re.search(line) + if search: + module = search.group(1) + modules[module] = [] - elif 'href="function.' in line: - for match in function_re.finditer(line): - fn = match.group(1) - if '->' not in fn and '::' not in fn and fn not in modules[module]: - modules[module].append(fn) + elif 'href="function.' in line: + for match in function_re.finditer(line): + fn = match.group(1) + if '->' not in fn and '::' not in fn and fn not in modules[module]: + modules[module].append(fn) if module: # These are dummy manual pages, not actual functions @@ -4726,9 +4727,8 @@ if __name__ == '__main__': # pragma: no cover def get_php_references(): download = urlretrieve(PHP_MANUAL_URL) - tar = tarfile.open(download[0]) - tar.extractall() - tar.close() + with tarfile.open(download[0]) as tar: + tar.extractall() for file in glob.glob("%s%s" % (PHP_MANUAL_DIR, PHP_REFERENCE_GLOB)): yield file os.remove(download[0]) diff --git a/pygments/lexers/_postgres_builtins.py b/pygments/lexers/_postgres_builtins.py index a71360f0..21086722 100644 --- a/pygments/lexers/_postgres_builtins.py +++ b/pygments/lexers/_postgres_builtins.py @@ -5,7 +5,7 @@ Self-updating data files for PostgreSQL lexer. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_scilab_builtins.py b/pygments/lexers/_scilab_builtins.py index ce0ac67d..b26a0085 100644 --- a/pygments/lexers/_scilab_builtins.py +++ b/pygments/lexers/_scilab_builtins.py @@ -5,7 +5,7 @@ Builtin list for the ScilabLexer. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_sourcemod_builtins.py b/pygments/lexers/_sourcemod_builtins.py index f08ea481..fa967167 100644 --- a/pygments/lexers/_sourcemod_builtins.py +++ b/pygments/lexers/_sourcemod_builtins.py @@ -8,7 +8,7 @@ Do not edit the FUNCTIONS list by hand. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_stan_builtins.py b/pygments/lexers/_stan_builtins.py index a189647a..e95f5b1e 100644 --- a/pygments/lexers/_stan_builtins.py +++ b/pygments/lexers/_stan_builtins.py @@ -4,24 +4,23 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This file contains the names of functions for Stan used by - ``pygments.lexers.math.StanLexer. This is for Stan language version 2.8.0. + ``pygments.lexers.math.StanLexer. This is for Stan language version 2.17.0. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ KEYWORDS = ( + 'break', + 'continue', 'else', 'for', 'if', 'in', - 'increment_log_prob', - 'integrate_ode', - 'lp__', 'print', 'reject', 'return', - 'while' + 'while', ) TYPES = ( @@ -35,18 +34,18 @@ TYPES = ( 'positive_ordered', 'real', 'row_vector', - 'row_vectormatrix', 'simplex', 'unit_vector', 'vector', - 'void') + 'void', +) FUNCTIONS = ( - 'Phi', - 'Phi_approx', 'abs', 'acos', 'acosh', + 'algebra_solver', + 'append_array', 'append_col', 'append_row', 'asin', @@ -54,55 +53,59 @@ FUNCTIONS = ( 'atan', 'atan2', 'atanh', - 'bernoulli_ccdf_log', 'bernoulli_cdf', - 'bernoulli_cdf_log', - 'bernoulli_log', - 'bernoulli_logit_log', + 'bernoulli_lccdf', + 'bernoulli_lcdf', + 'bernoulli_logit_lpmf', + 'bernoulli_logit_rng', + 'bernoulli_lpmf', 'bernoulli_rng', 'bessel_first_kind', 'bessel_second_kind', - 'beta_binomial_ccdf_log', 'beta_binomial_cdf', - 'beta_binomial_cdf_log', - 'beta_binomial_log', + 'beta_binomial_lccdf', + 'beta_binomial_lcdf', + 'beta_binomial_lpmf', 'beta_binomial_rng', - 'beta_ccdf_log', 'beta_cdf', - 'beta_cdf_log', - 'beta_log', + 'beta_lccdf', + 'beta_lcdf', + 'beta_lpdf', 'beta_rng', 'binary_log_loss', - 'binomial_ccdf_log', 'binomial_cdf', - 'binomial_cdf_log', 'binomial_coefficient_log', - 'binomial_log', - 'binomial_logit_log', + 'binomial_lccdf', + 'binomial_lcdf', + 'binomial_logit_lpmf', + 'binomial_lpmf', 'binomial_rng', 'block', - 'categorical_log', - 'categorical_logit_log', + 'categorical_logit_lpmf', + 'categorical_logit_rng', + 'categorical_lpmf', 'categorical_rng', - 'cauchy_ccdf_log', 'cauchy_cdf', - 'cauchy_cdf_log', - 'cauchy_log', + 'cauchy_lccdf', + 'cauchy_lcdf', + 'cauchy_lpdf', 'cauchy_rng', 'cbrt', 'ceil', - 'chi_square_ccdf_log', 'chi_square_cdf', - 'chi_square_cdf_log', - 'chi_square_log', + 'chi_square_lccdf', + 'chi_square_lcdf', + 'chi_square_lpdf', 'chi_square_rng', 'cholesky_decompose', + 'choose', 'col', 'cols', 'columns_dot_product', 'columns_dot_self', 'cos', 'cosh', + 'cov_exp_quad', 'crossprod', 'csr_extract_u', 'csr_extract_v', @@ -117,15 +120,15 @@ FUNCTIONS = ( 'diagonal', 'digamma', 'dims', - 'dirichlet_log', + 'dirichlet_lpdf', 'dirichlet_rng', 'distance', 'dot_product', 'dot_self', - 'double_exponential_ccdf_log', 'double_exponential_cdf', - 'double_exponential_cdf_log', - 'double_exponential_log', + 'double_exponential_lccdf', + 'double_exponential_lcdf', + 'double_exponential_lpdf', 'double_exponential_rng', 'e', 'eigenvalues_sym', @@ -134,16 +137,16 @@ FUNCTIONS = ( 'erfc', 'exp', 'exp2', - 'exp_mod_normal_ccdf_log', 'exp_mod_normal_cdf', - 'exp_mod_normal_cdf_log', - 'exp_mod_normal_log', + 'exp_mod_normal_lccdf', + 'exp_mod_normal_lcdf', + 'exp_mod_normal_lpdf', 'exp_mod_normal_rng', 'expm1', - 'exponential_ccdf_log', 'exponential_cdf', - 'exponential_cdf_log', - 'exponential_log', + 'exponential_lccdf', + 'exponential_lcdf', + 'exponential_lpdf', 'exponential_rng', 'fabs', 'falling_factorial', @@ -153,60 +156,65 @@ FUNCTIONS = ( 'fmax', 'fmin', 'fmod', - 'frechet_ccdf_log', 'frechet_cdf', - 'frechet_cdf_log', - 'frechet_log', + 'frechet_lccdf', + 'frechet_lcdf', + 'frechet_lpdf', 'frechet_rng', - 'gamma_ccdf_log', 'gamma_cdf', - 'gamma_cdf_log', - 'gamma_log', + 'gamma_lccdf', + 'gamma_lcdf', + 'gamma_lpdf', 'gamma_p', 'gamma_q', 'gamma_rng', - 'gaussian_dlm_obs_log', + 'gaussian_dlm_obs_lpdf', 'get_lp', - 'gumbel_ccdf_log', 'gumbel_cdf', - 'gumbel_cdf_log', - 'gumbel_log', + 'gumbel_lccdf', + 'gumbel_lcdf', + 'gumbel_lpdf', 'gumbel_rng', 'head', - 'hypergeometric_log', + 'hypergeometric_lpmf', 'hypergeometric_rng', 'hypot', - 'if_else', + 'inc_beta', 'int_step', + 'integrate_ode', + 'integrate_ode_bdf', + 'integrate_ode_rk45', 'inv', - 'inv_chi_square_ccdf_log', 'inv_chi_square_cdf', - 'inv_chi_square_cdf_log', - 'inv_chi_square_log', + 'inv_chi_square_lccdf', + 'inv_chi_square_lcdf', + 'inv_chi_square_lpdf', 'inv_chi_square_rng', 'inv_cloglog', - 'inv_gamma_ccdf_log', 'inv_gamma_cdf', - 'inv_gamma_cdf_log', - 'inv_gamma_log', + 'inv_gamma_lccdf', + 'inv_gamma_lcdf', + 'inv_gamma_lpdf', 'inv_gamma_rng', 'inv_logit', - 'inv_phi', + 'inv_Phi', 'inv_sqrt', 'inv_square', - 'inv_wishart_log', + 'inv_wishart_lpdf', 'inv_wishart_rng', 'inverse', 'inverse_spd', 'is_inf', 'is_nan', 'lbeta', + 'lchoose', 'lgamma', - 'lkj_corr_cholesky_log', + 'lkj_corr_cholesky_lpdf', 'lkj_corr_cholesky_rng', - 'lkj_corr_log', + 'lkj_corr_lpdf', 'lkj_corr_rng', 'lmgamma', + 'lmultiply', 'log', 'log10', 'log1m', @@ -223,81 +231,87 @@ FUNCTIONS = ( 'log_rising_factorial', 'log_softmax', 'log_sum_exp', - 'logistic_ccdf_log', 'logistic_cdf', - 'logistic_cdf_log', - 'logistic_log', + 'logistic_lccdf', + 'logistic_lcdf', + 'logistic_lpdf', 'logistic_rng', 'logit', - 'lognormal_ccdf_log', 'lognormal_cdf', - 'lognormal_cdf_log', - 'lognormal_log', + 'lognormal_lccdf', + 'lognormal_lcdf', + 'lognormal_lpdf', 'lognormal_rng', 'machine_precision', + 'matrix_exp', 'max', + 'mdivide_left_spd', 'mdivide_left_tri_low', + 'mdivide_right_spd', 'mdivide_right_tri_low', 'mean', 'min', 'modified_bessel_first_kind', 'modified_bessel_second_kind', - 'multi_gp_cholesky_log', - 'multi_gp_log', - 'multi_normal_cholesky_log', + 'multi_gp_cholesky_lpdf', + 'multi_gp_lpdf', + 'multi_normal_cholesky_lpdf', 'multi_normal_cholesky_rng', - 'multi_normal_log', - 'multi_normal_prec_log', + 'multi_normal_lpdf', + 'multi_normal_prec_lpdf', 'multi_normal_rng', - 'multi_student_t_log', + 'multi_student_t_lpdf', 'multi_student_t_rng', - 'multinomial_log', + 'multinomial_lpmf', 'multinomial_rng', 'multiply_log', 'multiply_lower_tri_self_transpose', - 'neg_binomial_2_ccdf_log', 'neg_binomial_2_cdf', - 'neg_binomial_2_cdf_log', - 'neg_binomial_2_log', - 'neg_binomial_2_log_log', + 'neg_binomial_2_lccdf', + 'neg_binomial_2_lcdf', + 'neg_binomial_2_log_lpmf', 'neg_binomial_2_log_rng', + 'neg_binomial_2_lpmf', 'neg_binomial_2_rng', - 'neg_binomial_ccdf_log', 'neg_binomial_cdf', - 'neg_binomial_cdf_log', - 'neg_binomial_log', + 'neg_binomial_lccdf', + 'neg_binomial_lcdf', + 'neg_binomial_lpmf', 'neg_binomial_rng', 'negative_infinity', - 'normal_ccdf_log', 'normal_cdf', - 'normal_cdf_log', - 'normal_log', + 'normal_lccdf', + 'normal_lcdf', + 'normal_lpdf', 'normal_rng', 'not_a_number', 'num_elements', - 'ordered_logistic_log', + 'ordered_logistic_lpmf', 'ordered_logistic_rng', 'owens_t', - 'pareto_ccdf_log', 'pareto_cdf', - 'pareto_cdf_log', - 'pareto_log', + 'pareto_lccdf', + 'pareto_lcdf', + 'pareto_lpdf', 'pareto_rng', - 'pareto_type_2_ccdf_log', 'pareto_type_2_cdf', - 'pareto_type_2_cdf_log', - 'pareto_type_2_log', + 'pareto_type_2_lccdf', + 'pareto_type_2_lcdf', + 'pareto_type_2_lpdf', 'pareto_type_2_rng', + 'Phi', + 'Phi_approx', 'pi', - 'poisson_ccdf_log', 'poisson_cdf', - 'poisson_cdf_log', - 'poisson_log', - 'poisson_log_log', + 'poisson_lccdf', + 'poisson_lcdf', + 'poisson_log_lpmf', 'poisson_log_rng', + 'poisson_lpmf', 'poisson_rng', 'positive_infinity', 'pow', + 'print', 'prod', 'qr_Q', 'qr_R', @@ -305,11 +319,12 @@ FUNCTIONS = ( 'quad_form_diag', 'quad_form_sym', 'rank', - 'rayleigh_ccdf_log', 'rayleigh_cdf', - 'rayleigh_cdf_log', - 'rayleigh_log', + 'rayleigh_lccdf', + 'rayleigh_lcdf', + 'rayleigh_lpdf', 'rayleigh_rng', + 'reject', 'rep_array', 'rep_matrix', 'rep_row_vector', @@ -320,10 +335,10 @@ FUNCTIONS = ( 'rows', 'rows_dot_product', 'rows_dot_self', - 'scaled_inv_chi_square_ccdf_log', 'scaled_inv_chi_square_cdf', - 'scaled_inv_chi_square_cdf_log', - 'scaled_inv_chi_square_log', + 'scaled_inv_chi_square_lccdf', + 'scaled_inv_chi_square_lcdf', + 'scaled_inv_chi_square_lpdf', 'scaled_inv_chi_square_rng', 'sd', 'segment', @@ -331,10 +346,10 @@ FUNCTIONS = ( 'singular_values', 'sinh', 'size', - 'skew_normal_ccdf_log', 'skew_normal_cdf', - 'skew_normal_cdf_log', - 'skew_normal_log', + 'skew_normal_lccdf', + 'skew_normal_lcdf', + 'skew_normal_lpdf', 'skew_normal_rng', 'softmax', 'sort_asc', @@ -346,10 +361,10 @@ FUNCTIONS = ( 'square', 'squared_distance', 'step', - 'student_t_ccdf_log', 'student_t_cdf', - 'student_t_cdf_log', - 'student_t_log', + 'student_t_lccdf', + 'student_t_lcdf', + 'student_t_lpdf', 'student_t_rng', 'sub_col', 'sub_row', @@ -357,6 +372,7 @@ FUNCTIONS = ( 'tail', 'tan', 'tanh', + 'target', 'tcrossprod', 'tgamma', 'to_array_1d', @@ -369,22 +385,22 @@ FUNCTIONS = ( 'trace_quad_form', 'trigamma', 'trunc', - 'uniform_ccdf_log', 'uniform_cdf', - 'uniform_cdf_log', - 'uniform_log', + 'uniform_lccdf', + 'uniform_lcdf', + 'uniform_lpdf', 'uniform_rng', 'variance', - 'von_mises_log', + 'von_mises_lpdf', 'von_mises_rng', - 'weibull_ccdf_log', 'weibull_cdf', - 'weibull_cdf_log', - 'weibull_log', + 'weibull_lccdf', + 'weibull_lcdf', + 'weibull_lpdf', 'weibull_rng', - 'wiener_log', - 'wishart_log', - 'wishart_rng' + 'wiener_lpdf', + 'wishart_lpdf', + 'wishart_rng', ) DISTRIBUTIONS = ( @@ -438,7 +454,7 @@ DISTRIBUTIONS = ( 'von_mises', 'weibull', 'wiener', - 'wishart' + 'wishart', ) RESERVED = ( @@ -469,19 +485,23 @@ RESERVED = ( 'do', 'double', 'dynamic_cast', + 'else', 'enum', 'explicit', 'export', 'extern', 'false', - 'false', 'float', + 'for', 'friend', 'fvar', 'goto', + 'if', + 'in', 'inline', 'int', 'long', + 'lp__', 'mutable', 'namespace', 'new', @@ -498,9 +518,16 @@ RESERVED = ( 'register', 'reinterpret_cast', 'repeat', + 'return', 'short', 'signed', 'sizeof', + 'STAN_MAJOR', + 'STAN_MATH_MAJOR', + 'STAN_MATH_MINOR', + 'STAN_MATH_PATCH', + 'STAN_MINOR', + 'STAN_PATCH', 'static', 'static_assert', 'static_cast', @@ -512,7 +539,6 @@ RESERVED = ( 'thread_local', 'throw', 'true', - 'true', 'try', 'typedef', 'typeid', @@ -526,7 +552,7 @@ RESERVED = ( 'void', 'volatile', 'wchar_t', + 'while', 'xor', - 'xor_eq' + 'xor_eq', ) - diff --git a/pygments/lexers/_stata_builtins.py b/pygments/lexers/_stata_builtins.py index 5f5f72a9..3f4abdcb 100644 --- a/pygments/lexers/_stata_builtins.py +++ b/pygments/lexers/_stata_builtins.py @@ -5,11 +5,15 @@ Builtins for Stata - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ +builtins_special = ( + "if", "in", "using", "replace", "by", "gen", "generate" +) + builtins_base = ( "if", "else", "in", "foreach", "for", "forv", "forva", "forval", "forvalu", "forvalue", "forvalues", "by", "bys", @@ -66,7 +70,7 @@ builtins_base = ( "doedit", "dotplot", "dotplot_7", "dprobit", "drawnorm", "drop", "ds", "ds_util", "dstdize", "duplicates", "durbina", "dwstat", "dydx", "e", "ed", "edi", "edit", "egen", - "eivreg", "emdef", "en", "enc", "enco", "encod", "encode", + "eivreg", "emdef", "end", "en", "enc", "enco", "encod", "encode", "eq", "erase", "ereg", "ereg_lf", "ereg_p", "ereg_sw", "ereghet", "ereghet_glf", "ereghet_glf_sh", "ereghet_gp", "ereghet_ilf", "ereghet_ilf_sh", "ereghet_ip", "eret", @@ -415,5 +419,3 @@ builtins_functions = ( "weekly", "wofd", "word", "wordcount", "year", "yearly", "yh", "ym", "yofd", "yq", "yw" ) - - diff --git a/pygments/lexers/_tsql_builtins.py b/pygments/lexers/_tsql_builtins.py index e29ed34b..dfc5f618 100644 --- a/pygments/lexers/_tsql_builtins.py +++ b/pygments/lexers/_tsql_builtins.py @@ -5,7 +5,7 @@ These are manually translated lists from https://msdn.microsoft.com. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_vbscript_builtins.py b/pygments/lexers/_vbscript_builtins.py new file mode 100644 index 00000000..e7520072 --- /dev/null +++ b/pygments/lexers/_vbscript_builtins.py @@ -0,0 +1,280 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers._vbscript_builtins + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + These are manually translated lists from + http://www.indusoft.com/pdf/VBScript%20Reference.pdf. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +KEYWORDS = [ + 'ByRef', + 'ByVal', + # dim: special rule + 'call', + 'case', + 'class', + # const: special rule + 'do', + 'each', + 'else', + 'elseif', + 'end', + 'erase', + 'execute', + 'function', + 'exit', + 'for', + 'function', + 'GetRef', + 'global', + 'if', + 'let', + 'loop', + 'next', + 'new', + # option: special rule + 'private', + 'public', + 'redim', + 'select', + 'set', + 'sub', + 'then', + 'wend', + 'while', + 'with', +] + +BUILTIN_FUNCTIONS = [ + 'Abs', + 'Array', + 'Asc', + 'Atn', + 'CBool', + 'CByte', + 'CCur', + 'CDate', + 'CDbl', + 'Chr', + 'CInt', + 'CLng', + 'Cos', + 'CreateObject', + 'CSng', + 'CStr', + 'Date', + 'DateAdd', + 'DateDiff', + 'DatePart', + 'DateSerial', + 'DateValue', + 'Day', + 'Eval', + 'Exp', + 'Filter', + 'Fix', + 'FormatCurrency', + 'FormatDateTime', + 'FormatNumber', + 'FormatPercent', + 'GetObject', + 'GetLocale', + 'Hex', + 'Hour', + 'InStr', + 'inStrRev', + 'Int', + 'IsArray', + 'IsDate', + 'IsEmpty', + 'IsNull', + 'IsNumeric', + 'IsObject', + 'Join', + 'LBound', + 'LCase', + 'Left', + 'Len', + 'LoadPicture', + 'Log', + 'LTrim', + 'Mid', + 'Minute', + 'Month', + 'MonthName', + 'MsgBox', + 'Now', + 'Oct', + 'Randomize', + 'RegExp', + 'Replace', + 'RGB', + 'Right', + 'Rnd', + 'Round', + 'RTrim', + 'ScriptEngine', + 'ScriptEngineBuildVersion', + 'ScriptEngineMajorVersion', + 'ScriptEngineMinorVersion', + 'Second', + 'SetLocale', + 'Sgn', + 'Space', + 'Split', + 'Sqr', + 'StrComp', + 'String', + 'StrReverse', + 'Tan', + 'Time', + 'Timer', + 'TimeSerial', + 'TimeValue', + 'Trim', + 'TypeName', + 'UBound', + 'UCase', + 'VarType', + 'Weekday', + 'WeekdayName', + 'Year', +] + +BUILTIN_VARIABLES = [ + 'Debug', + 'Dictionary', + 'Drive', + 'Drives', + 'Err', + 'File', + 'Files', + 'FileSystemObject', + 'Folder', + 'Folders', + 'Match', + 'Matches', + 'RegExp', + 'Submatches', + 'TextStream', +] + +OPERATORS = [ + '+', + '-', + '*', + '/', + '\\', + '^', + '|', + '<', + '<=', + '>', + '>=', + '=', + '<>', + '&', + '$', +] + +OPERATOR_WORDS = [ + 'mod', + 'and', + 'or', + 'xor', + 'eqv', + 'imp', + 'is', + 'not', +] + +BUILTIN_CONSTANTS = [ + 'False', + 'True', + 'vbAbort', + 'vbAbortRetryIgnore', + 'vbApplicationModal', + 'vbArray', + 'vbBinaryCompare', + 'vbBlack', + 'vbBlue', + 'vbBoole', + 'vbByte', + 'vbCancel', + 'vbCr', + 'vbCritical', + 'vbCrLf', + 'vbCurrency', + 'vbCyan', + 'vbDataObject', + 'vbDate', + 'vbDefaultButton1', + 'vbDefaultButton2', + 'vbDefaultButton3', + 'vbDefaultButton4', + 'vbDouble', + 'vbEmpty', + 'vbError', + 'vbExclamation', + 'vbFalse', + 'vbFirstFullWeek', + 'vbFirstJan1', + 'vbFormFeed', + 'vbFriday', + 'vbGeneralDate', + 'vbGreen', + 'vbIgnore', + 'vbInformation', + 'vbInteger', + 'vbLf', + 'vbLong', + 'vbLongDate', + 'vbLongTime', + 'vbMagenta', + 'vbMonday', + 'vbMsgBoxHelpButton', + 'vbMsgBoxRight', + 'vbMsgBoxRtlReading', + 'vbMsgBoxSetForeground', + 'vbNewLine', + 'vbNo', + 'vbNull', + 'vbNullChar', + 'vbNullString', + 'vbObject', + 'vbObjectError', + 'vbOK', + 'vbOKCancel', + 'vbOKOnly', + 'vbQuestion', + 'vbRed', + 'vbRetry', + 'vbRetryCancel', + 'vbSaturday', + 'vbShortDate', + 'vbShortTime', + 'vbSingle', + 'vbString', + 'vbSunday', + 'vbSystemModal', + 'vbTab', + 'vbTextCompare', + 'vbThursday', + 'vbTrue', + 'vbTuesday', + 'vbUseDefault', + 'vbUseSystem', + 'vbUseSystem', + 'vbVariant', + 'vbVerticalTab', + 'vbWednesday', + 'vbWhite', + 'vbYellow', + 'vbYes', + 'vbYesNo', + 'vbYesNoCancel', +] diff --git a/pygments/lexers/_vim_builtins.py b/pygments/lexers/_vim_builtins.py index 82586289..39c9ed19 100644 --- a/pygments/lexers/_vim_builtins.py +++ b/pygments/lexers/_vim_builtins.py @@ -5,7 +5,7 @@ This file is autogenerated by scripts/get_vimkw.py - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/actionscript.py b/pygments/lexers/actionscript.py index 84607e68..f4b4964e 100644 --- a/pygments/lexers/actionscript.py +++ b/pygments/lexers/actionscript.py @@ -5,7 +5,7 @@ Lexers for ActionScript and MXML. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -125,7 +125,7 @@ class ActionScript3Lexer(RegexLexer): 'text/actionscript3'] identifier = r'[$a-zA-Z_]\w*' - typeidentifier = identifier + '(?:\.<\w+>)?' + typeidentifier = identifier + r'(?:\.<\w+>)?' flags = re.DOTALL | re.MULTILINE tokens = { @@ -232,7 +232,7 @@ class MxmlLexer(RegexLexer): (r'/?\s*>', Name.Tag, '#pop'), ], 'attr': [ - ('\s+', Text), + (r'\s+', Text), ('".*?"', String, '#pop'), ("'.*?'", String, '#pop'), (r'[^\s>]+', String, '#pop'), diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py index cb200b9e..0e726339 100644 --- a/pygments/lexers/agile.py +++ b/pygments/lexers/agile.py @@ -5,7 +5,7 @@ Just export lexer classes previously contained in this module. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/algebra.py b/pygments/lexers/algebra.py index 15d68842..ba2a61a9 100644 --- a/pygments/lexers/algebra.py +++ b/pygments/lexers/algebra.py @@ -5,7 +5,7 @@ Lexers for computer algebra systems. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/ambient.py b/pygments/lexers/ambient.py index 53f3a5e1..7d42d12a 100644 --- a/pygments/lexers/ambient.py +++ b/pygments/lexers/ambient.py @@ -5,7 +5,7 @@ Lexers for AmbientTalk language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/ampl.py b/pygments/lexers/ampl.py index d439cb19..21e7847f 100644 --- a/pygments/lexers/ampl.py +++ b/pygments/lexers/ampl.py @@ -3,9 +3,9 @@ pygments.lexers.ampl ~~~~~~~~~~~~~~~~~~~~ - Lexers for the ampl language. <http://ampl.com/> + Lexers for the AMPL language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -18,7 +18,7 @@ __all__ = ['AmplLexer'] class AmplLexer(RegexLexer): """ - For AMPL source code. + For `AMPL <http://ampl.com/>`_ source code. .. versionadded:: 2.2 """ diff --git a/pygments/lexers/apl.py b/pygments/lexers/apl.py index b3414cc0..26962504 100644 --- a/pygments/lexers/apl.py +++ b/pygments/lexers/apl.py @@ -5,7 +5,7 @@ Lexers for APL. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -71,14 +71,14 @@ class APLLexer(RegexLexer): # # Numbers # ======= - (u'¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞)' - u'([Jj]¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞))?', + (u'¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞)' + u'([Jj]¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞))?', Number), # # Operators # ========== - (u'[\.\\\/⌿⍀¨⍣⍨⍠⍤∘]', Name.Attribute), # closest token type - (u'[+\-×÷⌈⌊∣|⍳?*⍟○!⌹<≤=>≥≠≡≢∊⍷∪∩~∨∧⍱⍲⍴,⍪⌽⊖⍉↑↓⊂⊃⌷⍋⍒⊤⊥⍕⍎⊣⊢⍁⍂≈⌸⍯↗]', + (u'[\\.\\\\\\/⌿⍀¨⍣⍨⍠⍤∘]', Name.Attribute), # closest token type + (u'[+\\-×÷⌈⌊∣|⍳?*⍟○!⌹<≤=>≥≠≡≢∊⍷∪∩~∨∧⍱⍲⍴,⍪⌽⊖⍉↑↓⊂⊃⌷⍋⍒⊤⊥⍕⍎⊣⊢⍁⍂≈⌸⍯↗]', Operator), # # Constant diff --git a/pygments/lexers/archetype.py b/pygments/lexers/archetype.py index 5d4eb9aa..68ec5c04 100644 --- a/pygments/lexers/archetype.py +++ b/pygments/lexers/archetype.py @@ -14,7 +14,7 @@ Contributed by Thomas Beale <https://github.com/wolandscat>, <https://bitbucket.org/thomas_beale>. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py index 5b8cab80..87dcff38 100644 --- a/pygments/lexers/asm.py +++ b/pygments/lexers/asm.py @@ -5,7 +5,7 @@ Lexers for assembly languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -20,7 +20,7 @@ from pygments.token import Text, Name, Number, String, Comment, Punctuation, \ __all__ = ['GasLexer', 'ObjdumpLexer', 'DObjdumpLexer', 'CppObjdumpLexer', 'CObjdumpLexer', 'HsailLexer', 'LlvmLexer', 'NasmLexer', - 'NasmObjdumpLexer', 'TasmLexer', 'Ca65Lexer'] + 'NasmObjdumpLexer', 'TasmLexer', 'Ca65Lexer', 'Dasm16Lexer'] class GasLexer(RegexLexer): @@ -35,7 +35,7 @@ class GasLexer(RegexLexer): #: optional Comment or Whitespace string = r'"(\\"|[^"])*"' char = r'[\w$.@-]' - identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)' + identifier = r'(?:[a-zA-Z$_]' + char + r'*|\.' + char + '+)' number = r'(?:0[xX][a-zA-Z0-9]+|\d+)' tokens = { @@ -53,6 +53,7 @@ class GasLexer(RegexLexer): ('@' + identifier, Name.Attribute), (number, Number.Integer), (r'[\r\n]+', Text, '#pop'), + (r'[;#].*?\n', Comment, '#pop'), include('punctuation'), include('whitespace') @@ -76,6 +77,7 @@ class GasLexer(RegexLexer): ('$'+number, Number.Integer), (r"$'(.|\\')'", String.Char), (r'[\r\n]+', Text, '#pop'), + (r'[;#].*?\n', Comment, '#pop'), include('punctuation'), include('whitespace') @@ -256,7 +258,7 @@ class HsailLexer(RegexLexer): (r'0[xX][a-fA-F0-9]+', Number.Hex), (ieeefloat, Number.Float), (float, Number.Float), - ('\d+', Number.Integer), + (r'\d+', Number.Integer), (r'[=<>{}\[\]()*.,:;!]|x\b', Punctuation) ], @@ -350,7 +352,7 @@ class LlvmLexer(RegexLexer): include('whitespace'), # Before keywords, because keywords are valid label names :(... - (identifier + '\s*:', Name.Label), + (identifier + r'\s*:', Name.Label), include('keyword'), @@ -375,54 +377,63 @@ class LlvmLexer(RegexLexer): 'keyword': [ # Regular keywords (words(( - 'begin', 'end', 'true', 'false', 'declare', 'define', 'global', - 'constant', 'private', 'linker_private', 'internal', - 'available_externally', 'linkonce', 'linkonce_odr', 'weak', - 'weak_odr', 'appending', 'dllimport', 'dllexport', 'common', - 'default', 'hidden', 'protected', 'extern_weak', 'external', - 'thread_local', 'zeroinitializer', 'undef', 'null', 'to', 'tail', - 'target', 'triple', 'datalayout', 'volatile', 'nuw', 'nsw', 'nnan', - 'ninf', 'nsz', 'arcp', 'fast', 'exact', 'inbounds', 'align', - 'addrspace', 'section', 'alias', 'module', 'asm', 'sideeffect', - 'gc', 'dbg', 'linker_private_weak', 'attributes', 'blockaddress', - 'initialexec', 'localdynamic', 'localexec', 'prefix', 'unnamed_addr', - 'ccc', 'fastcc', 'coldcc', 'x86_stdcallcc', 'x86_fastcallcc', - 'arm_apcscc', 'arm_aapcscc', 'arm_aapcs_vfpcc', 'ptx_device', - 'ptx_kernel', 'intel_ocl_bicc', 'msp430_intrcc', 'spir_func', - 'spir_kernel', 'x86_64_sysvcc', 'x86_64_win64cc', 'x86_thiscallcc', - 'cc', 'c', 'signext', 'zeroext', 'inreg', 'sret', 'nounwind', - 'noreturn', 'noalias', 'nocapture', 'byval', 'nest', 'readnone', - 'readonly', 'inlinehint', 'noinline', 'alwaysinline', 'optsize', 'ssp', - 'sspreq', 'noredzone', 'noimplicitfloat', 'naked', 'builtin', 'cold', - 'nobuiltin', 'noduplicate', 'nonlazybind', 'optnone', 'returns_twice', - 'sanitize_address', 'sanitize_memory', 'sanitize_thread', 'sspstrong', - 'uwtable', 'returned', 'type', 'opaque', 'eq', 'ne', 'slt', 'sgt', - 'sle', 'sge', 'ult', 'ugt', 'ule', 'uge', 'oeq', 'one', 'olt', 'ogt', - 'ole', 'oge', 'ord', 'uno', 'ueq', 'une', 'x', 'acq_rel', 'acquire', - 'alignstack', 'atomic', 'catch', 'cleanup', 'filter', 'inteldialect', - 'max', 'min', 'monotonic', 'nand', 'personality', 'release', 'seq_cst', - 'singlethread', 'umax', 'umin', 'unordered', 'xchg', 'add', 'fadd', - 'sub', 'fsub', 'mul', 'fmul', 'udiv', 'sdiv', 'fdiv', 'urem', 'srem', - 'frem', 'shl', 'lshr', 'ashr', 'and', 'or', 'xor', 'icmp', 'fcmp', - 'phi', 'call', 'trunc', 'zext', 'sext', 'fptrunc', 'fpext', 'uitofp', - 'sitofp', 'fptoui', 'fptosi', 'inttoptr', 'ptrtoint', 'bitcast', - 'addrspacecast', 'select', 'va_arg', 'ret', 'br', 'switch', 'invoke', - 'unwind', 'unreachable', 'indirectbr', 'landingpad', 'resume', - 'malloc', 'alloca', 'free', 'load', 'store', 'getelementptr', - 'extractelement', 'insertelement', 'shufflevector', 'getresult', - 'extractvalue', 'insertvalue', 'atomicrmw', 'cmpxchg', 'fence', - 'allocsize', 'amdgpu_cs', 'amdgpu_gs', 'amdgpu_kernel', 'amdgpu_ps', - 'amdgpu_vs', 'any', 'anyregcc', 'argmemonly', 'avr_intrcc', - 'avr_signalcc', 'caller', 'catchpad', 'catchret', 'catchswitch', - 'cleanuppad', 'cleanupret', 'comdat', 'convergent', 'cxx_fast_tlscc', - 'deplibs', 'dereferenceable', 'dereferenceable_or_null', 'distinct', - 'exactmatch', 'externally_initialized', 'from', 'ghccc', 'hhvm_ccc', - 'hhvmcc', 'ifunc', 'inaccessiblemem_or_argmemonly', 'inaccessiblememonly', - 'inalloca', 'jumptable', 'largest', 'local_unnamed_addr', 'minsize', - 'musttail', 'noduplicates', 'none', 'nonnull', 'norecurse', 'notail', - 'preserve_allcc', 'preserve_mostcc', 'prologue', 'safestack', 'samesize', - 'source_filename', 'swiftcc', 'swifterror', 'swiftself', 'webkit_jscc', - 'within', 'writeonly', 'x86_intrcc', 'x86_vectorcallcc'), + 'acq_rel', 'acquire', 'add', 'addrspace', 'addrspacecast', 'afn', 'alias', + 'aliasee', 'align', 'alignLog2', 'alignstack', 'alloca', 'allocsize', 'allOnes', + 'alwaysinline', 'amdgpu_cs', 'amdgpu_es', 'amdgpu_gs', 'amdgpu_hs', + 'amdgpu_kernel', 'amdgpu_ls', 'amdgpu_ps', 'amdgpu_vs', 'and', 'any', + 'anyregcc', 'appending', 'arcp', 'argmemonly', 'args', 'arm_aapcs_vfpcc', + 'arm_aapcscc', 'arm_apcscc', 'ashr', 'asm', 'atomic', 'atomicrmw', 'attributes', + 'available_externally', 'avr_intrcc', 'avr_signalcc', 'bit', 'bitcast', + 'bitMask', 'blockaddress', 'br', 'branchFunnel', 'builtin', 'byArg', 'byte', + 'byteArray', 'byval', 'c', 'call', 'callee', 'caller', 'calls', 'catch', + 'catchpad', 'catchret', 'catchswitch', 'cc', 'ccc', 'cleanup', 'cleanuppad', + 'cleanupret', 'cmpxchg', 'cold', 'coldcc', 'comdat', 'common', 'constant', + 'contract', 'convergent', 'critical', 'cxx_fast_tlscc', 'datalayout', 'declare', + 'default', 'define', 'deplibs', 'dereferenceable', 'dereferenceable_or_null', + 'distinct', 'dllexport', 'dllimport', 'double', 'dso_local', 'dso_preemptable', + 'dsoLocal', 'eq', 'exact', 'exactmatch', 'extern_weak', 'external', + 'externally_initialized', 'extractelement', 'extractvalue', 'fadd', 'false', + 'fast', 'fastcc', 'fcmp', 'fdiv', 'fence', 'filter', 'flags', 'float', 'fmul', + 'fp128', 'fpext', 'fptosi', 'fptoui', 'fptrunc', 'frem', 'from', 'fsub', + 'funcFlags', 'function', 'gc', 'getelementptr', 'ghccc', 'global', 'guid', 'gv', + 'half', 'hash', 'hhvm_ccc', 'hhvmcc', 'hidden', 'hot', 'hotness', 'icmp', + 'ifunc', 'inaccessiblemem_or_argmemonly', 'inaccessiblememonly', 'inalloca', + 'inbounds', 'indir', 'indirectbr', 'info', 'initialexec', 'inline', + 'inlineBits', 'inlinehint', 'inrange', 'inreg', 'insertelement', 'insertvalue', + 'insts', 'intel_ocl_bicc', 'inteldialect', 'internal', 'inttoptr', 'invoke', + 'jumptable', 'kind', 'label', 'landingpad', 'largest', 'linkage', 'linkonce', + 'linkonce_odr', 'live', 'load', 'local_unnamed_addr', 'localdynamic', + 'localexec', 'lshr', 'max', 'metadata', 'min', 'minsize', 'module', 'monotonic', + 'msp430_intrcc', 'mul', 'musttail', 'naked', 'name', 'nand', 'ne', 'nest', + 'ninf', 'nnan', 'noalias', 'nobuiltin', 'nocapture', 'nocf_check', + 'noduplicate', 'noduplicates', 'noimplicitfloat', 'noinline', 'none', + 'nonlazybind', 'nonnull', 'norecurse', 'noRecurse', 'noredzone', 'noreturn', + 'notail', 'notEligibleToImport', 'nounwind', 'nsw', 'nsz', 'null', 'nuw', 'oeq', + 'offset', 'oge', 'ogt', 'ole', 'olt', 'one', 'opaque', 'optforfuzzing', + 'optnone', 'optsize', 'or', 'ord', 'path', 'personality', 'phi', 'ppc_fp128', + 'prefix', 'preserve_allcc', 'preserve_mostcc', 'private', 'prologue', + 'protected', 'ptrtoint', 'ptx_device', 'ptx_kernel', 'readnone', 'readNone', + 'readonly', 'readOnly', 'reassoc', 'refs', 'relbf', 'release', 'resByArg', + 'resume', 'ret', 'returnDoesNotAlias', 'returned', 'returns_twice', 'safestack', + 'samesize', 'sanitize_address', 'sanitize_hwaddress', 'sanitize_memory', + 'sanitize_thread', 'sdiv', 'section', 'select', 'seq_cst', 'sext', 'sge', 'sgt', + 'shadowcallstack', 'shl', 'shufflevector', 'sideeffect', 'signext', 'single', + 'singleImpl', 'singleImplName', 'sitofp', 'sizeM1', 'sizeM1BitWidth', 'sle', + 'slt', 'source_filename', 'speculatable', 'spir_func', 'spir_kernel', 'srem', + 'sret', 'ssp', 'sspreq', 'sspstrong', 'store', 'strictfp', 'sub', 'summaries', + 'summary', 'swiftcc', 'swifterror', 'swiftself', 'switch', 'syncscope', 'tail', + 'target', 'thread_local', 'to', 'token', 'triple', 'true', 'trunc', 'type', + 'typeCheckedLoadConstVCalls', 'typeCheckedLoadVCalls', 'typeid', 'typeIdInfo', + 'typeTestAssumeConstVCalls', 'typeTestAssumeVCalls', 'typeTestRes', 'typeTests', + 'udiv', 'ueq', 'uge', 'ugt', 'uitofp', 'ule', 'ult', 'umax', 'umin', 'undef', + 'une', 'uniformRetVal', 'uniqueRetVal', 'unknown', 'unnamed_addr', 'uno', + 'unordered', 'unreachable', 'unsat', 'unwind', 'urem', 'uselistorder', + 'uselistorder_bb', 'uwtable', 'va_arg', 'variable', 'vFuncId', + 'virtualConstProp', 'void', 'volatile', 'weak', 'weak_odr', 'webkit_jscc', + 'win64cc', 'within', 'wpdRes', 'wpdResolutions', 'writeonly', 'x', + 'x86_64_sysvcc', 'x86_fastcallcc', 'x86_fp80', 'x86_intrcc', 'x86_mmx', + 'x86_regcallcc', 'x86_stdcallcc', 'x86_thiscallcc', 'x86_vectorcallcc', 'xchg', + 'xor', 'zeroext', 'zeroinitializer', 'zext'), suffix=r'\b'), Keyword), # Types @@ -444,6 +455,10 @@ class NasmLexer(RegexLexer): filenames = ['*.asm', '*.ASM'] mimetypes = ['text/x-nasm'] + # Tasm uses the same file endings, but TASM is not as common as NASM, so + # we prioritize NASM higher by default + priority = 1.0 + identifier = r'[a-z$._?][\w$.?#@~]*' hexn = r'(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)' octn = r'[0-7]+q' @@ -457,9 +472,11 @@ class NasmLexer(RegexLexer): r'mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]') wordop = r'seg|wrt|strict' type = r'byte|[dq]?word' - directives = (r'BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|' + # Directives must be followed by whitespace, otherwise CPU will match + # cpuid for instance. + directives = (r'(?:BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|' r'ORG|ALIGN|STRUC|ENDSTRUC|COMMON|CPU|GROUP|UPPERCASE|IMPORT|' - r'EXPORT|LIBRARY|MODULE') + r'EXPORT|LIBRARY|MODULE)\s+') flags = re.IGNORECASE | re.MULTILINE tokens = { @@ -507,6 +524,11 @@ class NasmLexer(RegexLexer): ], } + def analyse_text(text): + # Probably TASM + if re.match(r'PROC', text, re.IGNORECASE): + return False + class NasmObjdumpLexer(ObjdumpLexer): """ @@ -601,6 +623,11 @@ class TasmLexer(RegexLexer): ], } + def analyse_text(text): + # See above + if re.match(r'PROC', text, re.I): + return True + class Ca65Lexer(RegexLexer): """ @@ -639,3 +666,109 @@ class Ca65Lexer(RegexLexer): # comments in GAS start with "#" if re.match(r'^\s*;', text, re.MULTILINE): return 0.9 + + +class Dasm16Lexer(RegexLexer): + """ + Simple lexer for DCPU-16 Assembly + + Check http://0x10c.com/doc/dcpu-16.txt + + .. versionadded:: 2.4 + """ + name = 'DASM16' + aliases = ['dasm16'] + filenames = ['*.dasm16', '*.dasm'] + mimetypes = ['text/x-dasm16'] + + INSTRUCTIONS = [ + 'SET', + 'ADD', 'SUB', + 'MUL', 'MLI', + 'DIV', 'DVI', + 'MOD', 'MDI', + 'AND', 'BOR', 'XOR', + 'SHR', 'ASR', 'SHL', + 'IFB', 'IFC', 'IFE', 'IFN', 'IFG', 'IFA', 'IFL', 'IFU', + 'ADX', 'SBX', + 'STI', 'STD', + 'JSR', + 'INT', 'IAG', 'IAS', 'RFI', 'IAQ', 'HWN', 'HWQ', 'HWI', + ] + + REGISTERS = [ + 'A', 'B', 'C', + 'X', 'Y', 'Z', + 'I', 'J', + 'SP', 'PC', 'EX', + 'POP', 'PEEK', 'PUSH' + ] + + # Regexes yo + char = r'[a-zA-Z$._0-9@]' + identifier = r'(?:[a-zA-Z$_]' + char + r'*|\.' + char + '+)' + number = r'[+-]?(?:0[xX][a-zA-Z0-9]+|\d+)' + binary_number = r'0b[01_]+' + instruction = r'(?i)(' + '|'.join(INSTRUCTIONS) + ')' + single_char = r"'\\?" + char + "'" + string = r'"(\\"|[^"])*"' + + def guess_identifier(lexer, match): + ident = match.group(0) + klass = Name.Variable if ident.upper() in lexer.REGISTERS else Name.Label + yield match.start(), klass, ident + + tokens = { + 'root': [ + include('whitespace'), + (':' + identifier, Name.Label), + (identifier + ':', Name.Label), + (instruction, Name.Function, 'instruction-args'), + (r'\.' + identifier, Name.Function, 'data-args'), + (r'[\r\n]+', Text) + ], + + 'numeric' : [ + (binary_number, Number.Integer), + (number, Number.Integer), + (single_char, String), + ], + + 'arg' : [ + (identifier, guess_identifier), + include('numeric') + ], + + 'deref' : [ + (r'\+', Punctuation), + (r'\]', Punctuation, '#pop'), + include('arg'), + include('whitespace') + ], + + 'instruction-line' : [ + (r'[\r\n]+', Text, '#pop'), + (r';.*?$', Comment, '#pop'), + include('whitespace') + ], + + 'instruction-args': [ + (r',', Punctuation), + (r'\[', Punctuation, 'deref'), + include('arg'), + include('instruction-line') + ], + + 'data-args' : [ + (r',', Punctuation), + include('numeric'), + (string, String), + include('instruction-line') + ], + + 'whitespace': [ + (r'\n', Text), + (r'\s+', Text), + (r';.*?\n', Comment) + ], + } diff --git a/pygments/lexers/automation.py b/pygments/lexers/automation.py index 3ef42e48..5f27b6c6 100644 --- a/pygments/lexers/automation.py +++ b/pygments/lexers/automation.py @@ -5,7 +5,7 @@ Lexers for automation scripting languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/basic.py b/pygments/lexers/basic.py index e6545ee6..372c8229 100644 --- a/pygments/lexers/basic.py +++ b/pygments/lexers/basic.py @@ -5,18 +5,21 @@ Lexers for BASIC like languages (other than VB.net). - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re from pygments.lexer import RegexLexer, bygroups, default, words, include -from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ - Number, Punctuation +from pygments.token import Comment, Error, Keyword, Name, Number, \ + Punctuation, Operator, String, Text, Whitespace +from pygments.lexers import _vbscript_builtins + __all__ = ['BlitzBasicLexer', 'BlitzMaxLexer', 'MonkeyLexer', 'CbmBasicV2Lexer', - 'QBasicLexer'] + 'QBasicLexer', 'VBScriptLexer', 'BBCBasicLexer'] + class BlitzMaxLexer(RegexLexer): @@ -498,3 +501,159 @@ class QBasicLexer(RegexLexer): def analyse_text(text): if '$DYNAMIC' in text or '$STATIC' in text: return 0.9 + + +class VBScriptLexer(RegexLexer): + """ + VBScript is scripting language that is modeled on Visual Basic. + + .. versionadded:: 2.4 + """ + name = 'VBScript' + aliases = ['vbscript'] + filenames = ['*.vbs', '*.VBS'] + flags = re.IGNORECASE + + tokens = { + 'root': [ + (r"'[^\n]*", Comment.Single), + (r'\s+', Whitespace), + ('"', String.Double, 'string'), + ('&h[0-9a-f]+', Number.Hex), + # Float variant 1, for example: 1., 1.e2, 1.2e3 + (r'[0-9]+\.[0-9]*(e[+-]?[0-9]+)?', Number.Float), + (r'\.[0-9]+(e[+-]?[0-9]+)?', Number.Float), # Float variant 2, for example: .1, .1e2 + (r'[0-9]+e[+-]?[0-9]+', Number.Float), # Float variant 3, for example: 123e45 + (r'\d+', Number.Integer), + ('#.+#', String), # date or time value + (r'(dim)(\s+)([a-z_][a-z0-9_]*)', + bygroups(Keyword.Declaration, Whitespace, Name.Variable), 'dim_more'), + (r'(function|sub)(\s+)([a-z_][a-z0-9_]*)', + bygroups(Keyword.Declaration, Whitespace, Name.Function)), + (r'(class)(\s+)([a-z_][a-z0-9_]*)', bygroups(Keyword.Declaration, Whitespace, Name.Class)), + (r'(const)(\s+)([a-z_][a-z0-9_]*)', bygroups(Keyword.Declaration, Whitespace, Name.Constant)), + (r'(end)(\s+)(class|function|if|property|sub|with)', bygroups(Keyword, Whitespace, Keyword)), + (r'(on)(\s+)(error)(\s+)(goto)(\s+)(0)', + bygroups(Keyword, Whitespace, Keyword, Whitespace, Keyword, Whitespace, Number.Integer)), + (r'(on)(\s+)(error)(\s+)(resume)(\s+)(next)', + bygroups(Keyword, Whitespace, Keyword, Whitespace, Keyword, Whitespace, Keyword)), + (r'(option)(\s+)(explicit)', bygroups(Keyword, Whitespace, Keyword)), + (r'(property)(\s+)(get|let|set)(\s+)([a-z_][a-z0-9_]*)', + bygroups(Keyword.Declaration, Whitespace, Keyword.Declaration, Whitespace, Name.Property)), + (r'rem\s.*[^\n]*', Comment.Single), + (words(_vbscript_builtins.KEYWORDS, suffix=r'\b'), Keyword), + (words(_vbscript_builtins.OPERATORS), Operator), + (words(_vbscript_builtins.OPERATOR_WORDS, suffix=r'\b'), Operator.Word), + (words(_vbscript_builtins.BUILTIN_CONSTANTS, suffix=r'\b'), Name.Constant), + (words(_vbscript_builtins.BUILTIN_FUNCTIONS, suffix=r'\b'), Name.Builtin), + (words(_vbscript_builtins.BUILTIN_VARIABLES, suffix=r'\b'), Name.Builtin), + (r'[a-z_][a-z0-9_]*', Name), + (r'\b_\n', Operator), + (words(r'(),.:'), Punctuation), + (r'.+(\n)?', Error) + ], + 'dim_more': [ + (r'(\s*)(,)(\s*)([a-z_][a-z0-9]*)', bygroups(Whitespace, Punctuation, Whitespace, Name.Variable)), + default('#pop'), + ], + 'string': [ + (r'[^"\n]+', String.Double), + (r'\"\"', String.Double), + (r'"', String.Double, '#pop'), + (r'\n', Error, '#pop'), # Unterminated string + ], + } + + +class BBCBasicLexer(RegexLexer): + """ + BBC Basic was supplied on the BBC Micro, and later Acorn RISC OS. + It is also used by BBC Basic For Windows. + + .. versionadded:: 2.4 + """ + base_keywords = ['OTHERWISE', 'AND', 'DIV', 'EOR', 'MOD', 'OR', 'ERROR', + 'LINE', 'OFF', 'STEP', 'SPC', 'TAB', 'ELSE', 'THEN', + 'OPENIN', 'PTR', 'PAGE', 'TIME', 'LOMEM', 'HIMEM', 'ABS', + 'ACS', 'ADVAL', 'ASC', 'ASN', 'ATN', 'BGET', 'COS', 'COUNT', + 'DEG', 'ERL', 'ERR', 'EVAL', 'EXP', 'EXT', 'FALSE', 'FN', + 'GET', 'INKEY', 'INSTR', 'INT', 'LEN', 'LN', 'LOG', 'NOT', + 'OPENUP', 'OPENOUT', 'PI', 'POINT', 'POS', 'RAD', 'RND', + 'SGN', 'SIN', 'SQR', 'TAN', 'TO', 'TRUE', 'USR', 'VAL', + 'VPOS', 'CHR$', 'GET$', 'INKEY$', 'LEFT$', 'MID$', + 'RIGHT$', 'STR$', 'STRING$', 'EOF', 'PTR', 'PAGE', 'TIME', + 'LOMEM', 'HIMEM', 'SOUND', 'BPUT', 'CALL', 'CHAIN', 'CLEAR', + 'CLOSE', 'CLG', 'CLS', 'DATA', 'DEF', 'DIM', 'DRAW', 'END', + 'ENDPROC', 'ENVELOPE', 'FOR', 'GOSUB', 'GOTO', 'GCOL', 'IF', + 'INPUT', 'LET', 'LOCAL', 'MODE', 'MOVE', 'NEXT', 'ON', + 'VDU', 'PLOT', 'PRINT', 'PROC', 'READ', 'REM', 'REPEAT', + 'REPORT', 'RESTORE', 'RETURN', 'RUN', 'STOP', 'COLOUR', + 'TRACE', 'UNTIL', 'WIDTH', 'OSCLI'] + + basic5_keywords = ['WHEN', 'OF', 'ENDCASE', 'ENDIF', 'ENDWHILE', 'CASE', + 'CIRCLE', 'FILL', 'ORIGIN', 'POINT', 'RECTANGLE', 'SWAP', + 'WHILE', 'WAIT', 'MOUSE', 'QUIT', 'SYS', 'INSTALL', + 'LIBRARY', 'TINT', 'ELLIPSE', 'BEATS', 'TEMPO', 'VOICES', + 'VOICE', 'STEREO', 'OVERLAY', 'APPEND', 'AUTO', 'CRUNCH', + 'DELETE', 'EDIT', 'HELP', 'LIST', 'LOAD', 'LVAR', 'NEW', + 'OLD', 'RENUMBER', 'SAVE', 'TEXTLOAD', 'TEXTSAVE', + 'TWIN', 'TWINO', 'INSTALL', 'SUM', 'BEAT'] + + + name = 'BBC Basic' + aliases = ['bbcbasic'] + filenames = ['*.bbc'] + + tokens = { + 'root': [ + (r"[0-9]+", Name.Label), + (r"(\*)([^\n]*)", + bygroups(Keyword.Pseudo, Comment.Special)), + (r"", Whitespace, 'code'), + ], + + 'code': [ + (r"(REM)([^\n]*)", + bygroups(Keyword.Declaration, Comment.Single)), + (r'\n', Whitespace, 'root'), + (r'\s+', Whitespace), + (r':', Comment.Preproc), + + # Some special cases to make functions come out nicer + (r'(DEF)(\s*)(FN|PROC)([A-Za-z_@][\w@]*)', + bygroups(Keyword.Declaration, Whitespace, Keyword.Declaration, Name.Function)), + (r'(FN|PROC)([A-Za-z_@][\w@]*)', + bygroups(Keyword, Name.Function)), + + (r'(GOTO|GOSUB|THEN|RESTORE)(\s*)(\d+)', + bygroups(Keyword, Whitespace, Name.Label)), + + (r'(TRUE|FALSE)', Keyword.Constant), + (r'(PAGE|LOMEM|HIMEM|TIME|WIDTH|ERL|ERR|REPORT\$|POS|VPOS|VOICES)', Keyword.Pseudo), + + (words(base_keywords), Keyword), + (words(basic5_keywords), Keyword), + + ('"', String.Double, 'string'), + + ('%[01]{1,32}', Number.Bin), + ('&[0-9a-f]{1,8}', Number.Hex), + + (r'[+-]?[0-9]+\.[0-9]*(E[+-]?[0-9]+)?', Number.Float), + (r'[+-]?\.[0-9]+(E[+-]?[0-9]+)?', Number.Float), + (r'[+-]?[0-9]+E[+-]?[0-9]+', Number.Float), + (r'[+-]?\d+', Number.Integer), + + (r'([A-Za-z_@][\w@]*[%$]?)', Name.Variable), + (r'([+\-]=|[$!|?+\-*/%^=><();]|>=|<=|<>|<<|>>|>>>|,)', Operator), + ], + 'string': [ + (r'[^"\n]+', String.Double), + (r'"', String.Double, '#pop'), + (r'\n', Error, 'root'), # Unterminated string + ], + } + + def analyse_text(text): + if text.startswith('10REM >') or text.startswith('REM >'): + return 0.9 diff --git a/pygments/lexers/bibtex.py b/pygments/lexers/bibtex.py index a6159f81..855254ef 100644 --- a/pygments/lexers/bibtex.py +++ b/pygments/lexers/bibtex.py @@ -5,7 +5,7 @@ Lexers for BibTeX bibliography data and styles - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -101,12 +101,12 @@ class BibTeXLexer(ExtendedRegexLexer): 'quoted-string': [ (r'\{', String, 'braced-string'), ('"', String, '#pop'), - ('[^\{\"]+', String), + (r'[^\{\"]+', String), ], 'braced-string': [ (r'\{', String, '#push'), (r'\}', String, '#pop'), - ('[^\{\}]+', String), + (r'[^\{\}]+', String), ], 'whitespace': [ (r'\s+', Text), @@ -154,7 +154,7 @@ class BSTLexer(RegexLexer): default('#pop'), ], 'whitespace': [ - ('\s+', Text), + (r'\s+', Text), ('%.*?$', Comment.SingleLine), ], } diff --git a/pygments/lexers/boa.py b/pygments/lexers/boa.py new file mode 100644 index 00000000..a57c0e4a --- /dev/null +++ b/pygments/lexers/boa.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.boa + ~~~~~~~~~~~~~~~~~~~ + + Lexers for the Boa language. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, words +from pygments.token import String, Comment, Keyword, Name, Number, Text, \ + Operator, Punctuation + +__all__ = ['BoaLexer'] + +line_re = re.compile('.*?\n') + + +class BoaLexer(RegexLexer): + """ + Lexer for the `Boa <http://boa.cs.iastate.edu/docs/>`_ language. + + .. versionadded:: 2.4 + """ + name = 'Boa' + aliases = ['boa'] + filenames = ['*.boa'] + + reserved = words( + ('input', 'output', 'of', 'weight', 'before', 'after', 'stop', + 'ifall', 'foreach', 'exists', 'function', 'break', 'switch', 'case', + 'visitor', 'default', 'return', 'visit', 'while', 'if', 'else'), + suffix=r'\b', prefix=r'\b') + keywords = words( + ('bottom', 'collection', 'maximum', 'mean', 'minimum', 'set', 'sum', + 'top', 'string', 'int', 'bool', 'float', 'time', 'false', 'true', + 'array', 'map', 'stack', 'enum', 'type'), suffix=r'\b', prefix=r'\b') + classes = words( + ('Project', 'ForgeKind', 'CodeRepository', 'Revision', 'RepositoryKind', + 'ChangedFile', 'FileKind', 'ASTRoot', 'Namespace', 'Declaration', 'Type', + 'Method', 'Variable', 'Statement', 'Expression', 'Modifier', + 'StatementKind', 'ExpressionKind', 'ModifierKind', 'Visibility', + 'TypeKind', 'Person', 'ChangeKind'), + suffix=r'\b', prefix=r'\b') + operators = ('->', ':=', ':', '=', '<<', '!', '++', '||', + '&&', '+', '-', '*', ">", "<") + string_sep = ('`', '\"') + built_in_functions = words( + ( + # Array functions + 'new', 'sort', + # Date & Time functions + 'yearof', 'dayofyear', 'hourof', 'minuteof', 'secondof', 'now', + 'addday', 'addmonth', 'addweek', 'addyear', 'dayofmonth', 'dayofweek', + 'dayofyear', 'formattime', 'trunctoday', 'trunctohour', 'trunctominute', + 'trunctomonth', 'trunctosecond', 'trunctoyear', + # Map functions + 'clear', 'haskey', 'keys', 'lookup', 'remove', 'values', + # Math functions + 'abs', 'acos', 'acosh', 'asin', 'asinh', 'atan', 'atan2', 'atanh', + 'ceil', 'cos', 'cosh', 'exp', 'floor', 'highbit', 'isfinite', 'isinf', + 'isnan', 'isnormal', 'log', 'log10', 'max', 'min', 'nrand', 'pow', + 'rand', 'round', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', 'trunc', + # Other functions + 'def', 'hash', 'len', + # Set functions + 'add', 'contains', 'remove', + # String functions + 'format', 'lowercase', 'match', 'matchposns', 'matchstrs', 'regex', + 'split', 'splitall', 'splitn', 'strfind', 'strreplace', 'strrfind', + 'substring', 'trim', 'uppercase', + # Type Conversion functions + 'bool', 'float', 'int', 'string', 'time', + # Domain-Specific functions + 'getast', 'getsnapshot', 'hasfiletype', 'isfixingrevision', 'iskind', + 'isliteral', + ), + prefix=r'\b', + suffix=r'\(') + + tokens = { + 'root': [ + (r'#.*?$', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline), + (reserved, Keyword.Reserved), + (built_in_functions, Name.Function), + (keywords, Keyword.Type), + (classes, Name.Classes), + (words(operators), Operator), + (r'[][(),;{}\\.]', Punctuation), + (r'"(\\\\|\\"|[^"])*"', String), + (r'`(\\\\|\\`|[^`])*`', String), + (words(string_sep), String.Delimeter), + (r'[a-zA-Z_]+', Name.Variable), + (r'[0-9]+', Number.Integer), + (r'\s+?', Text), # Whitespace + ] + } diff --git a/pygments/lexers/business.py b/pygments/lexers/business.py index 552f3d9c..37f161a0 100644 --- a/pygments/lexers/business.py +++ b/pygments/lexers/business.py @@ -5,7 +5,7 @@ Lexers for "business-oriented" languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/c_cpp.py b/pygments/lexers/c_cpp.py index 691f5ab4..e676a8af 100644 --- a/pygments/lexers/c_cpp.py +++ b/pygments/lexers/c_cpp.py @@ -5,7 +5,7 @@ Lexers for C/C++ languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -36,7 +36,7 @@ class CFamilyLexer(RegexLexer): tokens = { 'whitespace': [ # preprocessor directives: without whitespace - ('^#if\s+0', Comment.Preproc, 'if0'), + (r'^#if\s+0', Comment.Preproc, 'if0'), ('^#', Comment.Preproc, 'macro'), # or with whitespace ('^(' + _ws1 + r')(#if\s+0)', @@ -84,7 +84,7 @@ class CFamilyLexer(RegexLexer): prefix=r'__', suffix=r'\b'), Keyword.Reserved), (r'(true|false|NULL)\b', Name.Builtin), (r'([a-zA-Z_]\w*)(\s*)(:)(?!:)', bygroups(Name.Label, Text, Punctuation)), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'root': [ include('whitespace'), @@ -190,9 +190,9 @@ class CLexer(CFamilyLexer): priority = 0.1 def analyse_text(text): - if re.search('^\s*#include [<"]', text, re.MULTILINE): + if re.search(r'^\s*#include [<"]', text, re.MULTILINE): return 0.1 - if re.search('^\s*#ifn?def ', text, re.MULTILINE): + if re.search(r'^\s*#ifn?def ', text, re.MULTILINE): return 0.1 diff --git a/pygments/lexers/c_like.py b/pygments/lexers/c_like.py index 38827219..cc88dfb9 100644 --- a/pygments/lexers/c_like.py +++ b/pygments/lexers/c_like.py @@ -5,7 +5,7 @@ Lexers for other C-like languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -20,7 +20,7 @@ from pygments.lexers.c_cpp import CLexer, CppLexer from pygments.lexers import _mql_builtins __all__ = ['PikeLexer', 'NesCLexer', 'ClayLexer', 'ECLexer', 'ValaLexer', - 'CudaLexer', 'SwigLexer', 'MqlLexer', 'ArduinoLexer'] + 'CudaLexer', 'SwigLexer', 'MqlLexer', 'ArduinoLexer', 'CharmciLexer'] class PikeLexer(CppLexer): @@ -245,7 +245,7 @@ class ValaLexer(RegexLexer): 'ulong', 'unichar', 'ushort'), suffix=r'\b'), Keyword.Type), (r'(true|false|null)\b', Name.Builtin), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'root': [ include('whitespace'), @@ -344,7 +344,7 @@ class SwigLexer(CppLexer): # SWIG directives (r'(%[a-z_][a-z0-9_]*)', Name.Function), # Special variables - ('\$\**\&?\w+', Name), + (r'\$\**\&?\w+', Name), # Stringification / additional preprocessor directives (r'##*[a-zA-Z_]\w*', Comment.Preproc), inherit, @@ -412,6 +412,7 @@ class MqlLexer(CppLexer): ], } + class ArduinoLexer(CppLexer): """ For `Arduino(tm) <https://arduino.cc/>`_ source. @@ -514,17 +515,16 @@ class ArduinoLexer(CppLexer): 'peek', 'beep', 'rect', 'line', 'open', 'seek', 'fill', 'size', 'turn', 'stop', 'home', 'find', 'step', 'tone', 'sqrt', 'RSSI', 'SSID', 'end', 'bit', 'tan', 'cos', 'sin', 'pow', 'map', 'abs', 'max', 'min', 'get', 'run', 'put', - 'isAlphaNumeric', 'isAlpha', 'isAscii', 'isWhitespace', 'isControl', 'isDigit', - 'isGraph', 'isLowerCase', 'isPrintable', 'isPunct', 'isSpace', 'isUpperCase', - 'isHexadecimalDigit')) + 'isAlphaNumeric', 'isAlpha', 'isAscii', 'isWhitespace', 'isControl', 'isDigit', + 'isGraph', 'isLowerCase', 'isPrintable', 'isPunct', 'isSpace', 'isUpperCase', + 'isHexadecimalDigit')) # do not highlight suppress_highlight = set(( - 'namespace', 'template', 'mutable', 'using', 'asm', 'typeid', - 'typename', 'this', 'alignof', 'constexpr', 'decltype', 'noexcept', + 'namespace', 'template', 'mutable', 'using', 'asm', 'typeid', + 'typename', 'this', 'alignof', 'constexpr', 'decltype', 'noexcept', 'static_assert', 'thread_local', 'restrict')) - def get_tokens_unprocessed(self, text): for index, token, value in CppLexer.get_tokens_unprocessed(self, text): if value in self.structure: @@ -539,3 +539,33 @@ class ArduinoLexer(CppLexer): yield index, Name.Function, value else: yield index, token, value + + +class CharmciLexer(CppLexer): + """ + For `Charm++ <https://charm.cs.illinois.edu>`_ interface files (.ci). + + .. versionadded:: 2.4 + """ + + name = 'Charmci' + aliases = ['charmci'] + filenames = ['*.ci'] + + mimetypes = [] + + tokens = { + 'statements': [ + (r'(module)(\s+)', bygroups(Keyword, Text), 'classname'), + (words(('mainmodule', 'mainchare', 'chare', 'array', 'group', + 'nodegroup', 'message', 'conditional')), Keyword), + (words(('entry', 'aggregate', 'threaded', 'sync', 'exclusive', + 'nokeep', 'notrace', 'immediate', 'expedited', 'inline', + 'local', 'python', 'accel', 'readwrite', 'writeonly', + 'accelblock', 'memcritical', 'packed', 'varsize', + 'initproc', 'initnode', 'initcall', 'stacksize', + 'createhere', 'createhome', 'reductiontarget', 'iget', + 'nocopy', 'mutable', 'migratable', 'readonly')), Keyword), + inherit, + ], + } diff --git a/pygments/lexers/capnproto.py b/pygments/lexers/capnproto.py index 203523a1..01524046 100644 --- a/pygments/lexers/capnproto.py +++ b/pygments/lexers/capnproto.py @@ -5,7 +5,7 @@ Lexers for the Cap'n Proto schema language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -44,34 +44,34 @@ class CapnProtoLexer(RegexLexer): ], 'type': [ (r'[^][=;,(){}$]+', Name.Class), - (r'[[(]', Name.Class, 'parentype'), + (r'[\[(]', Name.Class, 'parentype'), default('#pop'), ], 'parentype': [ (r'[^][;()]+', Name.Class), - (r'[[(]', Name.Class, '#push'), + (r'[\[(]', Name.Class, '#push'), (r'[])]', Name.Class, '#pop'), default('#pop'), ], 'expression': [ (r'[^][;,(){}$]+', Literal), - (r'[[(]', Literal, 'parenexp'), + (r'[\[(]', Literal, 'parenexp'), default('#pop'), ], 'parenexp': [ (r'[^][;()]+', Literal), - (r'[[(]', Literal, '#push'), + (r'[\[(]', Literal, '#push'), (r'[])]', Literal, '#pop'), default('#pop'), ], 'annotation': [ (r'[^][;,(){}=:]+', Name.Attribute), - (r'[[(]', Name.Attribute, 'annexp'), + (r'[\[(]', Name.Attribute, 'annexp'), default('#pop'), ], 'annexp': [ (r'[^][;()]+', Name.Attribute), - (r'[[(]', Name.Attribute, '#push'), + (r'[\[(]', Name.Attribute, '#push'), (r'[])]', Name.Attribute, '#pop'), default('#pop'), ], diff --git a/pygments/lexers/chapel.py b/pygments/lexers/chapel.py index 55bf0e1e..f2cd80dd 100644 --- a/pygments/lexers/chapel.py +++ b/pygments/lexers/chapel.py @@ -5,7 +5,7 @@ Lexer for the Chapel language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -42,16 +42,26 @@ class ChapelLexer(RegexLexer): (r'(bool|complex|imag|int|opaque|range|real|string|uint)\b', Keyword.Type), (words(( - 'align', 'as', 'atomic', 'begin', 'break', 'by', 'cobegin', - 'coforall', 'continue', 'delete', 'dmapped', 'do', 'domain', - 'else', 'enum', 'except', 'export', 'extern', 'for', 'forall', - 'if', 'index', 'inline', 'iter', 'label', 'lambda', 'let', - 'local', 'new', 'noinit', 'on', 'only', 'otherwise', 'pragma', - 'private', 'public', 'reduce', 'require', 'return', 'scan', - 'select', 'serial', 'single', 'sparse', 'subdomain', 'sync', - 'then', 'use', 'when', 'where', 'while', 'with', 'yield', + 'align', 'as', 'atomic', + 'begin', 'borrowed', 'break', 'by', + 'catch', 'cobegin', 'coforall', 'continue', + 'delete', 'dmapped', 'do', 'domain', + 'else', 'enum', 'except', 'export', 'extern', + 'for', 'forall', + 'if', 'index', 'inline', + 'label', 'lambda', 'let', 'local', + 'new', 'noinit', + 'on', 'only', 'otherwise', 'override', 'owned', + 'pragma', 'private', 'prototype', 'public', + 'reduce', 'require', 'return', + 'scan', 'select', 'serial', 'shared', 'single', 'sparse', 'subdomain', 'sync', + 'then', 'throw', 'throws', 'try', + 'unmanaged', 'use', + 'when', 'where', 'while', 'with', + 'yield', 'zip'), suffix=r'\b'), Keyword), + (r'(iter)((?:\s)+)', bygroups(Keyword, Text), 'procname'), (r'(proc)((?:\s)+)', bygroups(Keyword, Text), 'procname'), (r'(class|module|record|union)(\s+)', bygroups(Keyword, Text), 'classname'), @@ -96,7 +106,7 @@ class ChapelLexer(RegexLexer): (r'[a-zA-Z_][\w$]*', Name.Class, '#pop'), ], 'procname': [ - (r'([a-zA-Z_][\w$]+|\~[a-zA-Z_][\w$]+|[+*/!~%<>=&^|\-]{1,2})', + (r'([a-zA-Z_][.\w$]*|\~[a-zA-Z_][.\w$]*|[+*/!~%<>=&^|\-]{1,2})', Name.Function, '#pop'), ], } diff --git a/pygments/lexers/clean.py b/pygments/lexers/clean.py index ba2569f6..59b183b8 100644 --- a/pygments/lexers/clean.py +++ b/pygments/lexers/clean.py @@ -5,14 +5,13 @@ Lexer for the Clean language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -from pygments.lexer import ExtendedRegexLexer, LexerContext, \ - bygroups, words, include, default -from pygments.token import Comment, Keyword, Literal, Name, Number, Operator, \ - Punctuation, String, Text, Whitespace +from pygments.lexer import ExtendedRegexLexer, words, include, bygroups +from pygments.token import Comment, Error, Keyword, Literal, Name, Number, \ + Operator, Punctuation, String, Whitespace __all__ = ['CleanLexer'] @@ -28,261 +27,152 @@ class CleanLexer(ExtendedRegexLexer): aliases = ['clean'] filenames = ['*.icl', '*.dcl'] - def get_tokens_unprocessed(self, text=None, context=None): - ctx = LexerContext(text, 0) - ctx.indent = 0 - return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context=ctx) + keywords = ( + 'case', 'ccall', 'class', 'code', 'code inline', 'derive', 'export', + 'foreign', 'generic', 'if', 'in', 'infix', 'infixl', 'infixr', + 'instance', 'let', 'of', 'otherwise', 'special', 'stdcall', 'where', + 'with') - def check_class_not_import(lexer, match, ctx): - if match.group(0) == 'import': - yield match.start(), Keyword.Namespace, match.group(0) - ctx.stack = ctx.stack[:-1] + ['fromimportfunc'] - else: - yield match.start(), Name.Class, match.group(0) - ctx.pos = match.end() + modulewords = ('implementation', 'definition', 'system') - def check_instance_class(lexer, match, ctx): - if match.group(0) == 'instance' or match.group(0) == 'class': - yield match.start(), Keyword, match.group(0) - else: - yield match.start(), Name.Function, match.group(0) - ctx.stack = ctx.stack + ['fromimportfunctype'] - ctx.pos = match.end() - - @staticmethod - def indent_len(text): - # Tabs are four spaces: - # https://svn.cs.ru.nl/repos/clean-platform/trunk/doc/STANDARDS.txt - text = text.replace('\n', '') - return len(text.replace('\t', ' ')), len(text) - - def store_indent(lexer, match, ctx): - ctx.indent, _ = CleanLexer.indent_len(match.group(0)) - ctx.pos = match.end() - yield match.start(), Text, match.group(0) - - def check_indent1(lexer, match, ctx): - indent, reallen = CleanLexer.indent_len(match.group(0)) - if indent > ctx.indent: - yield match.start(), Whitespace, match.group(0) - ctx.pos = match.start() + reallen + 1 - else: - ctx.indent = 0 - ctx.pos = match.start() - ctx.stack = ctx.stack[:-1] - yield match.start(), Whitespace, match.group(0)[1:] - - def check_indent2(lexer, match, ctx): - indent, reallen = CleanLexer.indent_len(match.group(0)) - if indent > ctx.indent: - yield match.start(), Whitespace, match.group(0) - ctx.pos = match.start() + reallen + 1 - else: - ctx.indent = 0 - ctx.pos = match.start() - ctx.stack = ctx.stack[:-2] - - def check_indent3(lexer, match, ctx): - indent, reallen = CleanLexer.indent_len(match.group(0)) - if indent > ctx.indent: - yield match.start(), Whitespace, match.group(0) - ctx.pos = match.start() + reallen + 1 - else: - ctx.indent = 0 - ctx.pos = match.start() - ctx.stack = ctx.stack[:-3] - yield match.start(), Whitespace, match.group(0)[1:] - if match.group(0) == '\n\n': - ctx.pos = ctx.pos + 1 - - def skip(lexer, match, ctx): - ctx.stack = ctx.stack[:-1] - ctx.pos = match.end() - yield match.start(), Comment, match.group(0) - - keywords = ('class', 'instance', 'where', 'with', 'let', 'let!', - 'in', 'case', 'of', 'infix', 'infixr', 'infixl', 'generic', - 'derive', 'otherwise', 'code', 'inline') + lowerId = r'[a-z`][\w\d`]*' + upperId = r'[A-Z`][\w\d`]*' + funnyId = r'[~@#\$%\^?!+\-*<>\\/|&=:]+' + scoreUpperId = r'_' + upperId + scoreLowerId = r'_' + lowerId + moduleId = r'[a-zA-Z_][a-zA-Z0-9_.`]+' + classId = '|'.join([lowerId, upperId, funnyId]) tokens = { - 'common': [ - (r';', Punctuation, '#pop'), - (r'//', Comment, 'singlecomment'), - ], 'root': [ - # Comments + include('comments'), + include('keywords'), + include('module'), + include('import'), + include('whitespace'), + include('literals'), + include('operators'), + include('delimiters'), + include('names'), + ], + 'whitespace': [ + (r'\s+', Whitespace), + ], + 'comments': [ (r'//.*\n', Comment.Single), - (r'(?s)/\*\*.*?\*/', Comment.Special), - (r'(?s)/\*.*?\*/', Comment.Multi), - - # Modules, imports, etc. - (r'\b((?:implementation|definition|system)\s+)?(module)(\s+)([\w`.]+)', - bygroups(Keyword.Namespace, Keyword.Namespace, Text, Name.Class)), - (r'(?<=\n)import(?=\s)', Keyword.Namespace, 'import'), - (r'(?<=\n)from(?=\s)', Keyword.Namespace, 'fromimport'), - - # Keywords - # We cannot use (?s)^|(?<=\s) as prefix, so need to repeat this - (words(keywords, prefix=r'(?<=\s)', suffix=r'(?=\s)'), Keyword), - (words(keywords, prefix=r'^', suffix=r'(?=\s)'), Keyword), - - # Function definitions - (r'(?=\{\|)', Whitespace, 'genericfunction'), - (r'(?<=\n)([ \t]*)([\w`$()=\-<>~*\^|+&%]+)((?:\s+\w)*)(\s*)(::)', - bygroups(store_indent, Name.Function, Keyword.Type, Whitespace, - Punctuation), - 'functiondefargs'), - - # Type definitions - (r'(?<=\n)([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'), - (r'^([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'), - - # Literals - (r'\'\\?.(?<!\\)\'', String.Char), - (r'\'\\\d+\'', String.Char), - (r'\'\\\\\'', String.Char), # (special case for '\\') - (r'[+\-~]?\s*\d+\.\d+(E[+\-~]?\d+)?\b', Number.Float), - (r'[+\-~]?\s*0[0-7]\b', Number.Oct), - (r'[+\-~]?\s*0x[0-9a-fA-F]\b', Number.Hex), - (r'[+\-~]?\s*\d+\b', Number.Integer), - (r'"', String.Double, 'doubleqstring'), - (words(('True', 'False'), prefix=r'(?<=\s)', suffix=r'(?=\s)'), - Literal), - - # Qualified names - (r'(\')([\w.]+)(\'\.)', - bygroups(Punctuation, Name.Namespace, Punctuation)), - - # Everything else is some name - (r'([\w`$%/?@]+\.?)*[\w`$%/?@]+', Name), - - # Punctuation - (r'[{}()\[\],:;.#]', Punctuation), - (r'[+\-=!<>|&~*\^/]', Operator), - (r'\\\\', Operator), - - # Lambda expressions - (r'\\.*?(->|\.|=)', Name.Function), - - # Whitespace - (r'\s', Whitespace), - - include('common'), + (r'/\*', Comment.Multi, 'comments.in'), + (r'/\*\*', Comment.Special, 'comments.in'), ], - 'fromimport': [ - include('common'), - (r'([\w`.]+)', check_class_not_import), - (r'\n', Whitespace, '#pop'), - (r'\s', Whitespace), + 'comments.in': [ + (r'\*\/', Comment.Multi, '#pop'), + (r'/\*', Comment.Multi, '#push'), + (r'[^*/]+', Comment.Multi), + (r'\*(?!/)', Comment.Multi), + (r'/', Comment.Multi), ], - 'fromimportfunc': [ - include('common'), - (r'(::)(\s+)([^,\s]+)', bygroups(Punctuation, Text, Keyword.Type)), - (r'([\w`$()=\-<>~*\^|+&%/]+)', check_instance_class), - (r',', Punctuation), - (r'\n', Whitespace, '#pop'), - (r'\s', Whitespace), + 'keywords': [ + (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword), ], - 'fromimportfunctype': [ - include('common'), - (r'[{(\[]', Punctuation, 'combtype'), - (r',', Punctuation, '#pop'), - (r'[:;.#]', Punctuation), - (r'\n', Whitespace, '#pop:2'), - (r'[^\S\n]+', Whitespace), - (r'\S+', Keyword.Type), + 'module': [ + (words(modulewords, prefix=r'\b', suffix=r'\b'), Keyword.Namespace), + (r'\bmodule\b', Keyword.Namespace, 'module.name'), ], - 'combtype': [ - include('common'), - (r'[})\]]', Punctuation, '#pop'), - (r'[{(\[]', Punctuation, '#pop'), - (r'[,:;.#]', Punctuation), - (r'\s+', Whitespace), - (r'\S+', Keyword.Type), + 'module.name': [ + include('whitespace'), + (moduleId, Name.Class, '#pop'), ], 'import': [ - include('common'), - (words(('from', 'import', 'as', 'qualified'), - prefix='(?<=\s)', suffix='(?=\s)'), Keyword.Namespace), - (r'[\w`.]+', Name.Class), - (r'\n', Whitespace, '#pop'), - (r',', Punctuation), - (r'[^\S\n]+', Whitespace), - ], - 'singlecomment': [ - (r'(.)(?=\n)', skip), - (r'.+(?!\n)', Comment), - ], - 'doubleqstring': [ - (r'[^\\"]+', String.Double), - (r'"', String.Double, '#pop'), - (r'\\.', String.Double), - ], - 'typedef': [ - include('common'), - (r'[\w`]+', Keyword.Type), - (r'[:=|(),\[\]{}!*]', Punctuation), - (r'->', Punctuation), - (r'\n(?=[^\s|])', Whitespace, '#pop'), - (r'\s', Whitespace), - (r'.', Keyword.Type), + (r'\b(import)\b(\s*)', bygroups(Keyword, Whitespace), 'import.module'), + (r'\b(from)\b(\s*)\b(' + moduleId + r')\b(\s*)\b(import)\b', + bygroups(Keyword, Whitespace, Name.Class, Whitespace, Keyword), + 'import.what'), + ], + 'import.module': [ + (r'\b(qualified)\b(\s*)', bygroups(Keyword, Whitespace)), + (r'(\s*)\b(as)\b', bygroups(Whitespace, Keyword), ('#pop', 'import.module.as')), + (moduleId, Name.Class), + (r'(\s*)(,)(\s*)', bygroups(Whitespace, Punctuation, Whitespace)), + (r'\s*', Whitespace, '#pop'), + ], + 'import.module.as': [ + include('whitespace'), + (lowerId, Name.Class, '#pop'), + (upperId, Name.Class, '#pop'), + ], + 'import.what': [ + (r'\b(class)\b(\s+)(' + classId + r')', + bygroups(Keyword, Whitespace, Name.Class), 'import.what.class'), + (r'\b(instance)(\s+)(' + classId + r')(\s+)', + bygroups(Keyword, Whitespace, Name.Class, Whitespace), 'import.what.instance'), + (r'(::)(\s*)\b(' + upperId + r')\b', + bygroups(Punctuation, Whitespace, Name.Class), 'import.what.type'), + (r'\b(generic)\b(\s+)\b(' + lowerId + '|' + upperId + r')\b', + bygroups(Keyword, Whitespace, Name)), + include('names'), + (r'(,)(\s+)', bygroups(Punctuation, Whitespace)), + (r'$', Whitespace, '#pop'), + include('whitespace'), + ], + 'import.what.class': [ + (r',', Punctuation, '#pop'), + (r'\(', Punctuation, 'import.what.class.members'), + (r'$', Whitespace, '#pop:2'), + include('whitespace'), ], - 'genericfunction': [ - include('common'), - (r'\{\|', Punctuation), - (r'\|\}', Punctuation, '#pop'), + 'import.what.class.members': [ (r',', Punctuation), - (r'->', Punctuation), - (r'(\s+of\s+)(\{)', bygroups(Keyword, Punctuation), 'genericftypes'), - (r'\s', Whitespace), - (r'[\w`\[\]{}!]+', Keyword.Type), - (r'[*()]', Punctuation), + (r'\.\.', Punctuation), + (r'\)', Punctuation, '#pop'), + include('names'), + ], + 'import.what.instance': [ + (r'[,)]', Punctuation, '#pop'), + (r'\(', Punctuation, 'import.what.instance'), + (r'$', Whitespace, '#pop:2'), + include('whitespace'), + include('names'), + ], + 'import.what.type': [ + (r',', Punctuation, '#pop'), + (r'[({]', Punctuation, 'import.what.type.consesandfields'), + (r'$', Whitespace, '#pop:2'), + include('whitespace'), ], - 'genericftypes': [ - include('common'), - (r'[\w`]+', Keyword.Type), + 'import.what.type.consesandfields': [ (r',', Punctuation), - (r'\s', Whitespace), - (r'\}', Punctuation, '#pop'), - ], - 'functiondefargs': [ - include('common'), - (r'\n(\s*)', check_indent1), - (r'[!{}()\[\],:;.#]', Punctuation), - (r'->', Punctuation, 'functiondefres'), - (r'^(?=\S)', Whitespace, '#pop'), - (r'\S', Keyword.Type), - (r'\s', Whitespace), - ], - 'functiondefres': [ - include('common'), - (r'\n(\s*)', check_indent2), - (r'^(?=\S)', Whitespace, '#pop:2'), - (r'[!{}()\[\],:;.#]', Punctuation), - (r'\|', Punctuation, 'functiondefclasses'), - (r'\S', Keyword.Type), - (r'\s', Whitespace), - ], - 'functiondefclasses': [ - include('common'), - (r'\n(\s*)', check_indent3), - (r'^(?=\S)', Whitespace, '#pop:3'), - (r'[,&]', Punctuation), - (r'\[', Punctuation, 'functiondefuniquneq'), - (r'[\w`$()=\-<>~*\^|+&%/{}\[\]@]', Name.Function, 'functionname'), - (r'\s+', Whitespace), - ], - 'functiondefuniquneq': [ - include('common'), - (r'[a-z]+', Keyword.Type), - (r'\s+', Whitespace), - (r'<=|,', Punctuation), - (r'\]', Punctuation, '#pop') - ], - 'functionname': [ - include('common'), - (r'[\w`$()=\-<>~*\^|+&%/]+', Name.Function), - (r'(?=\{\|)', Punctuation, 'genericfunction'), - default('#pop'), + (r'\.\.', Punctuation), + (r'[)}]', Punctuation, '#pop'), + include('names'), + ], + 'literals': [ + (r'\'([^\'\\]|\\(x[\da-fA-F]+|\d+|.))\'', Literal.Char), + (r'[+~-]?0[0-7]+\b', Number.Oct), + (r'[+~-]?\d+\.\d+(E[+-]?\d+)?', Number.Float), + (r'[+~-]?\d+\b', Number.Integer), + (r'[+~-]?0x[\da-fA-F]+\b', Number.Hex), + (r'True|False', Literal), + (r'"', String.Double, 'literals.stringd'), + ], + 'literals.stringd': [ + (r'[^\\"\n]+', String.Double), + (r'"', String.Double, '#pop'), + (r'\\.', String.Double), + (r'[$\n]', Error, '#pop'), + ], + 'operators': [ + (r'[-~@#\$%\^?!+*<>\\/|&=:\.]+', Operator), + (r'\b_+\b', Operator), + ], + 'delimiters': [ + (r'[,;(){}\[\]]', Punctuation), + (r'(\')([\w`.]+)(\')', + bygroups(Punctuation, Name.Class, Punctuation)), + ], + 'names': [ + (lowerId, Name), + (scoreLowerId, Name), + (funnyId, Name.Function), + (upperId, Name.Class), + (scoreUpperId, Name.Class), ] } diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py index ab52a370..0dab602e 100644 --- a/pygments/lexers/compiled.py +++ b/pygments/lexers/compiled.py @@ -5,7 +5,7 @@ Just export lexer classes previously contained in this module. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py index c39b1a52..717246af 100644 --- a/pygments/lexers/configs.py +++ b/pygments/lexers/configs.py @@ -5,7 +5,7 @@ Lexers for configuration file formats. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -15,12 +15,13 @@ from pygments.lexer import RegexLexer, default, words, bygroups, include, using from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Whitespace, Literal from pygments.lexers.shell import BashLexer +from pygments.lexers.data import JsonLexer __all__ = ['IniLexer', 'RegeditLexer', 'PropertiesLexer', 'KconfigLexer', 'Cfengine3Lexer', 'ApacheConfLexer', 'SquidConfLexer', 'NginxConfLexer', 'LighttpdConfLexer', 'DockerLexer', 'TerraformLexer', 'TermcapLexer', 'TerminfoLexer', - 'PkgConfigLexer', 'PacmanConfLexer'] + 'PkgConfigLexer', 'PacmanConfLexer', 'AugeasLexer', 'TOMLLexer'] class IniLexer(RegexLexer): @@ -300,7 +301,7 @@ class ApacheConfLexer(RegexLexer): 'root': [ (r'\s+', Text), (r'(#.*?)$', Comment), - (r'(<[^\s>]+)(?:(\s+)(.*?))?(>)', + (r'(<[^\s>]+)(?:(\s+)(.*))?(>)', bygroups(Name.Tag, Text, String, Name.Tag)), (r'([a-z]\w*)(\s+)', bygroups(Name.Builtin, Text), 'value'), @@ -539,20 +540,25 @@ class DockerLexer(RegexLexer): filenames = ['Dockerfile', '*.docker'] mimetypes = ['text/x-dockerfile-config'] - _keywords = (r'(?:FROM|MAINTAINER|CMD|EXPOSE|ENV|ADD|ENTRYPOINT|' - r'VOLUME|WORKDIR)') - + _keywords = (r'(?:FROM|MAINTAINER|EXPOSE|WORKDIR|USER|STOPSIGNAL)') + _bash_keywords = (r'(?:RUN|CMD|ENTRYPOINT|ENV|ARG|LABEL|ADD|COPY)') + _lb = r'(?:\s*\\?\s*)' # dockerfile line break regex flags = re.IGNORECASE | re.MULTILINE tokens = { 'root': [ - (r'^(ONBUILD)(\s+)(%s)\b' % (_keywords,), - bygroups(Name.Keyword, Whitespace, Keyword)), - (r'^(%s)\b(.*)' % (_keywords,), bygroups(Keyword, String)), (r'#.*', Comment), - (r'RUN', Keyword), # Rest of line falls through + (r'(ONBUILD)(%s)' % (_lb,), bygroups(Keyword, using(BashLexer))), + (r'(HEALTHCHECK)((%s--\w+=\w+%s)*)' % (_lb, _lb), + bygroups(Keyword, using(BashLexer))), + (r'(VOLUME|ENTRYPOINT|CMD|SHELL)(%s)(\[.*?\])' % (_lb,), + bygroups(Keyword, using(BashLexer), using(JsonLexer))), + (r'(LABEL|ENV|ARG)((%s\w+=\w+%s)*)' % (_lb, _lb), + bygroups(Keyword, using(BashLexer))), + (r'(%s|VOLUME)\b(.*)' % (_keywords), bygroups(Keyword, String)), + (r'(%s)' % (_bash_keywords,), Keyword), (r'(.*\\\n)*.+', using(BashLexer)), - ], + ] } @@ -568,6 +574,8 @@ class TerraformLexer(RegexLexer): filenames = ['*.tf'] mimetypes = ['application/x-tf', 'application/x-terraform'] + embedded_keywords = ('ingress', 'egress', 'listener', 'default', 'connection', 'alias', 'tags', 'lifecycle', 'timeouts') + tokens = { 'root': [ include('string'), @@ -584,9 +592,8 @@ class TerraformLexer(RegexLexer): (r'(.*?)(\s*)(=)', bygroups(Name.Attribute, Text, Operator)), (words(('variable', 'resource', 'provider', 'provisioner', 'module'), prefix=r'\b', suffix=r'\b'), Keyword.Reserved, 'function'), - (words(('ingress', 'egress', 'listener', 'default', 'connection', 'alias'), - prefix=r'\b', suffix=r'\b'), Keyword.Declaration), - ('\$\{', String.Interpol, 'var_builtin'), + (words(embedded_keywords, prefix=r'\b', suffix=r'\b'), Keyword.Declaration), + (r'\$\{', String.Interpol, 'var_builtin'), ], 'function': [ (r'(\s+)(".*")(\s+)', bygroups(Text, String, Text)), @@ -831,3 +838,97 @@ class PacmanConfLexer(RegexLexer): (r'.', Text), ], } + + +class AugeasLexer(RegexLexer): + """ + Lexer for `Augeas <http://augeas.net>`_. + + .. versionadded:: 2.4 + """ + name = 'Augeas' + aliases = ['augeas'] + filenames = ['*.aug'] + + tokens = { + 'root': [ + (r'(module)(\s*)([^\s=]+)', bygroups(Keyword.Namespace, Text, Name.Namespace)), + (r'(let)(\s*)([^\s=]+)', bygroups(Keyword.Declaration, Text, Name.Variable)), + (r'(del|store|value|counter|seq|key|label|autoload|incl|excl|transform|test|get|put)(\s+)', bygroups(Name.Builtin, Text)), + (r'(\()([^:]+)(\:)(unit|string|regexp|lens|tree|filter)(\))', bygroups(Punctuation, Name.Variable, Punctuation, Keyword.Type, Punctuation)), + (r'\(\*', Comment.Multiline, 'comment'), + (r'[*+\-.;=?|]', Operator), + (r'[()\[\]{}]', Operator), + (r'"', String.Double, 'string'), + (r'\/', String.Regex, 'regex'), + (r'([A-Z]\w*)(\.)(\w+)', bygroups(Name.Namespace, Punctuation, Name.Variable)), + (r'.', Name.Variable), + (r'\s', Text), + ], + 'string': [ + (r'\\.', String.Escape), + (r'[^"]', String.Double), + (r'"', String.Double, '#pop'), + ], + 'regex': [ + (r'\\.', String.Escape), + (r'[^/]', String.Regex), + (r'\/', String.Regex, '#pop'), + ], + 'comment': [ + (r'[^*)]', Comment.Multiline), + (r'\(\*', Comment.Multiline, '#push'), + (r'\*\)', Comment.Multiline, '#pop'), + (r'[)*]', Comment.Multiline) + ], + } + + +class TOMLLexer(RegexLexer): + """ + Lexer for `TOML <https://github.com/toml-lang/toml>`_, a simple language + for config files. + + .. versionadded:: 2.4 + """ + + name = 'TOML' + aliases = ['toml'] + filenames = ['*.toml'] + + tokens = { + 'root': [ + + # Basics, comments, strings + (r'\s+', Text), + (r'#.*?$', Comment.Single), + # Basic string + (r'"(\\\\|\\"|[^"])*"', String), + # Literal string + (r'\'\'\'(.*)\'\'\'', String), + (r'\'[^\']*\'', String), + (r'(true|false)$', Keyword.Constant), + (r'[a-zA-Z_][\w\-]*', Name), + + (r'\[.*?\]$', Keyword), + # Datetime + # TODO this needs to be expanded, as TOML is rather flexible: + # https://github.com/toml-lang/toml#offset-date-time + (r'\d{4}-\d{2}-\d{2}(?:T| )\d{2}:\d{2}:\d{2}(?:Z|[-+]\d{2}:\d{2})', Number.Integer), + + # Numbers + (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float), + (r'\d+[eE][+-]?[0-9]+j?', Number.Float), + # Handle +-inf, +-infinity, +-nan + (r'[+-]?(?:(inf(?:inity)?)|nan)', Number.Float), + (r'[+-]?\d+', Number.Integer), + + # Punctuation + (r'[]{}:(),;[]', Punctuation), + (r'\.', Punctuation), + + # Operators + (r'=', Operator) + + ] + } diff --git a/pygments/lexers/console.py b/pygments/lexers/console.py index 77bb72e5..ab93b7b8 100644 --- a/pygments/lexers/console.py +++ b/pygments/lexers/console.py @@ -5,7 +5,7 @@ Lexers for misc console output. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/crystal.py b/pygments/lexers/crystal.py index bea4833f..1fa7270c 100644 --- a/pygments/lexers/crystal.py +++ b/pygments/lexers/crystal.py @@ -5,7 +5,7 @@ Lexer for Crystal. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 858aa348..4f754d45 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -3,18 +3,18 @@ pygments.lexers.csound ~~~~~~~~~~~~~~~~~~~~~~ - Lexers for CSound languages. + Lexers for Csound languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re from pygments.lexer import RegexLexer, bygroups, default, include, using, words -from pygments.token import Comment, Keyword, Name, Number, Operator, Punctuation, \ - String, Text -from pygments.lexers._csound_builtins import OPCODES +from pygments.token import Comment, Error, Keyword, Name, Number, Operator, Punctuation, \ + String, Text, Whitespace +from pygments.lexers._csound_builtins import OPCODES, DEPRECATED_OPCODES from pygments.lexers.html import HtmlLexer from pygments.lexers.python import PythonLexer from pygments.lexers.scripting import LuaLexer @@ -25,74 +25,104 @@ newline = (r'((?:(?:;|//).*)*)(\n)', bygroups(Comment.Single, Text)) class CsoundLexer(RegexLexer): - # Subclasses must define a 'single-line string' state. tokens = { 'whitespace': [ (r'[ \t]+', Text), - (r'\\\n', Text), - (r'/[*](.|\n)*?[*]/', Comment.Multiline) + (r'/[*](?:.|\n)*?[*]/', Comment.Multiline), + (r'(?:;|//).*$', Comment.Single), + (r'(\\)(\n)', bygroups(Whitespace, Text)) ], - 'macro call': [ - (r'(\$\w+\.?)(\()', bygroups(Comment.Preproc, Punctuation), - 'function macro call'), - (r'\$\w+(\.|\b)', Comment.Preproc) - ], - 'function macro call': [ - (r"((?:\\['\)]|[^'\)])+)(')", bygroups(Comment.Preproc, Punctuation)), - (r"([^'\)]+)(\))", bygroups(Comment.Preproc, Punctuation), '#pop') + 'preprocessor directives': [ + (r'#(?:e(?:nd(?:if)?|lse)\b|##)|@@?[ \t]*\d+', Comment.Preproc), + (r'#include', Comment.Preproc, 'include directive'), + (r'#[ \t]*define', Comment.Preproc, 'define directive'), + (r'#(?:ifn?def|undef)\b', Comment.Preproc, 'macro directive') ], - 'whitespace or macro call': [ + 'include directive': [ include('whitespace'), - include('macro call') + (r'([^ \t]).*?\1', String, '#pop') ], - 'preprocessor directives': [ - (r'#(e(nd(if)?|lse)|ifn?def|undef)\b|##', Comment.Preproc), - (r'#include\b', Comment.Preproc, 'include'), - (r'#[ \t]*define\b', Comment.Preproc, 'macro name'), - (r'@+[ \t]*\d*', Comment.Preproc) + 'define directive': [ + (r'\n', Text), + include('whitespace'), + (r'([A-Z_a-z]\w*)(\()', bygroups(Comment.Preproc, Punctuation), + ('#pop', 'macro parameter name list')), + (r'[A-Z_a-z]\w*', Comment.Preproc, ('#pop', 'before macro body')) ], - - 'include': [ + 'macro parameter name list': [ include('whitespace'), - (r'"', String, 'single-line string') + (r'[A-Z_a-z]\w*', Comment.Preproc), + (r"['#]", Punctuation), + (r'\)', Punctuation, ('#pop', 'before macro body')) ], - - 'macro name': [ + 'before macro body': [ + (r'\n', Text), include('whitespace'), - (r'(\w+)(\()', bygroups(Comment.Preproc, Text), - 'function macro argument list'), - (r'\w+', Comment.Preproc, 'object macro definition after name') + (r'#', Punctuation, ('#pop', 'macro body')) + ], + 'macro body': [ + (r'(?:\\(?!#)|[^#\\]|\n)+', Comment.Preproc), + (r'\\#', Comment.Preproc), + (r'(?<!\\)#', Punctuation, '#pop') ], - 'object macro definition after name': [ + + 'macro directive': [ include('whitespace'), - (r'#', Punctuation, 'object macro replacement text') + (r'[A-Z_a-z]\w*', Comment.Preproc, '#pop') + ], + + 'macro uses': [ + (r'(\$[A-Z_a-z]\w*\.?)(\()', bygroups(Comment.Preproc, Punctuation), + 'macro parameter value list'), + (r'\$[A-Z_a-z]\w*(?:\.|\b)', Comment.Preproc) ], - 'object macro replacement text': [ - (r'(\\#|[^#])+', Comment.Preproc), - (r'#', Punctuation, '#pop:3') + 'macro parameter value list': [ + (r'(?:[^\'#"{()]|\{(?!\{))+', Comment.Preproc), + (r"['#]", Punctuation), + (r'"', String, 'macro parameter value quoted string'), + (r'\{\{', String, 'macro parameter value braced string'), + (r'\(', Comment.Preproc, 'macro parameter value parenthetical'), + (r'\)', Punctuation, '#pop') ], - 'function macro argument list': [ - (r"(\w+)(['#])", bygroups(Comment.Preproc, Punctuation)), - (r'(\w+)(\))', bygroups(Comment.Preproc, Punctuation), - 'function macro definition after name') + 'macro parameter value quoted string': [ + (r"\\[#'()]", Comment.Preproc), + (r"[#'()]", Error), + include('quoted string') ], - 'function macro definition after name': [ - (r'[ \t]+', Text), - (r'#', Punctuation, 'function macro replacement text') + 'macro parameter value braced string': [ + (r"\\[#'()]", Comment.Preproc), + (r"[#'()]", Error), + include('braced string') + ], + 'macro parameter value parenthetical': [ + (r'(?:[^\\()]|\\\))+', Comment.Preproc), + (r'\(', Comment.Preproc, '#push'), + (r'\)', Comment.Preproc, '#pop') ], - 'function macro replacement text': [ - (r'(\\#|[^#])+', Comment.Preproc), - (r'#', Punctuation, '#pop:4') + + 'whitespace and macro uses': [ + include('whitespace'), + include('macro uses') + ], + + 'numbers': [ + (r'\d+[Ee][+-]?\d+|(\d+\.\d*|\d*\.\d+)([Ee][+-]?\d+)?', Number.Float), + (r'(0[Xx])([0-9A-Fa-f]+)', bygroups(Keyword.Type, Number.Hex)), + (r'\d+', Number.Integer) + ], + + 'braced string': [ + # Do nothing. This must be defined in subclasses. ] } class CsoundScoreLexer(CsoundLexer): """ - For `Csound <http://csound.github.io>`_ scores. + For `Csound <https://csound.github.io>`_ scores. .. versionadded:: 2.1 """ @@ -102,47 +132,77 @@ class CsoundScoreLexer(CsoundLexer): filenames = ['*.sco'] tokens = { - 'partial statement': [ + 'root': [ + (r'\n', Text), + include('whitespace and macro uses'), include('preprocessor directives'), - (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float), - (r'0[xX][a-fA-F0-9]+', Number.Hex), - (r'\d+', Number.Integer), - (r'"', String, 'single-line string'), - (r'[+\-*/%^!=<>|&#~.]', Operator), - (r'[]()[]', Punctuation), - (r'\w+', Comment.Preproc) - ], - 'statement': [ - include('whitespace or macro call'), - newline + ('#pop',), - include('partial statement') + (r'[abCdefiqstvxy]', Keyword), + # There is also a w statement that is generated internally and should not be + # used; see https://github.com/csound/csound/issues/750. + + (r'z', Keyword.Constant), + # z is a constant equal to 800,000,000,000. 800 billion seconds is about + # 25,367.8 years. See also + # https://csound.github.io/docs/manual/ScoreTop.html and + # https://github.com/csound/csound/search?q=stof+path%3AEngine+filename%3Asread.c. + + (r'([nNpP][pP])(\d+)', bygroups(Keyword, Number.Integer)), + + (r'[mn]', Keyword, 'mark statement'), + + include('numbers'), + (r'[!+\-*/^%&|<>#~.]', Operator), + (r'[()\[\]]', Punctuation), + (r'"', String, 'quoted string'), + (r'\{', Comment.Preproc, 'loop after left brace'), ], - 'root': [ - newline, - include('whitespace or macro call'), - (r'[{}]', Punctuation, 'statement'), - (r'[abefimq-tv-z]|[nN][pP]?', Keyword, 'statement') + 'mark statement': [ + include('whitespace and macro uses'), + (r'[A-Z_a-z]\w*', Name.Label), + (r'\n', Text, '#pop') ], - 'single-line string': [ + 'quoted string': [ (r'"', String, '#pop'), - (r'[^\\"]+', String) + (r'[^"$]+', String), + include('macro uses'), + (r'[$]', String) + ], + + 'loop after left brace': [ + include('whitespace and macro uses'), + (r'\d+', Number.Integer, ('#pop', 'loop after repeat count')), + ], + 'loop after repeat count': [ + include('whitespace and macro uses'), + (r'[A-Z_a-z]\w*', Comment.Preproc, ('#pop', 'loop')) + ], + 'loop': [ + (r'\}', Comment.Preproc, '#pop'), + include('root') + ], + + # Braced strings are not allowed in Csound scores, but this is needed + # because the superclass includes it. + 'braced string': [ + (r'\}\}', String, '#pop'), + (r'[^}]|\}(?!\})', String) ] } class CsoundOrchestraLexer(CsoundLexer): """ - For `Csound <http://csound.github.io>`_ orchestras. + For `Csound <https://csound.github.io>`_ orchestras. .. versionadded:: 2.1 """ name = 'Csound Orchestra' aliases = ['csound', 'csound-orc'] - filenames = ['*.orc'] + filenames = ['*.orc', '*.udo'] user_defined_opcodes = set() @@ -152,159 +212,189 @@ class CsoundOrchestraLexer(CsoundLexer): yield match.start(), Name.Function, opcode def name_callback(lexer, match): - name = match.group(0) - if re.match('p\d+$', name) or name in OPCODES: + name = match.group(1) + if name in OPCODES or name in DEPRECATED_OPCODES: yield match.start(), Name.Builtin, name + if match.group(2): + yield match.start(2), Punctuation, match.group(2) + yield match.start(3), Keyword.Type, match.group(3) elif name in lexer.user_defined_opcodes: yield match.start(), Name.Function, name else: - nameMatch = re.search(r'^(g?[aikSw])(\w+)', name) + nameMatch = re.search(r'^(g?[afikSw])(\w+)', name) if nameMatch: yield nameMatch.start(1), Keyword.Type, nameMatch.group(1) yield nameMatch.start(2), Name, nameMatch.group(2) else: yield match.start(), Name, name + # If there's a trailing :V, for example, we want to keep this around + # and emit it as well, otherwise this lexer will not pass round-trip + # testing + if match.group(2): + yield match.start(2), Punctuation, match.group(2) + yield match.start(3), Name, match.group(3) + tokens = { - 'label': [ - (r'\b(\w+)(:)', bygroups(Name.Label, Punctuation)) - ], + 'root': [ + (r'\n', Text), - 'partial expression': [ + (r'^([ \t]*)(\w+)(:)(?:[ \t]+|$)', bygroups(Text, Name.Label, Punctuation)), + + include('whitespace and macro uses'), include('preprocessor directives'), - (r'\b(0dbfs|k(r|smps)|nchnls(_i)?|sr)\b', Name.Variable.Global), - (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float), - (r'0[xX][a-fA-F0-9]+', Number.Hex), - (r'\d+', Number.Integer), - (r'"', String, 'single-line string'), - (r'\{\{', String, 'multi-line string'), - (r'[+\-*/%^!=&|<>#~¬]', Operator), - (r'[](),?:[]', Punctuation), + + (r'\binstr\b', Keyword.Declaration, 'instrument numbers and identifiers'), + (r'\bopcode\b', Keyword.Declaration, 'after opcode keyword'), + (r'\b(?:end(?:in|op))\b', Keyword.Declaration), + + include('partial statements') + ], + + 'partial statements': [ + (r'\b(?:0dbfs|A4|k(?:r|smps)|nchnls(?:_i)?|sr)\b', Name.Variable.Global), + + include('numbers'), + + (r'\+=|-=|\*=|/=|<<|>>|<=|>=|==|!=|&&|\|\||[~¬]|[=!+\-*/^%&|<>#?:]', Operator), + (r'[(),\[\]]', Punctuation), + + (r'"', String, 'quoted string'), + (r'\{\{', String, 'braced string'), + (words(( - # Keywords 'do', 'else', 'elseif', 'endif', 'enduntil', 'fi', 'if', 'ithen', 'kthen', 'od', 'then', 'until', 'while', - # Opcodes that act as control structures - 'return', 'timout' ), prefix=r'\b', suffix=r'\b'), Keyword), - (words(('goto', 'igoto', 'kgoto', 'rigoto', 'tigoto'), - prefix=r'\b', suffix=r'\b'), Keyword, 'goto label'), - (words(('cggoto', 'cigoto', 'cingoto', 'ckgoto', 'cngoto'), - prefix=r'\b', suffix=r'\b'), Keyword, - ('goto label', 'goto expression')), - (words(('loop_ge', 'loop_gt', 'loop_le', 'loop_lt'), - prefix=r'\b', suffix=r'\b'), Keyword, - ('goto label', 'goto expression', 'goto expression', 'goto expression')), - (r'\bscoreline(_i)?\b', Name.Builtin, 'scoreline opcode'), - (r'\bpyl?run[it]?\b', Name.Builtin, 'python opcode'), - (r'\blua_(exec|opdef)\b', Name.Builtin, 'lua opcode'), - (r'\b[a-zA-Z_]\w*\b', name_callback) - ], - - 'expression': [ - include('whitespace or macro call'), - newline + ('#pop',), - include('partial expression') - ], + (words(('return', 'rireturn'), prefix=r'\b', suffix=r'\b'), Keyword.Pseudo), - 'root': [ - newline, - include('whitespace or macro call'), - (r'\binstr\b', Keyword, ('instrument block', 'instrument name list')), - (r'\bopcode\b', Keyword, ('opcode block', 'opcode parameter list', - 'opcode types', 'opcode types', 'opcode name')), - include('label'), - default('expression') - ], - - 'instrument name list': [ - include('whitespace or macro call'), - (r'\d+|\+?[a-zA-Z_]\w*', Name.Function), - (r',', Punctuation), - newline + ('#pop',) + (r'\b[ik]?goto\b', Keyword, 'goto label'), + (r'\b(r(?:einit|igoto)|tigoto)(\(|\b)', bygroups(Keyword.Pseudo, Punctuation), + 'goto label'), + (r'\b(c(?:g|in?|k|nk?)goto)(\(|\b)', bygroups(Keyword.Pseudo, Punctuation), + ('goto label', 'goto argument')), + (r'\b(timout)(\(|\b)', bygroups(Keyword.Pseudo, Punctuation), + ('goto label', 'goto argument', 'goto argument')), + (r'\b(loop_[gl][et])(\(|\b)', bygroups(Keyword.Pseudo, Punctuation), + ('goto label', 'goto argument', 'goto argument', 'goto argument')), + + (r'\bprintk?s\b', Name.Builtin, 'prints opcode'), + (r'\b(?:readscore|scoreline(?:_i)?)\b', Name.Builtin, 'Csound score opcode'), + (r'\bpyl?run[it]?\b', Name.Builtin, 'Python opcode'), + (r'\blua_(?:exec|opdef)\b', Name.Builtin, 'Lua opcode'), + (r'\bp\d+\b', Name.Variable.Instance), + (r'\b([A-Z_a-z]\w*)(?:(:)([A-Za-z]))?\b', name_callback) ], - 'instrument block': [ - newline, - include('whitespace or macro call'), - (r'\bendin\b', Keyword, '#pop'), - include('label'), - default('expression') + + 'instrument numbers and identifiers': [ + include('whitespace and macro uses'), + (r'\d+|[A-Z_a-z]\w*', Name.Function), + (r'[+,]', Punctuation), + (r'\n', Text, '#pop') ], - 'opcode name': [ - include('whitespace or macro call'), - (r'[a-zA-Z_]\w*', opcode_name_callback, '#pop') + 'after opcode keyword': [ + include('whitespace and macro uses'), + (r'[A-Z_a-z]\w*', opcode_name_callback, ('#pop', 'opcode type signatures')), + (r'\n', Text, '#pop') ], - 'opcode types': [ - include('whitespace or macro call'), - (r'0|[]afijkKoOpPStV[]+', Keyword.Type, '#pop'), - (r',', Punctuation) + 'opcode type signatures': [ + include('whitespace and macro uses'), + + # https://github.com/csound/csound/search?q=XIDENT+path%3AEngine+filename%3Acsound_orc.lex + (r'0|[afijkKoOpPStV\[\]]+', Keyword.Type), + + (r',', Punctuation), + (r'\n', Text, '#pop') ], - 'opcode parameter list': [ - include('whitespace or macro call'), - newline + ('#pop',) + + 'quoted string': [ + (r'"', String, '#pop'), + (r'[^\\"$%)]+', String), + include('macro uses'), + include('escape sequences'), + include('format specifiers'), + (r'[\\$%)]', String) ], - 'opcode block': [ - newline, - include('whitespace or macro call'), - (r'\bendop\b', Keyword, '#pop'), - include('label'), - default('expression') + 'braced string': [ + (r'\}\}', String, '#pop'), + (r'(?:[^\\%)}]|\}(?!\}))+', String), + include('escape sequences'), + include('format specifiers'), + (r'[\\%)]', String) + ], + 'escape sequences': [ + # https://github.com/csound/csound/search?q=unquote_string+path%3AEngine+filename%3Acsound_orc_compile.c + (r'\\(?:[\\abnrt"]|[0-7]{1,3})', String.Escape) + ], + # Format specifiers are highlighted in all strings, even though only + # fprintks https://csound.github.io/docs/manual/fprintks.html + # fprints https://csound.github.io/docs/manual/fprints.html + # printf/printf_i https://csound.github.io/docs/manual/printf.html + # printks https://csound.github.io/docs/manual/printks.html + # prints https://csound.github.io/docs/manual/prints.html + # sprintf https://csound.github.io/docs/manual/sprintf.html + # sprintfk https://csound.github.io/docs/manual/sprintfk.html + # work with strings that contain format specifiers. In addition, these + # opcodes’ handling of format specifiers is inconsistent: + # - fprintks, fprints, printks, and prints do accept %a and %A + # specifiers, but can’t accept %s specifiers. + # - printf, printf_i, sprintf, and sprintfk don’t accept %a and %A + # specifiers, but can accept %s specifiers. + # See https://github.com/csound/csound/issues/747 for more information. + 'format specifiers': [ + (r'%[#0\- +]*\d*(?:\.\d+)?[diuoxXfFeEgGaAcs]', String.Interpol), + (r'%%', String.Escape) + ], + + 'goto argument': [ + include('whitespace and macro uses'), + (r',', Punctuation, '#pop'), + include('partial statements') ], - 'goto label': [ - include('whitespace or macro call'), + include('whitespace and macro uses'), (r'\w+', Name.Label, '#pop'), default('#pop') ], - 'goto expression': [ - include('whitespace or macro call'), - (r',', Punctuation, '#pop'), - include('partial expression') - ], - 'single-line string': [ - include('macro call'), - (r'"', String, '#pop'), - # From https://github.com/csound/csound/blob/develop/Opcodes/fout.c#L1405 - (r'%\d*(\.\d+)?[cdhilouxX]', String.Interpol), - (r'%[!%nNrRtT]|[~^]|\\([\\aAbBnNrRtT"]|[0-7]{1,3})', String.Escape), - (r'[^\\"~$%\^\n]+', String), - (r'[\\"~$%\^\n]', String) + 'prints opcode': [ + include('whitespace and macro uses'), + (r'"', String, 'prints quoted string'), + default('#pop') ], - 'multi-line string': [ - (r'\}\}', String, '#pop'), - (r'[^}]+|\}(?!\})', String) + 'prints quoted string': [ + (r'\\\\[aAbBnNrRtT]', String.Escape), + (r'%[!nNrRtT]|[~^]{1,2}', String.Escape), + include('quoted string') ], - 'scoreline opcode': [ - include('whitespace or macro call'), - (r'\{\{', String, 'scoreline'), - default('#pop') + 'Csound score opcode': [ + include('whitespace and macro uses'), + (r'\{\{', String, 'Csound score'), + (r'\n', Text, '#pop') ], - 'scoreline': [ + 'Csound score': [ (r'\}\}', String, '#pop'), (r'([^}]+)|\}(?!\})', using(CsoundScoreLexer)) ], - 'python opcode': [ - include('whitespace or macro call'), - (r'\{\{', String, 'python'), - default('#pop') + 'Python opcode': [ + include('whitespace and macro uses'), + (r'\{\{', String, 'Python'), + (r'\n', Text, '#pop') ], - 'python': [ + 'Python': [ (r'\}\}', String, '#pop'), (r'([^}]+)|\}(?!\})', using(PythonLexer)) ], - 'lua opcode': [ - include('whitespace or macro call'), - (r'"', String, 'single-line string'), - (r'\{\{', String, 'lua'), - (r',', Punctuation), - default('#pop') + 'Lua opcode': [ + include('whitespace and macro uses'), + (r'\{\{', String, 'Lua'), + (r'\n', Text, '#pop') ], - 'lua': [ + 'Lua': [ (r'\}\}', String, '#pop'), (r'([^}]+)|\}(?!\})', using(LuaLexer)) ] @@ -313,7 +403,7 @@ class CsoundOrchestraLexer(CsoundLexer): class CsoundDocumentLexer(RegexLexer): """ - For `Csound <http://csound.github.io>`_ documents. + For `Csound <https://csound.github.io>`_ documents. .. versionadded:: 2.1 """ @@ -331,15 +421,18 @@ class CsoundDocumentLexer(RegexLexer): # be XML files. tokens = { 'root': [ - newline, (r'/[*](.|\n)*?[*]/', Comment.Multiline), - (r'[^<&;/]+', Text), + (r'(?:;|//).*$', Comment.Single), + (r'[^/;<]+|/(?!/)', Text), + (r'<\s*CsInstruments', Name.Tag, ('orchestra', 'tag')), (r'<\s*CsScore', Name.Tag, ('score', 'tag')), - (r'<\s*[hH][tT][mM][lL]', Name.Tag, ('HTML', 'tag')), + (r'<\s*[Hh][Tt][Mm][Ll]', Name.Tag, ('HTML', 'tag')), + (r'<\s*[\w:.-]+', Name.Tag, 'tag'), (r'<\s*/\s*[\w:.-]+\s*>', Name.Tag) ], + 'orchestra': [ (r'<\s*/\s*CsInstruments\s*>', Name.Tag, '#pop'), (r'(.|\n)+?(?=<\s*/\s*CsInstruments\s*>)', using(CsoundOrchestraLexer)) @@ -349,9 +442,10 @@ class CsoundDocumentLexer(RegexLexer): (r'(.|\n)+?(?=<\s*/\s*CsScore\s*>)', using(CsoundScoreLexer)) ], 'HTML': [ - (r'<\s*/\s*[hH][tT][mM][lL]\s*>', Name.Tag, '#pop'), - (r'(.|\n)+?(?=<\s*/\s*[hH][tT][mM][lL]\s*>)', using(HtmlLexer)) + (r'<\s*/\s*[Hh][Tt][Mm][Ll]\s*>', Name.Tag, '#pop'), + (r'(.|\n)+?(?=<\s*/\s*[Hh][Tt][Mm][Ll]\s*>)', using(HtmlLexer)) ], + 'tag': [ (r'\s+', Text), (r'[\w.:-]+\s*=', Name.Attribute, 'attr'), diff --git a/pygments/lexers/css.py b/pygments/lexers/css.py index 29d83707..4c77efe0 100644 --- a/pygments/lexers/css.py +++ b/pygments/lexers/css.py @@ -5,7 +5,7 @@ Lexers for CSS and related stylesheet formats. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -125,7 +125,7 @@ _css_properties = ( 'wrap-flow', 'wrap-inside', 'wrap-through', 'writing-mode', 'z-index', ) -# List of keyword values obtained from: +# List of keyword values obtained from: # http://cssvalues.com/ _keyword_values = ( 'absolute', 'alias', 'all', 'all-petite-caps', 'all-scroll', @@ -263,7 +263,7 @@ _time_units = ( 's', 'ms', ) _all_units = _angle_units + _frequency_units + _length_units + \ - _resolution_units + _time_units + _resolution_units + _time_units class CssLexer(RegexLexer): @@ -322,16 +322,18 @@ class CssLexer(RegexLexer): include('urls'), (r'('+r'|'.join(_functional_notation_keyword_values)+r')(\()', bygroups(Name.Builtin, Punctuation), 'function-start'), - (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'), + (r'([a-zA-Z_][\w-]+)(\()', + bygroups(Name.Function, Punctuation), 'function-start'), (words(_keyword_values, suffix=r'\b'), Keyword.Constant), (words(_other_keyword_values, suffix=r'\b'), Keyword.Constant), (words(_color_keywords, suffix=r'\b'), Keyword.Constant), - (words(_css_properties, suffix=r'\b'), Keyword), # for transition-property etc. + # for transition-property etc. + (words(_css_properties, suffix=r'\b'), Keyword), (r'\!important', Comment.Preproc), (r'/\*(?:.|\n)*?\*/', Comment), include('numeric-values'), - + (r'[~^*!%&<>|+=@:./?-]+', Operator), (r'[\[\](),]+', Punctuation), (r'"(\\\\|\\"|[^"])*"', String.Double), @@ -351,7 +353,8 @@ class CssLexer(RegexLexer): # function-start may be entered recursively (r'(' + r'|'.join(_functional_notation_keyword_values) + r')(\()', bygroups(Name.Builtin, Punctuation), 'function-start'), - (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'), + (r'([a-zA-Z_][\w-]+)(\()', + bygroups(Name.Function, Punctuation), 'function-start'), (r'/\*(?:.|\n)*?\*/', Comment), include('numeric-values'), @@ -373,8 +376,8 @@ class CssLexer(RegexLexer): 'numeric-values': [ (r'\#[a-zA-Z0-9]{1,6}', Number.Hex), (r'[+\-]?[0-9]*[.][0-9]+', Number.Float, 'numeric-end'), - (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'), - ], + (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'), + ], 'numeric-end': [ (words(_all_units, suffix=r'\b'), Keyword.Type), (r'%', Keyword.Type), @@ -466,9 +469,9 @@ common_sass_tokens = { ], 'string-single': [ - (r"(\\.|#(?=[^\n{])|[^\n'#])+", String.Double), + (r"(\\.|#(?=[^\n{])|[^\n'#])+", String.Single), (r'#\{', String.Interpol, 'interpolation'), - (r"'", String.Double, '#pop'), + (r"'", String.Single, '#pop'), ], 'string-url': [ diff --git a/pygments/lexers/d.py b/pygments/lexers/d.py index 09e6fe87..b14f7dcd 100644 --- a/pygments/lexers/d.py +++ b/pygments/lexers/d.py @@ -5,7 +5,7 @@ Lexers for D languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/dalvik.py b/pygments/lexers/dalvik.py index c211f13e..0e39bb96 100644 --- a/pygments/lexers/dalvik.py +++ b/pygments/lexers/dalvik.py @@ -5,7 +5,7 @@ Pygments lexers for Dalvik VM-related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/data.py b/pygments/lexers/data.py index 296366c2..46ca7340 100644 --- a/pygments/lexers/data.py +++ b/pygments/lexers/data.py @@ -5,7 +5,7 @@ Lexers for data file format. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -205,7 +205,7 @@ class YamlLexer(ExtendedRegexLexer): bygroups(Text, Number), 'ignored-line'), ], - # the %YAG directive + # the %TAG directive 'tag-directive': [ # a tag handle and the corresponding prefix (r'([ ]+)(!|![\w-]*!)' @@ -218,7 +218,7 @@ class YamlLexer(ExtendedRegexLexer): 'indentation': [ # trailing whitespaces are ignored (r'[ ]*$', something(Text), '#pop:2'), - # whitespaces preceeding block collection indicators + # whitespaces preceding block collection indicators (r'[ ]+(?=[?:-](?:[ ]|$))', save_indent(Text)), # block collection indicators (r'[?:-](?=[ ]|$)', set_indent(Punctuation.Indicator)), @@ -232,6 +232,9 @@ class YamlLexer(ExtendedRegexLexer): (r'[ ]*(?=#|$)', something(Text), '#pop'), # whitespaces separating tokens (r'[ ]+', Text), + # key with colon + (r'''([^#,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''', + bygroups(Name.Tag, set_indent(Punctuation, implicit=True))), # tags, anchors and aliases, include('descriptors'), # block collections and scalars @@ -250,7 +253,7 @@ class YamlLexer(ExtendedRegexLexer): (r'!<[\w#;/?:@&=+$,.!~*\'()\[\]%-]+>', Keyword.Type), # a tag in the form '!', '!suffix' or '!handle!suffix' (r'!(?:[\w-]+!)?' - r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]+', Keyword.Type), + r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]*', Keyword.Type), # an anchor (r'&[\w-]+', Name.Label), # an alias @@ -308,6 +311,9 @@ class YamlLexer(ExtendedRegexLexer): # a flow mapping indicated by '{' and '}' 'flow-mapping': [ + # key with colon + (r'''([^,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''', + bygroups(Name.Tag, Punctuation)), # include flow collection rules include('flow-collection'), # the closing indicator diff --git a/pygments/lexers/diff.py b/pygments/lexers/diff.py index f7019440..5ff8375c 100644 --- a/pygments/lexers/diff.py +++ b/pygments/lexers/diff.py @@ -5,7 +5,7 @@ Lexers for diff/patch formats. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py index 4e2bc8ab..458a9eb4 100644 --- a/pygments/lexers/dotnet.py +++ b/pygments/lexers/dotnet.py @@ -5,7 +5,7 @@ Lexers for .net languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re @@ -58,7 +58,7 @@ class CSharpLexer(RegexLexer): # http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf levels = { - 'none': '@?[_a-zA-Z]\w*', + 'none': r'@?[_a-zA-Z]\w*', 'basic': ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' + '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc', 'Cf', 'Mn', 'Mc') + ']*'), @@ -171,7 +171,7 @@ class NemerleLexer(RegexLexer): # http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf levels = { - 'none': '@?[_a-zA-Z]\w*', + 'none': r'@?[_a-zA-Z]\w*', 'basic': ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' + '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc', 'Cf', 'Mn', 'Mc') + ']*'), @@ -352,13 +352,13 @@ class BooLexer(RegexLexer): ('[*/]', Comment.Multiline) ], 'funcname': [ - ('[a-zA-Z_]\w*', Name.Function, '#pop') + (r'[a-zA-Z_]\w*', Name.Function, '#pop') ], 'classname': [ - ('[a-zA-Z_]\w*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'namespace': [ - ('[a-zA-Z_][\w.]*', Name.Namespace, '#pop') + (r'[a-zA-Z_][\w.]*', Name.Namespace, '#pop') ] } @@ -413,7 +413,7 @@ class VbNetLexer(RegexLexer): 'Static', 'Step', 'Stop', 'SyncLock', 'Then', 'Throw', 'To', 'True', 'Try', 'TryCast', 'Wend', 'Using', 'When', 'While', 'Widening', 'With', 'WithEvents', 'WriteOnly'), - prefix='(?<!\.)', suffix=r'\b'), Keyword), + prefix=r'(?<!\.)', suffix=r'\b'), Keyword), (r'(?<!\.)End\b', Keyword, 'end'), (r'(?<!\.)(Dim|Const)\b', Keyword, 'dim'), (r'(?<!\.)(Function|Sub|Property)(\s+)', @@ -541,16 +541,13 @@ class VbNetAspxLexer(DelegatingLexer): # Very close to functional.OcamlLexer class FSharpLexer(RegexLexer): """ - For the F# language (version 3.0). - - AAAAACK Strings - http://research.microsoft.com/en-us/um/cambridge/projects/fsharp/manual/spec.html#_Toc335818775 + For the `F# language <https://fsharp.org/>`_ (version 3.0). .. versionadded:: 1.5 """ - name = 'FSharp' - aliases = ['fsharp'] + name = 'F#' + aliases = ['fsharp', 'f#'] filenames = ['*.fs', '*.fsi'] mimetypes = ['text/x-fsharp'] @@ -574,10 +571,10 @@ class FSharpLexer(RegexLexer): 'virtual', 'volatile', ] keyopts = [ - '!=', '#', '&&', '&', '\(', '\)', '\*', '\+', ',', '-\.', - '->', '-', '\.\.', '\.', '::', ':=', ':>', ':', ';;', ';', '<-', - '<\]', '<', '>\]', '>', '\?\?', '\?', '\[<', '\[\|', '\[', '\]', - '_', '`', '\{', '\|\]', '\|', '\}', '~', '<@@', '<@', '=', '@>', '@@>', + '!=', '#', '&&', '&', r'\(', r'\)', r'\*', r'\+', ',', r'-\.', + '->', '-', r'\.\.', r'\.', '::', ':=', ':>', ':', ';;', ';', '<-', + r'<\]', '<', r'>\]', '>', r'\?\?', r'\?', r'\[<', r'\[\|', r'\[', r'\]', + '_', '`', r'\{', r'\|\]', r'\|', r'\}', '~', '<@@', '<@', '=', '@>', '@@>', ] operators = r'[!$%&*+\./:<=>?@^|~-]' diff --git a/pygments/lexers/dsls.py b/pygments/lexers/dsls.py index a1426bd6..eb338d79 100644 --- a/pygments/lexers/dsls.py +++ b/pygments/lexers/dsls.py @@ -5,7 +5,7 @@ Lexers for various domain-specific languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -40,9 +40,9 @@ class ProtoBufLexer(RegexLexer): (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single), (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment.Multiline), (words(( - 'import', 'option', 'optional', 'required', 'repeated', 'default', - 'packed', 'ctype', 'extensions', 'to', 'max', 'rpc', 'returns', - 'oneof'), prefix=r'\b', suffix=r'\b'), + 'import', 'option', 'optional', 'required', 'repeated', + 'reserved', 'default', 'packed', 'ctype', 'extensions', 'to', + 'max', 'rpc', 'returns', 'oneof'), prefix=r'\b', suffix=r'\b'), Keyword), (words(( 'int32', 'int64', 'uint32', 'uint64', 'sint32', 'sint64', @@ -66,7 +66,7 @@ class ProtoBufLexer(RegexLexer): (r'[+-=]', Operator), (r'([a-zA-Z_][\w.]*)([ \t]*)(=)', bygroups(Name.Attribute, Text, Operator)), - ('[a-zA-Z_][\w.]*', Name), + (r'[a-zA-Z_][\w.]*', Name), ], 'package': [ (r'[a-zA-Z_]\w*', Name.Namespace, '#pop'), @@ -300,7 +300,7 @@ class PuppetLexer(RegexLexer): ], 'names': [ - ('[a-zA-Z_]\w*', Name.Attribute), + (r'[a-zA-Z_]\w*', Name.Attribute), (r'(\$\S+)(\[)(\S+)(\])', bygroups(Name.Variable, Punctuation, String, Punctuation)), (r'\$\S+', Name.Variable), @@ -688,7 +688,7 @@ class CrmshLexer(RegexLexer): (r'([\w#$-]+)(?:(:)(%s))?(?![\w#$-])' % rsc_role_action, bygroups(Name, Punctuation, Operator.Word)), # punctuation - (r'(\\(?=\n)|[[\](){}/:@])', Punctuation), + (r'(\\(?=\n)|[\[\](){}/:@])', Punctuation), (r'\s+|\n', Whitespace), ], } diff --git a/pygments/lexers/dylan.py b/pygments/lexers/dylan.py index f61bb60d..b9a13e27 100644 --- a/pygments/lexers/dylan.py +++ b/pygments/lexers/dylan.py @@ -5,7 +5,7 @@ Lexers for the Dylan language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -179,10 +179,10 @@ class DylanLexer(RegexLexer): (valid_name + ':', Keyword), # class names - (r'<' + valid_name + '>', Name.Class), + ('<' + valid_name + '>', Name.Class), # define variable forms. - (r'\*' + valid_name + '\*', Name.Variable.Global), + (r'\*' + valid_name + r'\*', Name.Variable.Global), # define constant forms. (r'\$' + valid_name, Name.Constant), @@ -260,7 +260,7 @@ class DylanConsoleLexer(Lexer): mimetypes = ['text/x-dylan-console'] _line_re = re.compile('.*?\n') - _prompt_re = re.compile('\?| ') + _prompt_re = re.compile(r'\?| ') def get_tokens_unprocessed(self, text): dylexer = DylanLexer(**self.options) diff --git a/pygments/lexers/ecl.py b/pygments/lexers/ecl.py index bd80ad19..c695c18c 100644 --- a/pygments/lexers/ecl.py +++ b/pygments/lexers/ecl.py @@ -5,7 +5,7 @@ Lexers for the ECL language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/eiffel.py b/pygments/lexers/eiffel.py index a90ab0a5..3f7ce55d 100644 --- a/pygments/lexers/eiffel.py +++ b/pygments/lexers/eiffel.py @@ -5,7 +5,7 @@ Lexer for the Eiffel language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/elm.py b/pygments/lexers/elm.py index 0fa36367..86dafc7c 100644 --- a/pygments/lexers/elm.py +++ b/pygments/lexers/elm.py @@ -5,7 +5,7 @@ Lexer for the Elm programming language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -27,7 +27,7 @@ class ElmLexer(RegexLexer): filenames = ['*.elm'] mimetypes = ['text/x-elm'] - validName = r'[a-z_][a-zA-Z_\']*' + validName = r'[a-z_][a-zA-Z0-9_\']*' specialName = r'^main ' diff --git a/pygments/lexers/erlang.py b/pygments/lexers/erlang.py index 9e7f85c1..07a46c80 100644 --- a/pygments/lexers/erlang.py +++ b/pygments/lexers/erlang.py @@ -5,7 +5,7 @@ Lexers for Erlang. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -163,7 +163,7 @@ class ErlangShellLexer(Lexer): filenames = ['*.erl-sh'] mimetypes = ['text/x-erl-shellsession'] - _prompt_re = re.compile(r'\d+>(?=\s|\Z)') + _prompt_re = re.compile(r'(?:\([\w@_.]+\))?\d+>(?=\s|\Z)') def get_tokens_unprocessed(self, text): erlexer = ErlangLexer(**self.options) @@ -344,7 +344,7 @@ class ElixirLexer(RegexLexer): op1_re = "|".join(re.escape(s) for s in OPERATORS1) ops_re = r'(?:%s|%s|%s)' % (op3_re, op2_re, op1_re) punctuation_re = "|".join(re.escape(s) for s in PUNCTUATION) - alnum = '\w' + alnum = r'\w' name_re = r'(?:\.\.\.|[a-z_]%s*[!?]?)' % alnum modname_re = r'[A-Z]%(alnum)s*(?:\.[A-Z]%(alnum)s*)*' % {'alnum': alnum} complex_name_re = r'(?:%s|%s|%s)' % (name_re, modname_re, ops_re) @@ -495,7 +495,7 @@ class ElixirConsoleLexer(Lexer): aliases = ['iex'] mimetypes = ['text/x-elixir-shellsession'] - _prompt_re = re.compile('(iex|\.{3})(\(\d+\))?> ') + _prompt_re = re.compile(r'(iex|\.{3})((?:\([\w@_.]+\))?\d+|\(\d+\))?> ') def get_tokens_unprocessed(self, text): exlexer = ElixirLexer(**self.options) diff --git a/pygments/lexers/esoteric.py b/pygments/lexers/esoteric.py index 793c28be..6946c902 100644 --- a/pygments/lexers/esoteric.py +++ b/pygments/lexers/esoteric.py @@ -5,7 +5,7 @@ Lexers for esoteric languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -245,7 +245,7 @@ class AheuiLexer(RegexLexer): Aheui_ is esoteric language based on Korean alphabets. - .. _Aheui:: http://aheui.github.io/ + .. _Aheui: http://aheui.github.io/ """ diff --git a/pygments/lexers/ezhil.py b/pygments/lexers/ezhil.py index ce1cdb2d..ee465885 100644 --- a/pygments/lexers/ezhil.py +++ b/pygments/lexers/ezhil.py @@ -5,7 +5,7 @@ Pygments lexers for Ezhil language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/factor.py b/pygments/lexers/factor.py index 09d85c27..4aed8f6e 100644 --- a/pygments/lexers/factor.py +++ b/pygments/lexers/factor.py @@ -5,7 +5,7 @@ Lexers for the Factor language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/fantom.py b/pygments/lexers/fantom.py index 3ea2177c..b1b0dd94 100644 --- a/pygments/lexers/fantom.py +++ b/pygments/lexers/fantom.py @@ -5,7 +5,7 @@ Lexer for the Fantom language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/felix.py b/pygments/lexers/felix.py index 8f0695b5..cf768a45 100644 --- a/pygments/lexers/felix.py +++ b/pygments/lexers/felix.py @@ -5,7 +5,7 @@ Lexer for the Felix language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/floscript.py b/pygments/lexers/floscript.py new file mode 100644 index 00000000..d5744331 --- /dev/null +++ b/pygments/lexers/floscript.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.floscript + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for FloScript + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, include +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['FloScriptLexer'] + + +class FloScriptLexer(RegexLexer): + """ + For `FloScript <https://github.com/ioflo/ioflo>`_ configuration language source code. + + .. versionadded:: 2.4 + """ + + name = 'FloScript' + aliases = ['floscript', 'flo'] + filenames = ['*.flo'] + + def innerstring_rules(ttype): + return [ + # the old style '%s' % (...) string formatting + (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' + '[hlL]?[E-GXc-giorsux%]', String.Interpol), + # backslashes, quotes and formatting signs must be parsed one at a time + (r'[^\\\'"%\n]+', ttype), + (r'[\'"\\]', ttype), + # unhandled string formatting sign + (r'%', ttype), + # newlines are an error (use "nl" state) + ] + + tokens = { + 'root': [ + (r'\n', Text), + (r'[^\S\n]+', Text), + + (r'[]{}:(),;[]', Punctuation), + (r'\\\n', Text), + (r'\\', Text), + (r'(to|by|with|from|per|for|cum|qua|via|as|at|in|of|on|re|is|if|be|into|' + r'and|not)\b', Operator.Word), + (r'!=|==|<<|>>|[-~+/*%=<>&^|.]', Operator), + (r'(load|init|server|logger|log|loggee|first|over|under|next|done|timeout|' + r'repeat|native|benter|enter|recur|exit|precur|renter|rexit|print|put|inc|' + r'copy|set|aux|rear|raze|go|let|do|bid|ready|start|stop|run|abort|use|flo|' + r'give|take)\b', Name.Builtin), + (r'(frame|framer|house)\b', Keyword), + ('"', String, 'string'), + + include('name'), + include('numbers'), + (r'#.+$', Comment.Singleline), + ], + 'string': [ + ('[^"]+', String), + ('"', String, '#pop'), + ], + 'numbers': [ + (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float), + (r'\d+[eE][+-]?[0-9]+j?', Number.Float), + (r'0[0-7]+j?', Number.Oct), + (r'0[bB][01]+', Number.Bin), + (r'0[xX][a-fA-F0-9]+', Number.Hex), + (r'\d+L', Number.Integer.Long), + (r'\d+j?', Number.Integer) + ], + + 'name': [ + (r'@[\w.]+', Name.Decorator), + (r'[a-zA-Z_]\w*', Name), + ], + } diff --git a/pygments/lexers/forth.py b/pygments/lexers/forth.py index a51f1b57..934bf478 100644 --- a/pygments/lexers/forth.py +++ b/pygments/lexers/forth.py @@ -3,7 +3,9 @@ pygments.lexers.forth ~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + Lexer for the Forth language. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/fortran.py b/pygments/lexers/fortran.py index 1a611c9d..4f5c3b25 100644 --- a/pygments/lexers/fortran.py +++ b/pygments/lexers/fortran.py @@ -5,7 +5,7 @@ Lexers for Fortran languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -158,6 +158,7 @@ class FortranLexer(RegexLexer): (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer), (r'[+-]?\d*\.\d+([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), (r'[+-]?\d+\.\d*([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), + (r'[+-]?\d+(\.\d*)?[ed][-+]?\d+(_[a-z]\w+)?', Number.Float), ], } diff --git a/pygments/lexers/foxpro.py b/pygments/lexers/foxpro.py index 7c0d2621..868a44d8 100644 --- a/pygments/lexers/foxpro.py +++ b/pygments/lexers/foxpro.py @@ -5,7 +5,7 @@ Simple lexer for Microsoft Visual FoxPro source code. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/freefem.py b/pygments/lexers/freefem.py new file mode 100644 index 00000000..c43b285d --- /dev/null +++ b/pygments/lexers/freefem.py @@ -0,0 +1,898 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.freefem + ~~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for FreeFem++ language. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, include, bygroups, inherit, words, \ + default +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +from pygments.lexers.c_cpp import CLexer, CppLexer +from pygments.lexers import _mql_builtins + +__all__ = ['FreeFemLexer'] + + +class FreeFemLexer(CppLexer): + """ + For `FreeFem++ <https://freefem.org/>`_ source. + + This is an extension of the CppLexer, as the FreeFem Language is a superset + of C++. + + .. versionadded:: 2.4 + """ + + name = 'Freefem' + aliases = ['freefem'] + filenames = ['*.edp'] + mimetypes = ['text/x-freefem'] + + # Language operators + operators = set(('+', '-', '*', '.*', '/', './', '%', '^', '^-1', ':', '\'')) + + # types + types = set(('bool', 'border', 'complex', 'dmatrix', 'fespace', 'func', 'gslspline', + 'ifstream', 'int', 'macro', 'matrix', 'mesh', 'mesh3', 'mpiComm', + 'mpiGroup', 'mpiRequest', 'NewMacro', 'EndMacro', 'ofstream', 'Pmmap', + 'problem', 'Psemaphore', 'real', 'solve', 'string', 'varf')) + + # finite element spaces + fespaces = set(('BDM1', 'BDM1Ortho', 'Edge03d', 'Edge13d', 'Edge23d', 'FEQF', 'HCT', + 'P0', 'P03d', 'P0Edge', 'P1', 'P13d', 'P1b', 'P1b3d', 'P1bl', 'P1bl3d', + 'P1dc', 'P1Edge', 'P1nc', 'P2', 'P23d', 'P2b', 'P2BR', 'P2dc', 'P2Edge', + 'P2h', 'P2Morley', 'P2pnc', 'P3', 'P3dc', 'P3Edge', 'P4', 'P4dc', + 'P4Edge', 'P5Edge', 'RT0', 'RT03d', 'RT0Ortho', 'RT1', 'RT1Ortho', + 'RT2', 'RT2Ortho')) + + # preprocessor + preprocessor = set(('ENDIFMACRO', 'include', 'IFMACRO', 'load')) + + # Language keywords + keywords = set(( + 'adj', + 'append', + 'area', + 'ARGV', + 'be', + 'binary', + 'BoundaryEdge', + 'bordermeasure', + 'CG', + 'Cholesky', + 'cin', + 'cout', + 'Crout', + 'default', + 'diag', + 'edgeOrientation', + 'endl', + 'false', + 'ffind', + 'FILE', + 'find', + 'fixed', + 'flush', + 'GMRES', + 'good', + 'hTriangle', + 'im', + 'imax', + 'imin', + 'InternalEdge', + 'l1', + 'l2', + 'label', + 'lenEdge', + 'length', + 'LINE', + 'linfty', + 'LU', + 'm', + 'max', + 'measure', + 'min', + 'mpiAnySource', + 'mpiBAND', + 'mpiBXOR', + 'mpiCommWorld', + 'mpiLAND', + 'mpiLOR', + 'mpiLXOR', + 'mpiMAX', + 'mpiMIN', + 'mpiPROD', + 'mpirank', + 'mpisize', + 'mpiSUM', + 'mpiUndefined', + 'n', + 'N', + 'nbe', + 'ndof', + 'ndofK', + 'noshowbase', + 'noshowpos', + 'notaregion', + 'nt', + 'nTonEdge', + 'nuEdge', + 'nuTriangle', + 'nv', + 'P', + 'pi', + 'precision', + 'qf1pE', + 'qf1pElump', + 'qf1pT', + 'qf1pTlump', + 'qfV1', + 'qfV1lump', + 'qf2pE', + 'qf2pT', + 'qf2pT4P1', + 'qfV2', + 'qf3pE', + 'qf4pE', + 'qf5pE', + 'qf5pT', + 'qfV5', + 'qf7pT', + 'qf9pT', + 'qfnbpE', + 'quantile', + 're', + 'region', + 'rfind', + 'scientific', + 'searchMethod', + 'setw', + 'showbase', + 'showpos', + 'sparsesolver', + 'sum', + 'tellp', + 'true', + 'UMFPACK', + 'unused', + 'whoinElement', + 'verbosity', + 'version', + 'volume', + 'x', + 'y', + 'z' + )) + + # Language shipped functions and class ( ) + functions = set(( + 'abs', + 'acos', + 'acosh', + 'adaptmesh', + 'adj', + 'AffineCG', + 'AffineGMRES', + 'arg', + 'asin', + 'asinh', + 'assert', + 'atan', + 'atan2', + 'atanh', + 'atof', + 'atoi', + 'BFGS', + 'broadcast', + 'buildlayers', + 'buildmesh', + 'ceil', + 'chi', + 'complexEigenValue', + 'copysign', + 'change', + 'checkmovemesh', + 'clock', + 'cmaes', + 'conj', + 'convect', + 'cos', + 'cosh', + 'cube', + 'd', + 'dd', + 'dfft', + 'diffnp', + 'diffpos', + 'dimKrylov', + 'dist', + 'dumptable', + 'dx', + 'dxx', + 'dxy', + 'dxz', + 'dy', + 'dyx', + 'dyy', + 'dyz', + 'dz', + 'dzx', + 'dzy', + 'dzz', + 'EigenValue', + 'emptymesh', + 'erf', + 'erfc', + 'exec', + 'exit', + 'exp', + 'fdim', + 'floor', + 'fmax', + 'fmin', + 'fmod', + 'freeyams', + 'getARGV', + 'getline', + 'gmshload', + 'gmshload3', + 'gslcdfugaussianP', + 'gslcdfugaussianQ', + 'gslcdfugaussianPinv', + 'gslcdfugaussianQinv', + 'gslcdfgaussianP', + 'gslcdfgaussianQ', + 'gslcdfgaussianPinv', + 'gslcdfgaussianQinv', + 'gslcdfgammaP', + 'gslcdfgammaQ', + 'gslcdfgammaPinv', + 'gslcdfgammaQinv', + 'gslcdfcauchyP', + 'gslcdfcauchyQ', + 'gslcdfcauchyPinv', + 'gslcdfcauchyQinv', + 'gslcdflaplaceP', + 'gslcdflaplaceQ', + 'gslcdflaplacePinv', + 'gslcdflaplaceQinv', + 'gslcdfrayleighP', + 'gslcdfrayleighQ', + 'gslcdfrayleighPinv', + 'gslcdfrayleighQinv', + 'gslcdfchisqP', + 'gslcdfchisqQ', + 'gslcdfchisqPinv', + 'gslcdfchisqQinv', + 'gslcdfexponentialP', + 'gslcdfexponentialQ', + 'gslcdfexponentialPinv', + 'gslcdfexponentialQinv', + 'gslcdfexppowP', + 'gslcdfexppowQ', + 'gslcdftdistP', + 'gslcdftdistQ', + 'gslcdftdistPinv', + 'gslcdftdistQinv', + 'gslcdffdistP', + 'gslcdffdistQ', + 'gslcdffdistPinv', + 'gslcdffdistQinv', + 'gslcdfbetaP', + 'gslcdfbetaQ', + 'gslcdfbetaPinv', + 'gslcdfbetaQinv', + 'gslcdfflatP', + 'gslcdfflatQ', + 'gslcdfflatPinv', + 'gslcdfflatQinv', + 'gslcdflognormalP', + 'gslcdflognormalQ', + 'gslcdflognormalPinv', + 'gslcdflognormalQinv', + 'gslcdfgumbel1P', + 'gslcdfgumbel1Q', + 'gslcdfgumbel1Pinv', + 'gslcdfgumbel1Qinv', + 'gslcdfgumbel2P', + 'gslcdfgumbel2Q', + 'gslcdfgumbel2Pinv', + 'gslcdfgumbel2Qinv', + 'gslcdfweibullP', + 'gslcdfweibullQ', + 'gslcdfweibullPinv', + 'gslcdfweibullQinv', + 'gslcdfparetoP', + 'gslcdfparetoQ', + 'gslcdfparetoPinv', + 'gslcdfparetoQinv', + 'gslcdflogisticP', + 'gslcdflogisticQ', + 'gslcdflogisticPinv', + 'gslcdflogisticQinv', + 'gslcdfbinomialP', + 'gslcdfbinomialQ', + 'gslcdfpoissonP', + 'gslcdfpoissonQ', + 'gslcdfgeometricP', + 'gslcdfgeometricQ', + 'gslcdfnegativebinomialP', + 'gslcdfnegativebinomialQ', + 'gslcdfpascalP', + 'gslcdfpascalQ', + 'gslinterpakima', + 'gslinterpakimaperiodic', + 'gslinterpcsplineperiodic', + 'gslinterpcspline', + 'gslinterpsteffen', + 'gslinterplinear', + 'gslinterppolynomial', + 'gslranbernoullipdf', + 'gslranbeta', + 'gslranbetapdf', + 'gslranbinomialpdf', + 'gslranexponential', + 'gslranexponentialpdf', + 'gslranexppow', + 'gslranexppowpdf', + 'gslrancauchy', + 'gslrancauchypdf', + 'gslranchisq', + 'gslranchisqpdf', + 'gslranerlang', + 'gslranerlangpdf', + 'gslranfdist', + 'gslranfdistpdf', + 'gslranflat', + 'gslranflatpdf', + 'gslrangamma', + 'gslrangammaint', + 'gslrangammapdf', + 'gslrangammamt', + 'gslrangammaknuth', + 'gslrangaussian', + 'gslrangaussianratiomethod', + 'gslrangaussianziggurat', + 'gslrangaussianpdf', + 'gslranugaussian', + 'gslranugaussianratiomethod', + 'gslranugaussianpdf', + 'gslrangaussiantail', + 'gslrangaussiantailpdf', + 'gslranugaussiantail', + 'gslranugaussiantailpdf', + 'gslranlandau', + 'gslranlandaupdf', + 'gslrangeometricpdf', + 'gslrangumbel1', + 'gslrangumbel1pdf', + 'gslrangumbel2', + 'gslrangumbel2pdf', + 'gslranlogistic', + 'gslranlogisticpdf', + 'gslranlognormal', + 'gslranlognormalpdf', + 'gslranlogarithmicpdf', + 'gslrannegativebinomialpdf', + 'gslranpascalpdf', + 'gslranpareto', + 'gslranparetopdf', + 'gslranpoissonpdf', + 'gslranrayleigh', + 'gslranrayleighpdf', + 'gslranrayleightail', + 'gslranrayleightailpdf', + 'gslrantdist', + 'gslrantdistpdf', + 'gslranlaplace', + 'gslranlaplacepdf', + 'gslranlevy', + 'gslranweibull', + 'gslranweibullpdf', + 'gslsfairyAi', + 'gslsfairyBi', + 'gslsfairyAiscaled', + 'gslsfairyBiscaled', + 'gslsfairyAideriv', + 'gslsfairyBideriv', + 'gslsfairyAiderivscaled', + 'gslsfairyBiderivscaled', + 'gslsfairyzeroAi', + 'gslsfairyzeroBi', + 'gslsfairyzeroAideriv', + 'gslsfairyzeroBideriv', + 'gslsfbesselJ0', + 'gslsfbesselJ1', + 'gslsfbesselJn', + 'gslsfbesselY0', + 'gslsfbesselY1', + 'gslsfbesselYn', + 'gslsfbesselI0', + 'gslsfbesselI1', + 'gslsfbesselIn', + 'gslsfbesselI0scaled', + 'gslsfbesselI1scaled', + 'gslsfbesselInscaled', + 'gslsfbesselK0', + 'gslsfbesselK1', + 'gslsfbesselKn', + 'gslsfbesselK0scaled', + 'gslsfbesselK1scaled', + 'gslsfbesselKnscaled', + 'gslsfbesselj0', + 'gslsfbesselj1', + 'gslsfbesselj2', + 'gslsfbesseljl', + 'gslsfbessely0', + 'gslsfbessely1', + 'gslsfbessely2', + 'gslsfbesselyl', + 'gslsfbesseli0scaled', + 'gslsfbesseli1scaled', + 'gslsfbesseli2scaled', + 'gslsfbesselilscaled', + 'gslsfbesselk0scaled', + 'gslsfbesselk1scaled', + 'gslsfbesselk2scaled', + 'gslsfbesselklscaled', + 'gslsfbesselJnu', + 'gslsfbesselYnu', + 'gslsfbesselInuscaled', + 'gslsfbesselInu', + 'gslsfbesselKnuscaled', + 'gslsfbesselKnu', + 'gslsfbessellnKnu', + 'gslsfbesselzeroJ0', + 'gslsfbesselzeroJ1', + 'gslsfbesselzeroJnu', + 'gslsfclausen', + 'gslsfhydrogenicR1', + 'gslsfdawson', + 'gslsfdebye1', + 'gslsfdebye2', + 'gslsfdebye3', + 'gslsfdebye4', + 'gslsfdebye5', + 'gslsfdebye6', + 'gslsfdilog', + 'gslsfmultiply', + 'gslsfellintKcomp', + 'gslsfellintEcomp', + 'gslsfellintPcomp', + 'gslsfellintDcomp', + 'gslsfellintF', + 'gslsfellintE', + 'gslsfellintRC', + 'gslsferfc', + 'gslsflogerfc', + 'gslsferf', + 'gslsferfZ', + 'gslsferfQ', + 'gslsfhazard', + 'gslsfexp', + 'gslsfexpmult', + 'gslsfexpm1', + 'gslsfexprel', + 'gslsfexprel2', + 'gslsfexpreln', + 'gslsfexpintE1', + 'gslsfexpintE2', + 'gslsfexpintEn', + 'gslsfexpintE1scaled', + 'gslsfexpintE2scaled', + 'gslsfexpintEnscaled', + 'gslsfexpintEi', + 'gslsfexpintEiscaled', + 'gslsfShi', + 'gslsfChi', + 'gslsfexpint3', + 'gslsfSi', + 'gslsfCi', + 'gslsfatanint', + 'gslsffermidiracm1', + 'gslsffermidirac0', + 'gslsffermidirac1', + 'gslsffermidirac2', + 'gslsffermidiracint', + 'gslsffermidiracmhalf', + 'gslsffermidirachalf', + 'gslsffermidirac3half', + 'gslsffermidiracinc0', + 'gslsflngamma', + 'gslsfgamma', + 'gslsfgammastar', + 'gslsfgammainv', + 'gslsftaylorcoeff', + 'gslsffact', + 'gslsfdoublefact', + 'gslsflnfact', + 'gslsflndoublefact', + 'gslsflnchoose', + 'gslsfchoose', + 'gslsflnpoch', + 'gslsfpoch', + 'gslsfpochrel', + 'gslsfgammaincQ', + 'gslsfgammaincP', + 'gslsfgammainc', + 'gslsflnbeta', + 'gslsfbeta', + 'gslsfbetainc', + 'gslsfgegenpoly1', + 'gslsfgegenpoly2', + 'gslsfgegenpoly3', + 'gslsfgegenpolyn', + 'gslsfhyperg0F1', + 'gslsfhyperg1F1int', + 'gslsfhyperg1F1', + 'gslsfhypergUint', + 'gslsfhypergU', + 'gslsfhyperg2F0', + 'gslsflaguerre1', + 'gslsflaguerre2', + 'gslsflaguerre3', + 'gslsflaguerren', + 'gslsflambertW0', + 'gslsflambertWm1', + 'gslsflegendrePl', + 'gslsflegendreP1', + 'gslsflegendreP2', + 'gslsflegendreP3', + 'gslsflegendreQ0', + 'gslsflegendreQ1', + 'gslsflegendreQl', + 'gslsflegendrePlm', + 'gslsflegendresphPlm', + 'gslsflegendrearraysize', + 'gslsfconicalPhalf', + 'gslsfconicalPmhalf', + 'gslsfconicalP0', + 'gslsfconicalP1', + 'gslsfconicalPsphreg', + 'gslsfconicalPcylreg', + 'gslsflegendreH3d0', + 'gslsflegendreH3d1', + 'gslsflegendreH3d', + 'gslsflog', + 'gslsflogabs', + 'gslsflog1plusx', + 'gslsflog1plusxmx', + 'gslsfpowint', + 'gslsfpsiint', + 'gslsfpsi', + 'gslsfpsi1piy', + 'gslsfpsi1int', + 'gslsfpsi1', + 'gslsfpsin', + 'gslsfsynchrotron1', + 'gslsfsynchrotron2', + 'gslsftransport2', + 'gslsftransport3', + 'gslsftransport4', + 'gslsftransport5', + 'gslsfsin', + 'gslsfcos', + 'gslsfhypot', + 'gslsfsinc', + 'gslsflnsinh', + 'gslsflncosh', + 'gslsfanglerestrictsymm', + 'gslsfanglerestrictpos', + 'gslsfzetaint', + 'gslsfzeta', + 'gslsfzetam1', + 'gslsfzetam1int', + 'gslsfhzeta', + 'gslsfetaint', + 'gslsfeta', + 'imag', + 'int1d', + 'int2d', + 'int3d', + 'intalledges', + 'intallfaces', + 'interpolate', + 'invdiff', + 'invdiffnp', + 'invdiffpos', + 'Isend', + 'isInf', + 'isNaN', + 'isoline', + 'Irecv', + 'j0', + 'j1', + 'jn', + 'jump', + 'lgamma', + 'LinearCG', + 'LinearGMRES', + 'log', + 'log10', + 'lrint', + 'lround', + 'max', + 'mean', + 'medit', + 'min', + 'mmg3d', + 'movemesh', + 'movemesh23', + 'mpiAlltoall', + 'mpiAlltoallv', + 'mpiAllgather', + 'mpiAllgatherv', + 'mpiAllReduce', + 'mpiBarrier', + 'mpiGather', + 'mpiGatherv', + 'mpiRank', + 'mpiReduce', + 'mpiScatter', + 'mpiScatterv', + 'mpiSize', + 'mpiWait', + 'mpiWaitAny', + 'mpiWtick', + 'mpiWtime', + 'mshmet', + 'NaN', + 'NLCG', + 'on', + 'plot', + 'polar', + 'Post', + 'pow', + 'processor', + 'processorblock', + 'projection', + 'randinit', + 'randint31', + 'randint32', + 'random', + 'randreal1', + 'randreal2', + 'randreal3', + 'randres53', + 'Read', + 'readmesh', + 'readmesh3', + 'Recv', + 'rint', + 'round', + 'savemesh', + 'savesol', + 'savevtk', + 'seekg', + 'Sent', + 'set', + 'sign', + 'signbit', + 'sin', + 'sinh', + 'sort', + 'splitComm', + 'splitmesh', + 'sqrt', + 'square', + 'srandom', + 'srandomdev', + 'Stringification', + 'swap', + 'system', + 'tan', + 'tanh', + 'tellg', + 'tetg', + 'tetgconvexhull', + 'tetgreconstruction', + 'tetgtransfo', + 'tgamma', + 'triangulate', + 'trunc', + 'Wait', + 'Write', + 'y0', + 'y1', + 'yn' + )) + + # function parameters + parameters = set(( + 'A', + 'A1', + 'abserror', + 'absolute', + 'aniso', + 'aspectratio', + 'B', + 'B1', + 'bb', + 'beginend', + 'bin', + 'boundary', + 'bw', + 'close', + 'cmm', + 'coef', + 'composante', + 'cutoff', + 'datafilename', + 'dataname', + 'dim', + 'distmax', + 'displacement', + 'doptions', + 'dparams', + 'eps', + 'err', + 'errg', + 'facemerge', + 'facetcl', + 'factorize', + 'file', + 'fill', + 'fixedborder', + 'flabel', + 'flags', + 'floatmesh', + 'floatsol', + 'fregion', + 'gradation', + 'grey', + 'hmax', + 'hmin', + 'holelist', + 'hsv', + 'init', + 'inquire', + 'inside', + 'IsMetric', + 'iso', + 'ivalue', + 'keepbackvertices', + 'label', + 'labeldown', + 'labelmid', + 'labelup', + 'levelset', + 'loptions', + 'lparams', + 'maxit', + 'maxsubdiv', + 'meditff', + 'mem', + 'memory', + 'metric', + 'mode', + 'nbarrow', + 'nbiso', + 'nbiter', + 'nbjacoby', + 'nboffacetcl', + 'nbofholes', + 'nbofregions', + 'nbregul', + 'nbsmooth', + 'nbvx', + 'ncv', + 'nev', + 'nomeshgeneration', + 'normalization', + 'omega', + 'op', + 'optimize', + 'option', + 'options', + 'order', + 'orientation', + 'periodic', + 'power', + 'precon', + 'prev', + 'ps', + 'ptmerge', + 'qfe', + 'qforder', + 'qft', + 'qfV', + 'ratio', + 'rawvector', + 'reffacelow', + 'reffacemid', + 'reffaceup', + 'refnum', + 'reftet', + 'reftri', + 'region', + 'regionlist', + 'renumv', + 'rescaling', + 'ridgeangle', + 'save', + 'sigma', + 'sizeofvolume', + 'smoothing', + 'solver', + 'sparams', + 'split', + 'splitin2', + 'splitpbedge', + 'stop', + 'strategy', + 'swap', + 'switch', + 'sym', + 't', + 'tgv', + 'thetamax', + 'tol', + 'tolpivot', + 'tolpivotsym', + 'transfo', + 'U2Vc', + 'value', + 'varrow', + 'vector', + 'veps', + 'viso', + 'wait', + 'width', + 'withsurfacemesh', + 'WindowIndex', + 'which', + 'zbound' + )) + + # deprecated + deprecated = set(('fixeborder',)) + + # do not highlight + suppress_highlight = set(( + 'alignof', + 'asm', + 'constexpr', + 'decltype', + 'div', + 'double', + 'grad', + 'mutable', + 'namespace', + 'noexcept', + 'restrict', + 'static_assert', + 'template', + 'this', + 'thread_local', + 'typeid', + 'typename', + 'using' + )) + + def get_tokens_unprocessed(self, text): + for index, token, value in CppLexer.get_tokens_unprocessed(self, text): + if value in self.operators: + yield index, Operator, value + elif value in self.types: + yield index, Keyword.Type, value + elif value in self.fespaces: + yield index, Name.Class, value + elif value in self.preprocessor: + yield index, Comment.Preproc, value + elif value in self.keywords: + yield index, Keyword.Reserved, value + elif value in self.functions: + yield index, Name.Function, value + elif value in self.parameters: + yield index, Keyword.Pseudo, value + elif value in self.suppress_highlight: + yield index, Name, value + else: + yield index, token, value diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py index 254df795..2d94aca3 100644 --- a/pygments/lexers/functional.py +++ b/pygments/lexers/functional.py @@ -5,7 +5,7 @@ Just export lexer classes previously contained in this module. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/go.py b/pygments/lexers/go.py index cc2a6d63..f6bb7fc8 100644 --- a/pygments/lexers/go.py +++ b/pygments/lexers/go.py @@ -5,7 +5,7 @@ Lexers for the Google Go language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/grammar_notation.py b/pygments/lexers/grammar_notation.py index 076249d3..66d3eb36 100644 --- a/pygments/lexers/grammar_notation.py +++ b/pygments/lexers/grammar_notation.py @@ -5,7 +5,7 @@ Lexers for grammer notations like BNF. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -158,7 +158,7 @@ class JsgfLexer(RegexLexer): (r'//.*', Comment.Single), ], 'non-comments': [ - ('\A#JSGF[^;]*', Comment.Preproc), + (r'\A#JSGF[^;]*', Comment.Preproc), (r'\s+', Text), (r';', Punctuation), (r'[=|()\[\]*+]', Operator), diff --git a/pygments/lexers/graph.py b/pygments/lexers/graph.py index 1a338246..f7b45e29 100644 --- a/pygments/lexers/graph.py +++ b/pygments/lexers/graph.py @@ -5,7 +5,7 @@ Lexers for graph query languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -22,9 +22,9 @@ __all__ = ['CypherLexer'] class CypherLexer(RegexLexer): """ For `Cypher Query Language - <http://docs.neo4j.org/chunked/milestone/cypher-query-lang.html>`_ + <https://neo4j.com/docs/developer-manual/3.3/cypher/>`_ - For the Cypher version in Neo4J 2.0 + For the Cypher version in Neo4j 3.3 .. versionadded:: 2.0 """ @@ -49,14 +49,19 @@ class CypherLexer(RegexLexer): ], 'keywords': [ (r'(create|order|match|limit|set|skip|start|return|with|where|' - r'delete|foreach|not|by)\b', Keyword), + r'delete|foreach|not|by|true|false)\b', Keyword), ], 'clauses': [ - # TODO: many missing ones, see http://docs.neo4j.org/refcard/2.0/ - (r'(all|any|as|asc|create|create\s+unique|delete|' - r'desc|distinct|foreach|in|is\s+null|limit|match|none|' - r'order\s+by|return|set|skip|single|start|union|where|with)\b', - Keyword), + # based on https://neo4j.com/docs/cypher-refcard/3.3/ + (r'(all|any|as|asc|ascending|assert|call|case|create|' + r'create\s+index|create\s+unique|delete|desc|descending|' + r'distinct|drop\s+constraint\s+on|drop\s+index\s+on|end|' + r'ends\s+with|fieldterminator|foreach|in|is\s+node\s+key|' + r'is\s+null|is\s+unique|limit|load\s+csv\s+from|match|merge|none|' + r'not|null|on\s+match|on\s+create|optional\s+match|order\s+by|' + r'remove|return|set|skip|single|start|starts\s+with|then|union|' + r'union\s+all|unwind|using\s+periodic\s+commit|yield|where|when|' + r'with)\b', Keyword), ], 'relations': [ (r'(-\[)(.*?)(\]->)', bygroups(Operator, using(this), Operator)), diff --git a/pygments/lexers/graphics.py b/pygments/lexers/graphics.py index c8af9f99..b0b9145e 100644 --- a/pygments/lexers/graphics.py +++ b/pygments/lexers/graphics.py @@ -5,7 +5,7 @@ Lexers for computer graphics and plotting related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -15,7 +15,7 @@ from pygments.token import Text, Comment, Operator, Keyword, Name, \ Number, Punctuation, String __all__ = ['GLShaderLexer', 'PostScriptLexer', 'AsymptoteLexer', 'GnuplotLexer', - 'PovrayLexer'] + 'PovrayLexer', 'HLSLShaderLexer'] class GLShaderLexer(RegexLexer): @@ -46,28 +46,102 @@ class GLShaderLexer(RegexLexer): (r'0[0-7]*', Number.Oct), (r'[1-9][0-9]*', Number.Integer), (words(( - 'attribute', 'const', 'uniform', 'varying', 'centroid', 'break', - 'continue', 'do', 'for', 'while', 'if', 'else', 'in', 'out', - 'inout', 'float', 'int', 'void', 'bool', 'true', 'false', - 'invariant', 'discard', 'return', 'mat2', 'mat3' 'mat4', - 'mat2x2', 'mat3x2', 'mat4x2', 'mat2x3', 'mat3x3', 'mat4x3', - 'mat2x4', 'mat3x4', 'mat4x4', 'vec2', 'vec3', 'vec4', - 'ivec2', 'ivec3', 'ivec4', 'bvec2', 'bvec3', 'bvec4', - 'sampler1D', 'sampler2D', 'sampler3D' 'samplerCube', - 'sampler1DShadow', 'sampler2DShadow', 'struct'), + # Storage qualifiers + 'attribute', 'const', 'uniform', 'varying', + 'buffer', 'shared', 'in', 'out', + # Layout qualifiers + 'layout', + # Interpolation qualifiers + 'flat', 'smooth', 'noperspective', + # Auxiliary qualifiers + 'centroid', 'sample', 'patch', + # Parameter qualifiers. Some double as Storage qualifiers + 'inout', + # Precision qualifiers + 'lowp', 'mediump', 'highp', 'precision', + # Invariance qualifiers + 'invariant', + # Precise qualifiers + 'precise', + # Memory qualifiers + 'coherent', 'volatile', 'restrict', 'readonly', 'writeonly', + # Statements + 'break', 'continue', 'do', 'for', 'while', 'switch', + 'case', 'default', 'if', 'else', 'subroutine', + 'discard', 'return', 'struct'), prefix=r'\b', suffix=r'\b'), Keyword), (words(( - 'asm', 'class', 'union', 'enum', 'typedef', 'template', 'this', - 'packed', 'goto', 'switch', 'default', 'inline', 'noinline', - 'volatile', 'public', 'static', 'extern', 'external', 'interface', - 'long', 'short', 'double', 'half', 'fixed', 'unsigned', 'lowp', - 'mediump', 'highp', 'precision', 'input', 'output', - 'hvec2', 'hvec3', 'hvec4', 'dvec2', 'dvec3', 'dvec4', - 'fvec2', 'fvec3', 'fvec4', 'sampler2DRect', 'sampler3DRect', - 'sampler2DRectShadow', 'sizeof', 'cast', 'namespace', 'using'), + # Boolean values + 'true', 'false'), prefix=r'\b', suffix=r'\b'), - Keyword), # future use + Keyword.Constant), + (words(( + # Miscellaneous types + 'void', 'atomic_uint', + # Floating-point scalars and vectors + 'float', 'vec2', 'vec3', 'vec4', + 'double', 'dvec2', 'dvec3', 'dvec4', + # Integer scalars and vectors + 'int', 'ivec2', 'ivec3', 'ivec4', + 'uint', 'uvec2', 'uvec3', 'uvec4', + # Boolean scalars and vectors + 'bool', 'bvec2', 'bvec3', 'bvec4', + # Matrices + 'mat2', 'mat3', 'mat4', 'dmat2', 'dmat3', 'dmat4', + 'mat2x2', 'mat2x3', 'mat2x4', 'dmat2x2', 'dmat2x3', 'dmat2x4', + 'mat3x2', 'mat3x3', 'mat3x4', 'dmat3x2', 'dmat3x3', + 'dmat3x4', 'mat4x2', 'mat4x3', 'mat4x4', 'dmat4x2', 'dmat4x3', 'dmat4x4', + # Floating-point samplers + 'sampler1D', 'sampler2D', 'sampler3D', 'samplerCube', + 'sampler1DArray', 'sampler2DArray', 'samplerCubeArray', + 'sampler2DRect', 'samplerBuffer', + 'sampler2DMS', 'sampler2DMSArray', + # Shadow samplers + 'sampler1DShadow', 'sampler2DShadow', 'samplerCubeShadow', + 'sampler1DArrayShadow', 'sampler2DArrayShadow', + 'samplerCubeArrayShadow', 'sampler2DRectShadow', + # Signed integer samplers + 'isampler1D', 'isampler2D', 'isampler3D', 'isamplerCube', + 'isampler1DArray', 'isampler2DArray', 'isamplerCubeArray', + 'isampler2DRect', 'isamplerBuffer', + 'isampler2DMS', 'isampler2DMSArray', + # Unsigned integer samplers + 'usampler1D', 'usampler2D', 'usampler3D', 'usamplerCube', + 'usampler1DArray', 'usampler2DArray', 'usamplerCubeArray', + 'usampler2DRect', 'usamplerBuffer', + 'usampler2DMS', 'usampler2DMSArray', + # Floating-point image types + 'image1D', 'image2D', 'image3D', 'imageCube', + 'image1DArray', 'image2DArray', 'imageCubeArray', + 'image2DRect', 'imageBuffer', + 'image2DMS', 'image2DMSArray', + # Signed integer image types + 'iimage1D', 'iimage2D', 'iimage3D', 'iimageCube', + 'iimage1DArray', 'iimage2DArray', 'iimageCubeArray', + 'iimage2DRect', 'iimageBuffer', + 'iimage2DMS', 'iimage2DMSArray', + # Unsigned integer image types + 'uimage1D', 'uimage2D', 'uimage3D', 'uimageCube', + 'uimage1DArray', 'uimage2DArray', 'uimageCubeArray', + 'uimage2DRect', 'uimageBuffer', + 'uimage2DMS', 'uimage2DMSArray'), + prefix=r'\b', suffix=r'\b'), + Keyword.Type), + (words(( + # Reserved for future use. + 'common', 'partition', 'active', 'asm', 'class', + 'union', 'enum', 'typedef', 'template', 'this', + 'resource', 'goto', 'inline', 'noinline', 'public', + 'static', 'extern', 'external', 'interface', 'long', + 'short', 'half', 'fixed', 'unsigned', 'superp', 'input', + 'output', 'hvec2', 'hvec3', 'hvec4', 'fvec2', 'fvec3', + 'fvec4', 'sampler3DRect', 'filter', 'sizeof', 'cast', + 'namespace', 'using'), + prefix=r'\b', suffix=r'\b'), + Keyword.Reserved), + # All names beginning with "gl_" are reserved. + (r'gl_\w*', Name.Builtin), (r'[a-zA-Z_]\w*', Name), (r'\.', Punctuation), (r'\s+', Text), @@ -75,6 +149,160 @@ class GLShaderLexer(RegexLexer): } +class HLSLShaderLexer(RegexLexer): + """ + HLSL (Microsoft Direct3D Shader) lexer. + + .. versionadded:: 2.3 + """ + name = 'HLSL' + aliases = ['hlsl'] + filenames = ['*.hlsl', '*.hlsli'] + mimetypes = ['text/x-hlsl'] + + tokens = { + 'root': [ + (r'^#.*', Comment.Preproc), + (r'//.*', Comment.Single), + (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), + (r'\+|-|~|!=?|\*|/|%|<<|>>|<=?|>=?|==?|&&?|\^|\|\|?', + Operator), + (r'[?:]', Operator), # quick hack for ternary + (r'\bdefined\b', Operator), + (r'[;{}(),.\[\]]', Punctuation), + # FIXME when e is present, no decimal point needed + (r'[+-]?\d*\.\d+([eE][-+]?\d+)?f?', Number.Float), + (r'[+-]?\d+\.\d*([eE][-+]?\d+)?f?', Number.Float), + (r'0[xX][0-9a-fA-F]*', Number.Hex), + (r'0[0-7]*', Number.Oct), + (r'[1-9][0-9]*', Number.Integer), + (r'"', String, 'string'), + (words(( + 'asm','asm_fragment','break','case','cbuffer','centroid','class', + 'column_major','compile','compile_fragment','const','continue', + 'default','discard','do','else','export','extern','for','fxgroup', + 'globallycoherent','groupshared','if','in','inline','inout', + 'interface','line','lineadj','linear','namespace','nointerpolation', + 'noperspective','NULL','out','packoffset','pass','pixelfragment', + 'point','precise','return','register','row_major','sample', + 'sampler','shared','stateblock','stateblock_state','static', + 'struct','switch','tbuffer','technique','technique10', + 'technique11','texture','typedef','triangle','triangleadj', + 'uniform','vertexfragment','volatile','while'), + prefix=r'\b', suffix=r'\b'), + Keyword), + (words(('true','false'), prefix=r'\b', suffix=r'\b'), + Keyword.Constant), + (words(( + 'auto','catch','char','const_cast','delete','dynamic_cast','enum', + 'explicit','friend','goto','long','mutable','new','operator', + 'private','protected','public','reinterpret_cast','short','signed', + 'sizeof','static_cast','template','this','throw','try','typename', + 'union','unsigned','using','virtual'), + prefix=r'\b', suffix=r'\b'), + Keyword.Reserved), + (words(( + 'dword','matrix','snorm','string','unorm','unsigned','void','vector', + 'BlendState','Buffer','ByteAddressBuffer','ComputeShader', + 'DepthStencilState','DepthStencilView','DomainShader', + 'GeometryShader','HullShader','InputPatch','LineStream', + 'OutputPatch','PixelShader','PointStream','RasterizerState', + 'RenderTargetView','RasterizerOrderedBuffer', + 'RasterizerOrderedByteAddressBuffer', + 'RasterizerOrderedStructuredBuffer','RasterizerOrderedTexture1D', + 'RasterizerOrderedTexture1DArray','RasterizerOrderedTexture2D', + 'RasterizerOrderedTexture2DArray','RasterizerOrderedTexture3D', + 'RWBuffer','RWByteAddressBuffer','RWStructuredBuffer', + 'RWTexture1D','RWTexture1DArray','RWTexture2D','RWTexture2DArray', + 'RWTexture3D','SamplerState','SamplerComparisonState', + 'StructuredBuffer','Texture1D','Texture1DArray','Texture2D', + 'Texture2DArray','Texture2DMS','Texture2DMSArray','Texture3D', + 'TextureCube','TextureCubeArray','TriangleStream','VertexShader'), + prefix=r'\b', suffix=r'\b'), + Keyword.Type), + (words(( + 'bool','double','float','int','half','min16float','min10float', + 'min16int','min12int','min16uint','uint'), + prefix=r'\b', suffix=r'([1-4](x[1-4])?)?\b'), + Keyword.Type), # vector and matrix types + (words(( + 'abort','abs','acos','all','AllMemoryBarrier', + 'AllMemoryBarrierWithGroupSync','any','AppendStructuredBuffer', + 'asdouble','asfloat','asin','asint','asuint','asuint','atan', + 'atan2','ceil','CheckAccessFullyMapped','clamp','clip', + 'CompileShader','ConsumeStructuredBuffer','cos','cosh','countbits', + 'cross','D3DCOLORtoUBYTE4','ddx','ddx_coarse','ddx_fine','ddy', + 'ddy_coarse','ddy_fine','degrees','determinant', + 'DeviceMemoryBarrier','DeviceMemoryBarrierWithGroupSync','distance', + 'dot','dst','errorf','EvaluateAttributeAtCentroid', + 'EvaluateAttributeAtSample','EvaluateAttributeSnapped','exp', + 'exp2','f16tof32','f32tof16','faceforward','firstbithigh', + 'firstbitlow','floor','fma','fmod','frac','frexp','fwidth', + 'GetRenderTargetSampleCount','GetRenderTargetSamplePosition', + 'GlobalOrderedCountIncrement','GroupMemoryBarrier', + 'GroupMemoryBarrierWithGroupSync','InterlockedAdd','InterlockedAnd', + 'InterlockedCompareExchange','InterlockedCompareStore', + 'InterlockedExchange','InterlockedMax','InterlockedMin', + 'InterlockedOr','InterlockedXor','isfinite','isinf','isnan', + 'ldexp','length','lerp','lit','log','log10','log2','mad','max', + 'min','modf','msad4','mul','noise','normalize','pow','printf', + 'Process2DQuadTessFactorsAvg','Process2DQuadTessFactorsMax', + 'Process2DQuadTessFactorsMin','ProcessIsolineTessFactors', + 'ProcessQuadTessFactorsAvg','ProcessQuadTessFactorsMax', + 'ProcessQuadTessFactorsMin','ProcessTriTessFactorsAvg', + 'ProcessTriTessFactorsMax','ProcessTriTessFactorsMin', + 'QuadReadLaneAt','QuadSwapX','QuadSwapY','radians','rcp', + 'reflect','refract','reversebits','round','rsqrt','saturate', + 'sign','sin','sincos','sinh','smoothstep','sqrt','step','tan', + 'tanh','tex1D','tex1D','tex1Dbias','tex1Dgrad','tex1Dlod', + 'tex1Dproj','tex2D','tex2D','tex2Dbias','tex2Dgrad','tex2Dlod', + 'tex2Dproj','tex3D','tex3D','tex3Dbias','tex3Dgrad','tex3Dlod', + 'tex3Dproj','texCUBE','texCUBE','texCUBEbias','texCUBEgrad', + 'texCUBElod','texCUBEproj','transpose','trunc','WaveAllBitAnd', + 'WaveAllMax','WaveAllMin','WaveAllBitOr','WaveAllBitXor', + 'WaveAllEqual','WaveAllProduct','WaveAllSum','WaveAllTrue', + 'WaveAnyTrue','WaveBallot','WaveGetLaneCount','WaveGetLaneIndex', + 'WaveGetOrderedIndex','WaveIsHelperLane','WaveOnce', + 'WavePrefixProduct','WavePrefixSum','WaveReadFirstLane', + 'WaveReadLaneAt'), + prefix=r'\b', suffix=r'\b'), + Name.Builtin), # built-in functions + (words(( + 'SV_ClipDistance','SV_ClipDistance0','SV_ClipDistance1', + 'SV_Culldistance','SV_CullDistance0','SV_CullDistance1', + 'SV_Coverage','SV_Depth','SV_DepthGreaterEqual', + 'SV_DepthLessEqual','SV_DispatchThreadID','SV_DomainLocation', + 'SV_GroupID','SV_GroupIndex','SV_GroupThreadID','SV_GSInstanceID', + 'SV_InnerCoverage','SV_InsideTessFactor','SV_InstanceID', + 'SV_IsFrontFace','SV_OutputControlPointID','SV_Position', + 'SV_PrimitiveID','SV_RenderTargetArrayIndex','SV_SampleIndex', + 'SV_StencilRef','SV_TessFactor','SV_VertexID', + 'SV_ViewportArrayIndex'), + prefix=r'\b', suffix=r'\b'), + Name.Decorator), # system-value semantics + (r'\bSV_Target[0-7]?\b', Name.Decorator), + (words(( + 'allow_uav_condition','branch','call','domain','earlydepthstencil', + 'fastopt','flatten','forcecase','instance','loop','maxtessfactor', + 'numthreads','outputcontrolpoints','outputtopology','partitioning', + 'patchconstantfunc','unroll'), + prefix=r'\b', suffix=r'\b'), + Name.Decorator), # attributes + (r'[a-zA-Z_]\w*', Name), + (r'\\$', Comment.Preproc), # backslash at end of line -- usually macro continuation + (r'\s+', Text), + ], + 'string': [ + (r'"', String, '#pop'), + (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|' + r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape), + (r'[^\\"\n]+', String), # all other characters + (r'\\\n', String), # line continuation + (r'\\', String), # stray backslash + ], + } + + class PostScriptLexer(RegexLexer): """ Lexer for PostScript files. @@ -233,8 +461,8 @@ class AsymptoteLexer(RegexLexer): r'bounds|coord|frame|guide|horner|int|linefit|marginT|pair|pen|' r'picture|position|real|revolution|slice|splitface|ticksgridT|' r'tickvalues|tree|triple|vertex|void)\b', Keyword.Type), - ('[a-zA-Z_]\w*:(?!:)', Name.Label), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*:(?!:)', Name.Label), + (r'[a-zA-Z_]\w*', Name), ], 'root': [ include('whitespace'), @@ -334,9 +562,9 @@ class GnuplotLexer(RegexLexer): (_shortened_many('pwd$', 're$read', 'res$et', 'scr$eendump', 'she$ll', 'test$'), Keyword, 'noargs'), - ('([a-zA-Z_]\w*)(\s*)(=)', + (r'([a-zA-Z_]\w*)(\s*)(=)', bygroups(Name.Variable, Text, Operator), 'genericargs'), - ('([a-zA-Z_]\w*)(\s*\(.*?\)\s*)(=)', + (r'([a-zA-Z_]\w*)(\s*\(.*?\)\s*)(=)', bygroups(Name.Function, Text, Operator), 'genericargs'), (r'@[a-zA-Z_]\w*', Name.Constant), # macros (r';', Keyword), @@ -382,7 +610,7 @@ class GnuplotLexer(RegexLexer): (r'(\d+\.\d*|\.\d+)', Number.Float), (r'-?\d+', Number.Integer), ('[,.~!%^&*+=|?:<>/-]', Operator), - ('[{}()\[\]]', Punctuation), + (r'[{}()\[\]]', Punctuation), (r'(eq|ne)\b', Operator.Word), (r'([a-zA-Z_]\w*)(\s*)(\()', bygroups(Name.Function, Text, Punctuation)), diff --git a/pygments/lexers/haskell.py b/pygments/lexers/haskell.py index 1a2f2217..d9eecaaf 100644 --- a/pygments/lexers/haskell.py +++ b/pygments/lexers/haskell.py @@ -5,19 +5,19 @@ Lexers for Haskell and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re from pygments.lexer import Lexer, RegexLexer, bygroups, do_insertions, \ - default, include + default, include, inherit from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Generic from pygments import unistring as uni -__all__ = ['HaskellLexer', 'IdrisLexer', 'AgdaLexer', 'CryptolLexer', +__all__ = ['HaskellLexer', 'HspecLexer', 'IdrisLexer', 'AgdaLexer', 'CryptolLexer', 'LiterateHaskellLexer', 'LiterateIdrisLexer', 'LiterateAgdaLexer', 'LiterateCryptolLexer', 'KokaLexer'] @@ -72,11 +72,15 @@ class HaskellLexer(RegexLexer): (r':[:!#$%&*+.\\/<=>?@^|~-]*', Keyword.Type), # Constructor operators (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator), # Other operators # Numbers - (r'\d+[eE][+-]?\d+', Number.Float), - (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float), - (r'0[oO][0-7]+', Number.Oct), - (r'0[xX][\da-fA-F]+', Number.Hex), - (r'\d+', Number.Integer), + (r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*_*[pP][+-]?\d(_*\d)*', Number.Float), + (r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*\.[\da-fA-F](_*[\da-fA-F])*' + r'(_*[pP][+-]?\d(_*\d)*)?', Number.Float), + (r'\d(_*\d)*_*[eE][+-]?\d(_*\d)*', Number.Float), + (r'\d(_*\d)*\.\d(_*\d)*(_*[eE][+-]?\d(_*\d)*)?', Number.Float), + (r'0[bB]_*[01](_*[01])*', Number.Bin), + (r'0[oO]_*[0-7](_*[0-7])*', Number.Oct), + (r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*', Number.Hex), + (r'\d(_*\d)*', Number.Integer), # Character/String Literals (r"'", String.Char, 'character'), (r'"', String, 'string'), @@ -154,6 +158,28 @@ class HaskellLexer(RegexLexer): } +class HspecLexer(HaskellLexer): + """ + A Haskell lexer with support for Hspec constructs. + + .. versionadded:: 2.4.0 + """ + + name = 'Hspec' + aliases = ['hspec'] + filenames = [] + mimetypes = [] + + tokens = { + 'root': [ + (r'(it\s*)("[^"]*")', bygroups(Text, String.Doc)), + (r'(describe\s*)("[^"]*")', bygroups(Text, String.Doc)), + (r'(context\s*)("[^"]*")', bygroups(Text, String.Doc)), + inherit, + ], + } + + class IdrisLexer(RegexLexer): """ A lexer for the dependently typed programming language Idris. @@ -418,8 +444,8 @@ class CryptolLexer(RegexLexer): (r'[A-Z]\w*', Keyword.Type), (r'(_[\w\']+|[a-z][\w\']*)', Name.Function), # TODO: these don't match the comments in docs, remove. - #(r'--(?![!#$%&*+./<=>?@^|_~:\\]).*?$', Comment.Single), - #(r'{-', Comment.Multiline, 'comment'), + # (r'--(?![!#$%&*+./<=>?@^|_~:\\]).*?$', Comment.Single), + # (r'{-', Comment.Multiline, 'comment'), (r',', Punctuation), (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator), # (HACK, but it makes sense to push two instances, believe me) @@ -677,10 +703,10 @@ class KokaLexer(RegexLexer): symbols = r'[$%&*+@!/\\^~=.:\-?|<>]+' # symbol boundary: an operator keyword should not be followed by any of these - sboundary = '(?!'+symbols+')' + sboundary = '(?!' + symbols + ')' # name boundary: a keyword should not be followed by any of these - boundary = '(?![\w/])' + boundary = r'(?![\w/])' # koka token abstractions tokenType = Name.Attribute diff --git a/pygments/lexers/haxe.py b/pygments/lexers/haxe.py index 6f5c3599..38578e5b 100644 --- a/pygments/lexers/haxe.py +++ b/pygments/lexers/haxe.py @@ -5,7 +5,7 @@ Lexers for Haxe and related stuff. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -43,7 +43,7 @@ class HaxeLexer(ExtendedRegexLexer): typeid = r'_*[A-Z]\w*' # combined ident and dollar and idtype - ident = r'(?:_*[a-z]\w*|_+[0-9]\w*|' + typeid + '|_+|\$\w+)' + ident = r'(?:_*[a-z]\w*|_+[0-9]\w*|' + typeid + r'|_+|\$\w+)' binop = (r'(?:%=|&=|\|=|\^=|\+=|\-=|\*=|/=|<<=|>\s*>\s*=|>\s*>\s*>\s*=|==|' r'!=|<=|>\s*=|&&|\|\||<<|>>>|>\s*>|\.\.\.|<|>|%|&|\||\^|\+|\*|' @@ -182,7 +182,7 @@ class HaxeLexer(ExtendedRegexLexer): (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float), (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float), (r'[0-9]+\.[0-9]+', Number.Float), - (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float), + (r'[0-9]+\.(?!' + ident + r'|\.\.)', Number.Float), # Int (r'0x[0-9a-fA-F]+', Number.Hex), @@ -219,7 +219,7 @@ class HaxeLexer(ExtendedRegexLexer): (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), (r'[0-9]+\.[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), - (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, ('#pop', 'preproc-expr-chain')), + (r'[0-9]+\.(?!' + ident + r'|\.\.)', Number.Float, ('#pop', 'preproc-expr-chain')), # Int (r'0x[0-9a-fA-F]+', Number.Hex, ('#pop', 'preproc-expr-chain')), @@ -456,7 +456,7 @@ class HaxeLexer(ExtendedRegexLexer): (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'expr-chain')), (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float, ('#pop', 'expr-chain')), (r'[0-9]+\.[0-9]+', Number.Float, ('#pop', 'expr-chain')), - (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, ('#pop', 'expr-chain')), + (r'[0-9]+\.(?!' + ident + r'|\.\.)', Number.Float, ('#pop', 'expr-chain')), # Int (r'0x[0-9a-fA-F]+', Number.Hex, ('#pop', 'expr-chain')), @@ -711,7 +711,7 @@ class HaxeLexer(ExtendedRegexLexer): (r'[0-9]+[eE][+\-]?[0-9]+', Number.Float, '#pop'), (r'[0-9]+\.[0-9]*[eE][+\-]?[0-9]+', Number.Float, '#pop'), (r'[0-9]+\.[0-9]+', Number.Float, '#pop'), - (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, '#pop'), + (r'[0-9]+\.(?!' + ident + r'|\.\.)', Number.Float, '#pop'), # Int (r'0x[0-9a-fA-F]+', Number.Hex, '#pop'), diff --git a/pygments/lexers/hdl.py b/pygments/lexers/hdl.py index 57fb7ac9..e7c7617b 100644 --- a/pygments/lexers/hdl.py +++ b/pygments/lexers/hdl.py @@ -5,7 +5,7 @@ Lexers for hardware descriptor languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/hexdump.py b/pygments/lexers/hexdump.py index cba49be7..da285432 100644 --- a/pygments/lexers/hexdump.py +++ b/pygments/lexers/hexdump.py @@ -5,7 +5,7 @@ Lexers for hexadecimal dumps. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/html.py b/pygments/lexers/html.py index 73f020fa..ae38167c 100644 --- a/pygments/lexers/html.py +++ b/pygments/lexers/html.py @@ -5,7 +5,7 @@ Lexers for HTML, XML and related markup. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -220,7 +220,7 @@ class XmlLexer(RegexLexer): (r'/?\s*>', Name.Tag, '#pop'), ], 'attr': [ - ('\s+', Text), + (r'\s+', Text), ('".*?"', String, '#pop'), ("'.*?'", String, '#pop'), (r'[^\s>]+', String, '#pop'), @@ -313,7 +313,7 @@ class HamlLexer(ExtendedRegexLexer): include('css'), (r'%[\w:-]+', Name.Tag, 'tag'), (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'), - (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)', + (r'(/)(\[' + _dot + r'*?\])(' + _dot + r'*\n)', bygroups(Comment, Comment.Special, Comment), '#pop'), (r'/' + _dot + r'*\n', _starts_block(Comment, 'html-comment-block'), @@ -330,8 +330,8 @@ class HamlLexer(ExtendedRegexLexer): 'tag': [ include('css'), - (r'\{(,\n|' + _dot + ')*?\}', using(RubyLexer)), - (r'\[' + _dot + '*?\]', using(RubyLexer)), + (r'\{(,\n|' + _dot + r')*?\}', using(RubyLexer)), + (r'\[' + _dot + r'*?\]', using(RubyLexer)), (r'\(', Text, 'html-attributes'), (r'/[ \t]*\n', Punctuation, '#pop:2'), (r'[<>]{1,2}(?=[ \t=])', Punctuation), @@ -340,7 +340,7 @@ class HamlLexer(ExtendedRegexLexer): 'plain': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Text), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(RubyLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -373,7 +373,7 @@ class HamlLexer(ExtendedRegexLexer): 'filter-block': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Name.Decorator), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(RubyLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -422,7 +422,7 @@ class ScamlLexer(ExtendedRegexLexer): include('css'), (r'%[\w:-]+', Name.Tag, 'tag'), (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'), - (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)', + (r'(/)(\[' + _dot + r'*?\])(' + _dot + r'*\n)', bygroups(Comment, Comment.Special, Comment), '#pop'), (r'/' + _dot + r'*\n', _starts_block(Comment, 'html-comment-block'), @@ -442,8 +442,8 @@ class ScamlLexer(ExtendedRegexLexer): 'tag': [ include('css'), - (r'\{(,\n|' + _dot + ')*?\}', using(ScalaLexer)), - (r'\[' + _dot + '*?\]', using(ScalaLexer)), + (r'\{(,\n|' + _dot + r')*?\}', using(ScalaLexer)), + (r'\[' + _dot + r'*?\]', using(ScalaLexer)), (r'\(', Text, 'html-attributes'), (r'/[ \t]*\n', Punctuation, '#pop:2'), (r'[<>]{1,2}(?=[ \t=])', Punctuation), @@ -452,7 +452,7 @@ class ScamlLexer(ExtendedRegexLexer): 'plain': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Text), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(ScalaLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -485,7 +485,7 @@ class ScamlLexer(ExtendedRegexLexer): 'filter-block': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Name.Decorator), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(ScalaLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -530,7 +530,7 @@ class PugLexer(ExtendedRegexLexer): 'content': [ include('css'), (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'), - (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)', + (r'(/)(\[' + _dot + r'*?\])(' + _dot + r'*\n)', bygroups(Comment, Comment.Special, Comment), '#pop'), (r'/' + _dot + r'*\n', _starts_block(Comment, 'html-comment-block'), @@ -551,8 +551,8 @@ class PugLexer(ExtendedRegexLexer): 'tag': [ include('css'), - (r'\{(,\n|' + _dot + ')*?\}', using(ScalaLexer)), - (r'\[' + _dot + '*?\]', using(ScalaLexer)), + (r'\{(,\n|' + _dot + r')*?\}', using(ScalaLexer)), + (r'\[' + _dot + r'*?\]', using(ScalaLexer)), (r'\(', Text, 'html-attributes'), (r'/[ \t]*\n', Punctuation, '#pop:2'), (r'[<>]{1,2}(?=[ \t=])', Punctuation), @@ -561,7 +561,7 @@ class PugLexer(ExtendedRegexLexer): 'plain': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Text), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(ScalaLexer), String.Interpol)), (r'\n', Text, 'root'), ], @@ -594,7 +594,7 @@ class PugLexer(ExtendedRegexLexer): 'filter-block': [ (r'([^#\n]|#[^{\n]|(\\\\)*\\#\{)+', Name.Decorator), - (r'(#\{)(' + _dot + '*?)(\})', + (r'(#\{)(' + _dot + r'*?)(\})', bygroups(String.Interpol, using(ScalaLexer), String.Interpol)), (r'\n', Text, 'root'), ], diff --git a/pygments/lexers/idl.py b/pygments/lexers/idl.py index 2fc39318..292f2de7 100644 --- a/pygments/lexers/idl.py +++ b/pygments/lexers/idl.py @@ -5,7 +5,7 @@ Lexers for IDL. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -53,7 +53,7 @@ class IDLLexer(RegexLexer): 'broyden', 'butterworth', 'bytarr', 'byte', 'byteorder', 'bytscl', 'caldat', 'calendar', 'call_external', 'call_function', 'call_method', 'call_procedure', 'canny', - 'catch', 'cd', 'cdf_\w*', 'ceil', 'chebyshev', + 'catch', 'cd', r'cdf_\w*', 'ceil', 'chebyshev', 'check_math', 'chisqr_cvf', 'chisqr_pdf', 'choldc', 'cholsol', 'cindgen', 'cir_3pnt', 'close', 'cluster', 'cluster_tree', 'clust_wts', @@ -87,7 +87,7 @@ class IDLLexer(RegexLexer): 'dlm_load', 'dlm_register', 'doc_library', 'double', 'draw_roi', 'edge_dog', 'efont', 'eigenql', 'eigenvec', 'ellipse', 'elmhes', 'emboss', 'empty', 'enable_sysrtn', - 'eof', 'eos_\w*', 'erase', 'erf', 'erfc', 'erfcx', + 'eof', r'eos_\w*', 'erase', 'erf', 'erfc', 'erfcx', 'erode', 'errorplot', 'errplot', 'estimator_filter', 'execute', 'exit', 'exp', 'expand', 'expand_path', 'expint', 'extrac', 'extract_slice', 'factorial', 'fft', 'filepath', @@ -104,11 +104,11 @@ class IDLLexer(RegexLexer): 'gauss_cvf', 'gauss_pdf', 'gauss_smooth', 'getenv', 'getwindows', 'get_drive_list', 'get_dxf_objects', 'get_kbrd', 'get_login_info', 'get_lun', 'get_screen_size', - 'greg2jul', 'grib_\w*', 'grid3', 'griddata', + 'greg2jul', r'grib_\w*', 'grid3', 'griddata', 'grid_input', 'grid_tps', 'gs_iter', - 'h5[adfgirst]_\w*', 'h5_browser', 'h5_close', + r'h5[adfgirst]_\w*', 'h5_browser', 'h5_close', 'h5_create', 'h5_get_libversion', 'h5_open', 'h5_parse', - 'hanning', 'hash', 'hdf_\w*', 'heap_free', + 'hanning', 'hash', r'hdf_\w*', 'heap_free', 'heap_gc', 'heap_nosave', 'heap_refcount', 'heap_save', 'help', 'hilbert', 'histogram', 'hist_2d', 'hist_equal', 'hls', 'hough', 'hqr', 'hsv', 'h_eq_ct', 'h_eq_int', @@ -156,7 +156,7 @@ class IDLLexer(RegexLexer): 'modifyct', 'moment', 'morph_close', 'morph_distance', 'morph_gradient', 'morph_hitormiss', 'morph_open', 'morph_thin', 'morph_tophat', 'multi', 'm_correlate', - 'ncdf_\w*', 'newton', 'noise_hurl', 'noise_pick', + r'ncdf_\w*', 'newton', 'noise_hurl', 'noise_pick', 'noise_scatter', 'noise_slur', 'norm', 'n_elements', 'n_params', 'n_tags', 'objarr', 'obj_class', 'obj_destroy', 'obj_hasmethod', 'obj_isa', 'obj_new', 'obj_valid', diff --git a/pygments/lexers/igor.py b/pygments/lexers/igor.py index 1a21fe87..f4a22e1e 100644 --- a/pygments/lexers/igor.py +++ b/pygments/lexers/igor.py @@ -5,7 +5,7 @@ Lexers for Igor Pro. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -49,51 +49,71 @@ class IgorLexer(RegexLexer): ) operations = ( 'Abort', 'AddFIFOData', 'AddFIFOVectData', 'AddMovieAudio', 'AddMovieFrame', - 'AdoptFiles', 'APMath', 'Append', 'AppendImage', 'AppendLayoutObject', - 'AppendMatrixContour', 'AppendText', 'AppendToGizmo', 'AppendToGraph', - 'AppendToLayout', 'AppendToTable', 'AppendXYZContour', 'AutoPositionWindow', - 'BackgroundInfo', 'Beep', 'BoundingBall', 'BoxSmooth', 'BrowseURL', 'BuildMenu', - 'Button', 'cd', 'Chart', 'CheckBox', 'CheckDisplayed', 'ChooseColor', 'Close', - 'CloseHelp', 'CloseMovie', 'CloseProc', 'ColorScale', 'ColorTab2Wave', - 'Concatenate', 'ControlBar', 'ControlInfo', 'ControlUpdate', - 'ConvertGlobalStringTextEncoding', 'ConvexHull', 'Convolve', 'CopyFile', - 'CopyFolder', 'CopyScales', 'Correlate', 'CreateAliasShortcut', 'CreateBrowser', - 'Cross', 'CtrlBackground', 'CtrlFIFO', 'CtrlNamedBackground', 'Cursor', - 'CurveFit', 'CustomControl', 'CWT', 'Debugger', 'DebuggerOptions', 'DefaultFont', - 'DefaultGuiControls', 'DefaultGuiFont', 'DefaultTextEncoding', 'DefineGuide', - 'DelayUpdate', 'DeleteAnnotations', 'DeleteFile', 'DeleteFolder', 'DeletePoints', - 'Differentiate', 'dir', 'Display', 'DisplayHelpTopic', 'DisplayProcedure', - 'DoAlert', 'DoIgorMenu', 'DoUpdate', 'DoWindow', 'DoXOPIdle', 'DPSS', - 'DrawAction', 'DrawArc', 'DrawBezier', 'DrawLine', 'DrawOval', 'DrawPICT', - 'DrawPoly', 'DrawRect', 'DrawRRect', 'DrawText', 'DrawUserShape', 'DSPDetrend', - 'DSPPeriodogram', 'Duplicate', 'DuplicateDataFolder', 'DWT', 'EdgeStats', 'Edit', - 'ErrorBars', 'EstimatePeakSizes', 'Execute', 'ExecuteScriptText', + 'AddWavesToBoxPlot', 'AddWavesToViolinPlot', 'AdoptFiles', 'APMath', 'Append', + 'AppendBoxPlot', 'AppendImage', 'AppendLayoutObject', 'AppendMatrixContour', + 'AppendText', 'AppendToGizmo', 'AppendToGraph', 'AppendToLayout', 'AppendToTable', + 'AppendViolinPlot', 'AppendXYZContour', 'AutoPositionWindow', + 'AxonTelegraphFindServers', 'BackgroundInfo', 'Beep', 'BoundingBall', 'BoxSmooth', + 'BrowseURL', 'BuildMenu', 'Button', 'cd', 'Chart', 'CheckBox', 'CheckDisplayed', + 'ChooseColor', 'Close', 'CloseHelp', 'CloseMovie', 'CloseProc', 'ColorScale', + 'ColorTab2Wave', 'Concatenate', 'ControlBar', 'ControlInfo', 'ControlUpdate', + 'ConvertGlobalStringTextEncoding', 'ConvexHull', 'Convolve', 'CopyDimLabels', + 'CopyFile', 'CopyFolder', 'CopyScales', 'Correlate', 'CreateAliasShortcut', + 'CreateBrowser', 'Cross', 'CtrlBackground', 'CtrlFIFO', 'CtrlNamedBackground', + 'Cursor', 'CurveFit', 'CustomControl', 'CWT', 'DAQmx_AI_SetupReader', + 'DAQmx_AO_SetOutputs', 'DAQmx_CTR_CountEdges', 'DAQmx_CTR_OutputPulse', + 'DAQmx_CTR_Period', 'DAQmx_CTR_PulseWidth', 'DAQmx_DIO_Config', + 'DAQmx_DIO_WriteNewData', 'DAQmx_Scan', 'DAQmx_WaveformGen', 'Debugger', + 'DebuggerOptions', 'DefaultFont', 'DefaultGuiControls', 'DefaultGuiFont', + 'DefaultTextEncoding', 'DefineGuide', 'DelayUpdate', 'DeleteAnnotations', + 'DeleteFile', 'DeleteFolder', 'DeletePoints', 'Differentiate', 'dir', 'Display', + 'DisplayHelpTopic', 'DisplayProcedure', 'DoAlert', 'DoIgorMenu', 'DoUpdate', + 'DoWindow', 'DoXOPIdle', 'DPSS', 'DrawAction', 'DrawArc', 'DrawBezier', + 'DrawLine', 'DrawOval', 'DrawPICT', 'DrawPoly', 'DrawRect', 'DrawRRect', + 'DrawText', 'DrawUserShape', 'DSPDetrend', 'DSPPeriodogram', 'Duplicate', + 'DuplicateDataFolder', 'DWT', 'EdgeStats', 'Edit', 'ErrorBars', + 'EstimatePeakSizes', 'Execute', 'ExecuteScriptText', 'ExperimentInfo', 'ExperimentModified', 'ExportGizmo', 'Extract', 'FastGaussTransform', 'FastOp', - 'FBinRead', 'FBinWrite', 'FFT', 'FIFOStatus', 'FIFO2Wave', 'FilterFIR', + 'FBinRead', 'FBinWrite', 'FFT', 'FGetPos', 'FIFOStatus', 'FIFO2Wave', 'FilterFIR', 'FilterIIR', 'FindAPeak', 'FindContour', 'FindDuplicates', 'FindLevel', 'FindLevels', 'FindPeak', 'FindPointsInPoly', 'FindRoots', 'FindSequence', - 'FindValue', 'FPClustering', 'fprintf', 'FReadLine', 'FSetPos', 'FStatus', - 'FTPCreateDirectory', 'FTPDelete', 'FTPDownload', 'FTPUpload', 'FuncFit', - 'FuncFitMD', 'GBLoadWave', 'GetAxis', 'GetCamera', 'GetFileFolderInfo', + 'FindValue', 'FMaxFlat', 'FPClustering', 'fprintf', 'FReadLine', 'FSetPos', + 'FStatus', 'FTPCreateDirectory', 'FTPDelete', 'FTPDownload', 'FTPUpload', + 'FuncFit', 'FuncFitMD', 'GBLoadWave', 'GetAxis', 'GetCamera', 'GetFileFolderInfo', 'GetGizmo', 'GetLastUserMenuInfo', 'GetMarquee', 'GetMouse', 'GetSelection', - 'GetWindow', 'GPIBReadBinaryWave2', 'GPIBReadBinary2', 'GPIBReadWave2', - 'GPIBRead2', 'GPIBWriteBinaryWave2', 'GPIBWriteBinary2', 'GPIBWriteWave2', - 'GPIBWrite2', 'GPIB2', 'GraphNormal', 'GraphWaveDraw', 'GraphWaveEdit', 'Grep', - 'GroupBox', 'Hanning', 'HDF5CloseFile', 'HDF5CloseGroup', 'HDF5ConvertColors', - 'HDF5CreateFile', 'HDF5CreateGroup', 'HDF5CreateLink', 'HDF5Dump', - 'HDF5DumpErrors', 'HDF5DumpState', 'HDF5ListAttributes', 'HDF5ListGroup', - 'HDF5LoadData', 'HDF5LoadGroup', 'HDF5LoadImage', 'HDF5OpenFile', 'HDF5OpenGroup', - 'HDF5SaveData', 'HDF5SaveGroup', 'HDF5SaveImage', 'HDF5TestOperation', - 'HDF5UnlinkObject', 'HideIgorMenus', 'HideInfo', 'HideProcedures', 'HideTools', - 'HilbertTransform', 'Histogram', 'ICA', 'IFFT', 'ImageAnalyzeParticles', - 'ImageBlend', 'ImageBoundaryToMask', 'ImageEdgeDetection', 'ImageFileInfo', - 'ImageFilter', 'ImageFocus', 'ImageFromXYZ', 'ImageGenerateROIMask', 'ImageGLCM', + 'GetWindow', 'GISCreateVectorLayer', 'GISGetRasterInfo', + 'GISGetRegisteredFileInfo', 'GISGetVectorLayerInfo', 'GISLoadRasterData', + 'GISLoadVectorData', 'GISRasterizeVectorData', 'GISRegisterFile', + 'GISTransformCoords', 'GISUnRegisterFile', 'GISWriteFieldData', + 'GISWriteGeometryData', 'GISWriteRaster', 'GPIBReadBinaryWave2', + 'GPIBReadBinary2', 'GPIBReadWave2', 'GPIBRead2', 'GPIBWriteBinaryWave2', + 'GPIBWriteBinary2', 'GPIBWriteWave2', 'GPIBWrite2', 'GPIB2', 'GraphNormal', + 'GraphWaveDraw', 'GraphWaveEdit', 'Grep', 'GroupBox', 'Hanning', 'HDFInfo', + 'HDFReadImage', 'HDFReadSDS', 'HDFReadVset', 'HDF5CloseFile', 'HDF5CloseGroup', + 'HDF5ConvertColors', 'HDF5CreateFile', 'HDF5CreateGroup', 'HDF5CreateLink', + 'HDF5Dump', 'HDF5DumpErrors', 'HDF5DumpState', 'HDF5FlushFile', + 'HDF5ListAttributes', 'HDF5ListGroup', 'HDF5LoadData', 'HDF5LoadGroup', + 'HDF5LoadImage', 'HDF5OpenFile', 'HDF5OpenGroup', 'HDF5SaveData', 'HDF5SaveGroup', + 'HDF5SaveImage', 'HDF5TestOperation', 'HDF5UnlinkObject', 'HideIgorMenus', + 'HideInfo', 'HideProcedures', 'HideTools', 'HilbertTransform', 'Histogram', 'ICA', + 'IFFT', 'ImageAnalyzeParticles', 'ImageBlend', 'ImageBoundaryToMask', + 'ImageComposite', 'ImageEdgeDetection', 'ImageFileInfo', 'ImageFilter', + 'ImageFocus', 'ImageFromXYZ', 'ImageGenerateROIMask', 'ImageGLCM', 'ImageHistModification', 'ImageHistogram', 'ImageInterpolate', 'ImageLineProfile', 'ImageLoad', 'ImageMorphology', 'ImageRegistration', 'ImageRemoveBackground', 'ImageRestore', 'ImageRotate', 'ImageSave', 'ImageSeedFill', 'ImageSkeleton3d', 'ImageSnake', 'ImageStats', 'ImageThreshold', 'ImageTransform', 'ImageUnwrapPhase', 'ImageWindow', 'IndexSort', 'InsertPoints', 'Integrate', 'IntegrateODE', 'Integrate2D', 'Interpolate2', 'Interpolate3D', 'Interp3DPath', + 'ITCCloseAll2', 'ITCCloseDevice2', 'ITCConfigAllChannels2', + 'ITCConfigChannelReset2', 'ITCConfigChannelUpload2', 'ITCConfigChannel2', + 'ITCFIFOAvailableAll2', 'ITCFIFOAvailable2', 'ITCGetAllChannelsConfig2', + 'ITCGetChannelConfig2', 'ITCGetCurrentDevice2', 'ITCGetDeviceInfo2', + 'ITCGetDevices2', 'ITCGetErrorString2', 'ITCGetSerialNumber2', 'ITCGetState2', + 'ITCGetVersions2', 'ITCInitialize2', 'ITCOpenDevice2', 'ITCReadADC2', + 'ITCReadDigital2', 'ITCReadTimer2', 'ITCSelectDevice2', 'ITCSetDAC2', + 'ITCSetGlobals2', 'ITCSetModes2', 'ITCSetState2', 'ITCStartAcq2', 'ITCStopAcq2', + 'ITCUpdateFIFOPositionAll2', 'ITCUpdateFIFOPosition2', 'ITCWriteDigital2', 'JCAMPLoadWave', 'JointHistogram', 'KillBackground', 'KillControl', 'KillDataFolder', 'KillFIFO', 'KillFreeAxis', 'KillPath', 'KillPICTs', 'KillStrings', 'KillVariables', 'KillWaves', 'KillWindow', 'KMeans', 'Label', @@ -104,39 +124,48 @@ class IgorLexer(RegexLexer): 'MatrixFilter', 'MatrixGaussJ', 'MatrixGLM', 'MatrixInverse', 'MatrixLinearSolve', 'MatrixLinearSolveTD', 'MatrixLLS', 'MatrixLUBkSub', 'MatrixLUD', 'MatrixLUDTD', 'MatrixMultiply', 'MatrixOP', 'MatrixSchur', 'MatrixSolve', 'MatrixSVBkSub', - 'MatrixSVD', 'MatrixTranspose', 'MeasureStyledText', 'MLLoadWave', 'Modify', - 'ModifyBrowser', 'ModifyCamera', 'ModifyContour', 'ModifyControl', - 'ModifyControlList', 'ModifyFreeAxis', 'ModifyGizmo', 'ModifyGraph', - 'ModifyImage', 'ModifyLayout', 'ModifyPanel', 'ModifyTable', 'ModifyWaterfall', - 'MoveDataFolder', 'MoveFile', 'MoveFolder', 'MoveString', 'MoveSubwindow', - 'MoveVariable', 'MoveWave', 'MoveWindow', 'MultiTaperPSD', - 'MultiThreadingControl', 'NeuralNetworkRun', 'NeuralNetworkTrain', 'NewCamera', - 'NewDataFolder', 'NewFIFO', 'NewFIFOChan', 'NewFreeAxis', 'NewGizmo', 'NewImage', - 'NewLayout', 'NewMovie', 'NewNotebook', 'NewPanel', 'NewPath', 'NewWaterfall', - 'NI4882', 'Note', 'Notebook', 'NotebookAction', 'Open', 'OpenHelp', - 'OpenNotebook', 'Optimize', 'ParseOperationTemplate', 'PathInfo', 'PauseForUser', - 'PauseUpdate', 'PCA', 'PlayMovie', 'PlayMovieAction', 'PlaySound', - 'PopupContextualMenu', 'PopupMenu', 'Preferences', 'PrimeFactors', 'Print', - 'printf', 'PrintGraphs', 'PrintLayout', 'PrintNotebook', 'PrintSettings', - 'PrintTable', 'Project', 'PulseStats', 'PutScrapText', 'pwd', 'Quit', - 'RatioFromNumber', 'Redimension', 'Remove', 'RemoveContour', 'RemoveFromGizmo', - 'RemoveFromGraph', 'RemoveFromLayout', 'RemoveFromTable', 'RemoveImage', - 'RemoveLayoutObjects', 'RemovePath', 'Rename', 'RenameDataFolder', 'RenamePath', - 'RenamePICT', 'RenameWindow', 'ReorderImages', 'ReorderTraces', 'ReplaceText', - 'ReplaceWave', 'Resample', 'ResumeUpdate', 'Reverse', 'Rotate', 'Save', - 'SaveData', 'SaveExperiment', 'SaveGraphCopy', 'SaveNotebook', + 'MatrixSVD', 'MatrixTranspose', 'MCC_FindServers', 'MeasureStyledText', + 'MFR_CheckForNewBricklets', + 'MFR_CloseResultFile', 'MFR_CreateOverviewTable', 'MFR_GetBrickletCount', + 'MFR_GetBrickletData', 'MFR_GetBrickletDeployData', 'MFR_GetBrickletMetaData', + 'MFR_GetBrickletRawData', 'MFR_GetReportTemplate', 'MFR_GetResultFileMetaData', + 'MFR_GetResultFileName', 'MFR_GetVernissageVersion', 'MFR_GetVersion', + 'MFR_GetXOPErrorMessage', 'MFR_OpenResultFile', + 'MLLoadWave', 'Modify', 'ModifyBoxPlot', 'ModifyBrowser', 'ModifyCamera', + 'ModifyContour', 'ModifyControl', 'ModifyControlList', 'ModifyFreeAxis', + 'ModifyGizmo', 'ModifyGraph', 'ModifyImage', 'ModifyLayout', 'ModifyPanel', + 'ModifyTable', 'ModifyViolinPlot', 'ModifyWaterfall', 'MoveDataFolder', + 'MoveFile', 'MoveFolder', 'MoveString', 'MoveSubwindow', 'MoveVariable', + 'MoveWave', 'MoveWindow', 'MultiTaperPSD', 'MultiThreadingControl', + 'NC_CloseFile', 'NC_DumpErrors', 'NC_Inquire', 'NC_ListAttributes', + 'NC_ListObjects', 'NC_LoadData', 'NC_OpenFile', 'NeuralNetworkRun', + 'NeuralNetworkTrain', 'NewCamera', 'NewDataFolder', 'NewFIFO', 'NewFIFOChan', + 'NewFreeAxis', 'NewGizmo', 'NewImage', 'NewLayout', 'NewMovie', 'NewNotebook', + 'NewPanel', 'NewPath', 'NewWaterfall', 'NILoadWave', 'NI4882', 'Note', 'Notebook', + 'NotebookAction', 'Open', 'OpenHelp', 'OpenNotebook', 'Optimize', + 'ParseOperationTemplate', 'PathInfo', 'PauseForUser', 'PauseUpdate', 'PCA', + 'PlayMovie', 'PlayMovieAction', 'PlaySound', 'PopupContextualMenu', 'PopupMenu', + 'Preferences', 'PrimeFactors', 'Print', 'printf', 'PrintGraphs', 'PrintLayout', + 'PrintNotebook', 'PrintSettings', 'PrintTable', 'Project', 'PulseStats', + 'PutScrapText', 'pwd', 'Quit', 'RatioFromNumber', 'Redimension', 'Remez', + 'Remove', 'RemoveContour', 'RemoveFromGizmo', 'RemoveFromGraph', + 'RemoveFromLayout', 'RemoveFromTable', 'RemoveImage', 'RemoveLayoutObjects', + 'RemovePath', 'Rename', 'RenameDataFolder', 'RenamePath', 'RenamePICT', + 'RenameWindow', 'ReorderImages', 'ReorderTraces', 'ReplaceText', 'ReplaceWave', + 'Resample', 'ResumeUpdate', 'Reverse', 'Rotate', 'Save', 'SaveData', + 'SaveExperiment', 'SaveGizmoCopy', 'SaveGraphCopy', 'SaveNotebook', 'SavePackagePreferences', 'SavePICT', 'SaveTableCopy', 'SetActiveSubwindow', 'SetAxis', 'SetBackground', 'SetDashPattern', 'SetDataFolder', 'SetDimLabel', - 'SetDrawEnv', 'SetDrawLayer', 'SetFileFolderInfo', 'SetFormula', 'SetIgorHook', - 'SetIgorMenuMode', 'SetIgorOption', 'SetMarquee', 'SetProcessSleep', - 'SetRandomSeed', 'SetScale', 'SetVariable', 'SetWaveLock', 'SetWaveTextEncoding', - 'SetWindow', 'ShowIgorMenus', 'ShowInfo', 'ShowTools', 'Silent', 'Sleep', - 'Slider', 'Smooth', 'SmoothCustom', 'Sort', 'SortColumns', 'SoundInRecord', - 'SoundInSet', 'SoundInStartChart', 'SoundInStatus', 'SoundInStopChart', - 'SoundLoadWave', 'SoundSaveWave', 'SphericalInterpolate', 'SphericalTriangulate', - 'SplitString', 'SplitWave', 'sprintf', 'sscanf', 'Stack', 'StackWindows', - 'StatsAngularDistanceTest', 'StatsANOVA1Test', 'StatsANOVA2NRTest', - 'StatsANOVA2RMTest', 'StatsANOVA2Test', 'StatsChiTest', + 'SetDrawEnv', 'SetDrawLayer', 'SetFileFolderInfo', 'SetFormula', 'SetIdlePeriod', + 'SetIgorHook', 'SetIgorMenuMode', 'SetIgorOption', 'SetMarquee', + 'SetProcessSleep', 'SetRandomSeed', 'SetScale', 'SetVariable', 'SetWaveLock', + 'SetWaveTextEncoding', 'SetWindow', 'ShowIgorMenus', 'ShowInfo', 'ShowTools', + 'Silent', 'Sleep', 'Slider', 'Smooth', 'SmoothCustom', 'Sort', 'SortColumns', + 'SoundInRecord', 'SoundInSet', 'SoundInStartChart', 'SoundInStatus', + 'SoundInStopChart', 'SoundLoadWave', 'SoundSaveWave', 'SphericalInterpolate', + 'SphericalTriangulate', 'SplitString', 'SplitWave', 'sprintf', 'SQLHighLevelOp', + 'sscanf', 'Stack', 'StackWindows', 'StatsAngularDistanceTest', 'StatsANOVA1Test', + 'StatsANOVA2NRTest', 'StatsANOVA2RMTest', 'StatsANOVA2Test', 'StatsChiTest', 'StatsCircularCorrelationTest', 'StatsCircularMeans', 'StatsCircularMoments', 'StatsCircularTwoSampleTest', 'StatsCochranTest', 'StatsContingencyTable', 'StatsDIPTest', 'StatsDunnettTest', 'StatsFriedmanTest', 'StatsFTest', @@ -148,23 +177,30 @@ class IgorLexer(RegexLexer): 'StatsSignTest', 'StatsSRTest', 'StatsTTest', 'StatsTukeyTest', 'StatsVariancesTest', 'StatsWatsonUSquaredTest', 'StatsWatsonWilliamsTest', 'StatsWheelerWatsonTest', 'StatsWilcoxonRankTest', 'StatsWRCorrelationTest', - 'String', 'StructGet', 'StructPut', 'SumDimension', 'SumSeries', 'TabControl', - 'Tag', 'TextBox', 'ThreadGroupPutDF', 'ThreadStart', 'Tile', 'TileWindows', + 'STFT', 'String', 'StructFill', 'StructGet', 'StructPut', 'SumDimension', + 'SumSeries', 'TabControl', 'Tag', 'TDMLoadData', 'TDMSaveData', 'TextBox', + 'ThreadGroupPutDF', 'ThreadStart', 'TickWavesFromAxis', 'Tile', 'TileWindows', 'TitleBox', 'ToCommandLine', 'ToolsGrid', 'Triangulate3d', 'Unwrap', 'URLRequest', 'ValDisplay', 'Variable', 'VDTClosePort2', 'VDTGetPortList2', 'VDTGetStatus2', 'VDTOpenPort2', 'VDTOperationsPort2', 'VDTReadBinaryWave2', 'VDTReadBinary2', 'VDTReadHexWave2', 'VDTReadHex2', 'VDTReadWave2', 'VDTRead2', 'VDTTerminalPort2', 'VDTWriteBinaryWave2', 'VDTWriteBinary2', 'VDTWriteHexWave2', 'VDTWriteHex2', - 'VDTWriteWave2', 'VDTWrite2', 'VDT2', 'WaveMeanStdv', 'WaveStats', + 'VDTWriteWave2', 'VDTWrite2', 'VDT2', 'VISAControl', 'VISARead', 'VISAReadBinary', + 'VISAReadBinaryWave', 'VISAReadWave', 'VISAWrite', 'VISAWriteBinary', + 'VISAWriteBinaryWave', 'VISAWriteWave', 'WaveMeanStdv', 'WaveStats', 'WaveTransform', 'wfprintf', 'WignerTransform', 'WindowFunction', 'XLLoadWave' ) functions = ( 'abs', 'acos', 'acosh', 'AddListItem', 'AiryA', 'AiryAD', 'AiryB', 'AiryBD', 'alog', 'AnnotationInfo', 'AnnotationList', 'area', 'areaXY', 'asin', 'asinh', - 'atan', 'atanh', 'atan2', 'AxisInfo', 'AxisList', 'AxisValFromPixel', 'Besseli', - 'Besselj', 'Besselk', 'Bessely', 'beta', 'betai', 'BinarySearch', - 'BinarySearchInterp', 'binomial', 'binomialln', 'binomialNoise', 'cabs', - 'CaptureHistory', 'CaptureHistoryStart', 'ceil', 'cequal', 'char2num', + 'atan', 'atanh', 'atan2', 'AxisInfo', 'AxisList', 'AxisValFromPixel', + 'AxonTelegraphAGetDataNum', 'AxonTelegraphAGetDataString', + 'AxonTelegraphAGetDataStruct', 'AxonTelegraphGetDataNum', + 'AxonTelegraphGetDataString', 'AxonTelegraphGetDataStruct', + 'AxonTelegraphGetTimeoutMs', 'AxonTelegraphSetTimeoutMs', 'Base64Decode', + 'Base64Encode', 'Besseli', 'Besselj', 'Besselk', 'Bessely', 'beta', 'betai', + 'BinarySearch', 'BinarySearchInterp', 'binomial', 'binomialln', 'binomialNoise', + 'cabs', 'CaptureHistory', 'CaptureHistoryStart', 'ceil', 'cequal', 'char2num', 'chebyshev', 'chebyshevU', 'CheckName', 'ChildWindowList', 'CleanupName', 'cmplx', 'cmpstr', 'conj', 'ContourInfo', 'ContourNameList', 'ContourNameToWaveRef', 'ContourZ', 'ControlNameList', 'ConvertTextEncoding', 'cos', 'cosh', @@ -172,37 +208,70 @@ class IgorLexer(RegexLexer): 'CreationDate', 'csc', 'csch', 'CsrInfo', 'CsrWave', 'CsrWaveRef', 'CsrXWave', 'CsrXWaveRef', 'CTabList', 'DataFolderDir', 'DataFolderExists', 'DataFolderRefsEqual', 'DataFolderRefStatus', 'date', 'datetime', 'DateToJulian', - 'date2secs', 'Dawson', 'DDERequestString', 'defined', 'deltax', 'digamma', - 'dilogarithm', 'DimDelta', 'DimOffset', 'DimSize', 'ei', 'enoise', 'equalWaves', - 'erf', 'erfc', 'erfcw', 'exists', 'exp', 'ExpConvExp', 'ExpConvExpFit', - 'ExpConvExpFitBL', 'ExpConvExpFit1Shape', 'ExpConvExpFit1ShapeBL', 'ExpGauss', - 'ExpGaussFit', 'ExpGaussFitBL', 'ExpGaussFit1Shape', 'ExpGaussFit1ShapeBL', - 'expInt', 'expIntegralE1', 'expNoise', 'factorial', 'fakedata', 'faverage', - 'faverageXY', 'FetchURL', 'FindDimLabel', 'FindListItem', 'floor', 'FontList', - 'FontSizeHeight', 'FontSizeStringWidth', 'FresnelCos', 'FresnelSin', + 'date2secs', 'Dawson', 'defined', 'deltax', 'digamma', 'dilogarithm', 'DimDelta', + 'DimOffset', 'DimSize', 'ei', 'enoise', 'equalWaves', 'erf', 'erfc', 'erfcw', + 'exists', 'exp', 'expInt', 'expIntegralE1', 'expNoise', 'factorial', 'Faddeeva', + 'fakedata', 'faverage', 'faverageXY', 'fDAQmx_AI_GetReader', + 'fDAQmx_AO_UpdateOutputs', 'fDAQmx_ConnectTerminals', 'fDAQmx_CTR_Finished', + 'fDAQmx_CTR_IsFinished', 'fDAQmx_CTR_IsPulseFinished', 'fDAQmx_CTR_ReadCounter', + 'fDAQmx_CTR_ReadWithOptions', 'fDAQmx_CTR_SetPulseFrequency', 'fDAQmx_CTR_Start', + 'fDAQmx_DeviceNames', 'fDAQmx_DIO_Finished', 'fDAQmx_DIO_PortWidth', + 'fDAQmx_DIO_Read', 'fDAQmx_DIO_Write', 'fDAQmx_DisconnectTerminals', + 'fDAQmx_ErrorString', 'fDAQmx_ExternalCalDate', 'fDAQmx_NumAnalogInputs', + 'fDAQmx_NumAnalogOutputs', 'fDAQmx_NumCounters', 'fDAQmx_NumDIOPorts', + 'fDAQmx_ReadChan', 'fDAQmx_ReadNamedChan', 'fDAQmx_ResetDevice', + 'fDAQmx_ScanGetAvailable', 'fDAQmx_ScanGetNextIndex', 'fDAQmx_ScanStart', + 'fDAQmx_ScanStop', 'fDAQmx_ScanWait', 'fDAQmx_ScanWaitWithTimeout', + 'fDAQmx_SelfCalDate', 'fDAQmx_SelfCalibration', 'fDAQmx_WaveformStart', + 'fDAQmx_WaveformStop', 'fDAQmx_WF_IsFinished', 'fDAQmx_WF_WaitUntilFinished', + 'fDAQmx_WriteChan', 'FetchURL', 'FindDimLabel', 'FindListItem', 'floor', + 'FontList', 'FontSizeHeight', 'FontSizeStringWidth', 'FresnelCos', 'FresnelSin', 'FuncRefInfo', 'FunctionInfo', 'FunctionList', 'FunctionPath', 'gamma', 'gammaEuler', 'gammaInc', 'gammaNoise', 'gammln', 'gammp', 'gammq', 'Gauss', - 'GaussFit', 'GaussFitBL', 'GaussFit1Width', 'GaussFit1WidthBL', 'Gauss1D', - 'Gauss2D', 'gcd', 'GetBrowserLine', 'GetBrowserSelection', 'GetDataFolder', - 'GetDataFolderDFR', 'GetDefaultFont', 'GetDefaultFontSize', 'GetDefaultFontStyle', - 'GetDimLabel', 'GetEnvironmentVariable', 'GetErrMessage', 'GetFormula', - 'GetIndependentModuleName', 'GetIndexedObjName', 'GetIndexedObjNameDFR', - 'GetKeyState', 'GetRTErrMessage', 'GetRTError', 'GetRTLocation', 'GetRTLocInfo', - 'GetRTStackInfo', 'GetScrapText', 'GetUserData', 'GetWavesDataFolder', - 'GetWavesDataFolderDFR', 'GizmoInfo', 'GizmoScale', 'gnoise', 'GrepList', - 'GrepString', 'GuideInfo', 'GuideNameList', 'Hash', 'hcsr', 'HDF5AttributeInfo', + 'Gauss1D', 'Gauss2D', 'gcd', 'GetBrowserLine', 'GetBrowserSelection', + 'GetDataFolder', 'GetDataFolderDFR', 'GetDefaultFont', 'GetDefaultFontSize', + 'GetDefaultFontStyle', 'GetDimLabel', 'GetEnvironmentVariable', 'GetErrMessage', + 'GetFormula', 'GetIndependentModuleName', 'GetIndexedObjName', + 'GetIndexedObjNameDFR', 'GetKeyState', 'GetRTErrMessage', 'GetRTError', + 'GetRTLocation', 'GetRTLocInfo', 'GetRTStackInfo', 'GetScrapText', 'GetUserData', + 'GetWavesDataFolder', 'GetWavesDataFolderDFR', 'GISGetAllFileFormats', + 'GISSRefsAreEqual', 'GizmoInfo', 'GizmoScale', 'gnoise', 'GrepList', 'GrepString', + 'GuideInfo', 'GuideNameList', 'Hash', 'hcsr', 'HDF5AttributeInfo', 'HDF5DatasetInfo', 'HDF5LibraryInfo', 'HDF5TypeInfo', 'hermite', 'hermiteGauss', 'HyperGNoise', 'HyperGPFQ', 'HyperG0F1', 'HyperG1F1', 'HyperG2F1', 'IgorInfo', 'IgorVersion', 'imag', 'ImageInfo', 'ImageNameList', 'ImageNameToWaveRef', - 'IndependentModuleList', 'IndexedDir', 'IndexedFile', 'Inf', 'Integrate1D', - 'interp', 'Interp2D', 'Interp3D', 'inverseERF', 'inverseERFC', 'ItemsInList', - 'JacobiCn', 'JacobiSn', 'JulianToDate', 'Laguerre', 'LaguerreA', 'LaguerreGauss', - 'LambertW', 'LayoutInfo', 'leftx', 'LegendreA', 'limit', 'ListMatch', - 'ListToTextWave', 'ListToWaveRefWave', 'ln', 'log', 'logNormalNoise', - 'LorentzianFit', 'LorentzianFitBL', 'LorentzianFit1Width', - 'LorentzianFit1WidthBL', 'lorentzianNoise', 'LowerStr', 'MacroList', 'magsqr', - 'MandelbrotPoint', 'MarcumQ', 'MatrixCondition', 'MatrixDet', 'MatrixDot', - 'MatrixRank', 'MatrixTrace', 'max', 'mean', 'median', 'min', 'mod', 'ModDate', + 'IndependentModuleList', 'IndexedDir', 'IndexedFile', 'IndexToScale', 'Inf', + 'Integrate1D', 'interp', 'Interp2D', 'Interp3D', 'inverseERF', 'inverseERFC', + 'ItemsInList', 'JacobiCn', 'JacobiSn', 'JulianToDate', 'Laguerre', 'LaguerreA', + 'LaguerreGauss', 'LambertW', 'LayoutInfo', 'leftx', 'LegendreA', 'limit', + 'ListMatch', 'ListToTextWave', 'ListToWaveRefWave', 'ln', 'log', 'logNormalNoise', + 'lorentzianNoise', 'LowerStr', 'MacroList', 'magsqr', 'MandelbrotPoint', + 'MarcumQ', 'MatrixCondition', 'MatrixDet', 'MatrixDot', 'MatrixRank', + 'MatrixTrace', 'max', 'MCC_AutoBridgeBal', 'MCC_AutoFastComp', + 'MCC_AutoPipetteOffset', 'MCC_AutoSlowComp', 'MCC_AutoWholeCellComp', + 'MCC_GetBridgeBalEnable', 'MCC_GetBridgeBalResist', 'MCC_GetFastCompCap', + 'MCC_GetFastCompTau', 'MCC_GetHolding', 'MCC_GetHoldingEnable', 'MCC_GetMode', + 'MCC_GetNeutralizationCap', 'MCC_GetNeutralizationEnable', + 'MCC_GetOscKillerEnable', 'MCC_GetPipetteOffset', 'MCC_GetPrimarySignalGain', + 'MCC_GetPrimarySignalHPF', 'MCC_GetPrimarySignalLPF', 'MCC_GetRsCompBandwidth', + 'MCC_GetRsCompCorrection', 'MCC_GetRsCompEnable', 'MCC_GetRsCompPrediction', + 'MCC_GetSecondarySignalGain', 'MCC_GetSecondarySignalLPF', 'MCC_GetSlowCompCap', + 'MCC_GetSlowCompTau', 'MCC_GetSlowCompTauX20Enable', + 'MCC_GetSlowCurrentInjEnable', 'MCC_GetSlowCurrentInjLevel', + 'MCC_GetSlowCurrentInjSetlTime', 'MCC_GetWholeCellCompCap', + 'MCC_GetWholeCellCompEnable', 'MCC_GetWholeCellCompResist', + 'MCC_SelectMultiClamp700B', 'MCC_SetBridgeBalEnable', 'MCC_SetBridgeBalResist', + 'MCC_SetFastCompCap', 'MCC_SetFastCompTau', 'MCC_SetHolding', + 'MCC_SetHoldingEnable', 'MCC_SetMode', 'MCC_SetNeutralizationCap', + 'MCC_SetNeutralizationEnable', 'MCC_SetOscKillerEnable', 'MCC_SetPipetteOffset', + 'MCC_SetPrimarySignalGain', 'MCC_SetPrimarySignalHPF', 'MCC_SetPrimarySignalLPF', + 'MCC_SetRsCompBandwidth', 'MCC_SetRsCompCorrection', 'MCC_SetRsCompEnable', + 'MCC_SetRsCompPrediction', 'MCC_SetSecondarySignalGain', + 'MCC_SetSecondarySignalLPF', 'MCC_SetSlowCompCap', 'MCC_SetSlowCompTau', + 'MCC_SetSlowCompTauX20Enable', 'MCC_SetSlowCurrentInjEnable', + 'MCC_SetSlowCurrentInjLevel', 'MCC_SetSlowCurrentInjSetlTime', 'MCC_SetTimeoutMs', + 'MCC_SetWholeCellCompCap', 'MCC_SetWholeCellCompEnable', + 'MCC_SetWholeCellCompResist', 'mean', 'median', 'min', 'mod', 'ModDate', 'MPFXEMGPeak', 'MPFXExpConvExpPeak', 'MPFXGaussPeak', 'MPFXLorenzianPeak', 'MPFXVoigtPeak', 'NameOfWave', 'NaN', 'NewFreeDataFolder', 'NewFreeWave', 'norm', 'NormalizeUnicode', 'note', 'NumberByKey', 'numpnts', 'numtype', @@ -217,9 +286,30 @@ class IgorLexer(RegexLexer): 'SelectNumber', 'SelectString', 'SetEnvironmentVariable', 'sign', 'sin', 'sinc', 'sinh', 'sinIntegral', 'SortList', 'SpecialCharacterInfo', 'SpecialCharacterList', 'SpecialDirPath', 'SphericalBessJ', 'SphericalBessJD', 'SphericalBessY', - 'SphericalBessYD', 'SphericalHarmonics', 'sqrt', 'StartMSTimer', 'StatsBetaCDF', - 'StatsBetaPDF', 'StatsBinomialCDF', 'StatsBinomialPDF', 'StatsCauchyCDF', - 'StatsCauchyPDF', 'StatsChiCDF', 'StatsChiPDF', 'StatsCMSSDCDF', + 'SphericalBessYD', 'SphericalHarmonics', 'SQLAllocHandle', 'SQLAllocStmt', + 'SQLBinaryWavesToTextWave', 'SQLBindCol', 'SQLBindParameter', 'SQLBrowseConnect', + 'SQLBulkOperations', 'SQLCancel', 'SQLCloseCursor', 'SQLColAttributeNum', + 'SQLColAttributeStr', 'SQLColumnPrivileges', 'SQLColumns', 'SQLConnect', + 'SQLDataSources', 'SQLDescribeCol', 'SQLDescribeParam', 'SQLDisconnect', + 'SQLDriverConnect', 'SQLDrivers', 'SQLEndTran', 'SQLError', 'SQLExecDirect', + 'SQLExecute', 'SQLFetch', 'SQLFetchScroll', 'SQLForeignKeys', 'SQLFreeConnect', + 'SQLFreeEnv', 'SQLFreeHandle', 'SQLFreeStmt', 'SQLGetConnectAttrNum', + 'SQLGetConnectAttrStr', 'SQLGetCursorName', 'SQLGetDataNum', 'SQLGetDataStr', + 'SQLGetDescFieldNum', 'SQLGetDescFieldStr', 'SQLGetDescRec', 'SQLGetDiagFieldNum', + 'SQLGetDiagFieldStr', 'SQLGetDiagRec', 'SQLGetEnvAttrNum', 'SQLGetEnvAttrStr', + 'SQLGetFunctions', 'SQLGetInfoNum', 'SQLGetInfoStr', 'SQLGetStmtAttrNum', + 'SQLGetStmtAttrStr', 'SQLGetTypeInfo', 'SQLMoreResults', 'SQLNativeSql', + 'SQLNumParams', 'SQLNumResultCols', 'SQLNumResultRowsIfKnown', + 'SQLNumRowsFetched', 'SQLParamData', 'SQLPrepare', 'SQLPrimaryKeys', + 'SQLProcedureColumns', 'SQLProcedures', 'SQLPutData', 'SQLReinitialize', + 'SQLRowCount', 'SQLSetConnectAttrNum', 'SQLSetConnectAttrStr', 'SQLSetCursorName', + 'SQLSetDescFieldNum', 'SQLSetDescFieldStr', 'SQLSetDescRec', 'SQLSetEnvAttrNum', + 'SQLSetEnvAttrStr', 'SQLSetPos', 'SQLSetStmtAttrNum', 'SQLSetStmtAttrStr', + 'SQLSpecialColumns', 'SQLStatistics', 'SQLTablePrivileges', 'SQLTables', + 'SQLTextWaveToBinaryWaves', 'SQLTextWaveTo2DBinaryWave', 'SQLUpdateBoundValues', + 'SQLXOPCheckState', 'SQL2DBinaryWaveToTextWave', 'sqrt', 'StartMSTimer', + 'StatsBetaCDF', 'StatsBetaPDF', 'StatsBinomialCDF', 'StatsBinomialPDF', + 'StatsCauchyCDF', 'StatsCauchyPDF', 'StatsChiCDF', 'StatsChiPDF', 'StatsCMSSDCDF', 'StatsCorrelation', 'StatsDExpCDF', 'StatsDExpPDF', 'StatsErlangCDF', 'StatsErlangPDF', 'StatsErrorPDF', 'StatsEValueCDF', 'StatsEValuePDF', 'StatsExpCDF', 'StatsExpPDF', 'StatsFCDF', 'StatsFPDF', 'StatsFriedmanCDF', @@ -250,19 +340,66 @@ class IgorLexer(RegexLexer): 'StopMSTimer', 'StringByKey', 'stringCRC', 'StringFromList', 'StringList', 'stringmatch', 'strlen', 'strsearch', 'StrVarOrDefault', 'str2num', 'StudentA', 'StudentT', 'sum', 'SVAR_Exists', 'TableInfo', 'TagVal', 'TagWaveRef', 'tan', - 'tanh', 'TextEncodingCode', 'TextEncodingName', 'TextFile', 'ThreadGroupCreate', + 'tango_close_device', 'tango_command_inout', 'tango_compute_image_proj', + 'tango_get_dev_attr_list', 'tango_get_dev_black_box', 'tango_get_dev_cmd_list', + 'tango_get_dev_status', 'tango_get_dev_timeout', 'tango_get_error_stack', + 'tango_open_device', 'tango_ping_device', 'tango_read_attribute', + 'tango_read_attributes', 'tango_reload_dev_interface', + 'tango_resume_attr_monitor', 'tango_set_attr_monitor_period', + 'tango_set_dev_timeout', 'tango_start_attr_monitor', 'tango_stop_attr_monitor', + 'tango_suspend_attr_monitor', 'tango_write_attribute', 'tango_write_attributes', + 'tanh', 'TDMAddChannel', 'TDMAddGroup', 'TDMAppendDataValues', + 'TDMAppendDataValuesTime', 'TDMChannelPropertyExists', 'TDMCloseChannel', + 'TDMCloseFile', 'TDMCloseGroup', 'TDMCreateChannelProperty', 'TDMCreateFile', + 'TDMCreateFileProperty', 'TDMCreateGroupProperty', 'TDMFilePropertyExists', + 'TDMGetChannelPropertyNames', 'TDMGetChannelPropertyNum', + 'TDMGetChannelPropertyStr', 'TDMGetChannelPropertyTime', + 'TDMGetChannelPropertyType', 'TDMGetChannels', 'TDMGetChannelStringPropertyLen', + 'TDMGetDataType', 'TDMGetDataValues', 'TDMGetDataValuesTime', + 'TDMGetFilePropertyNames', 'TDMGetFilePropertyNum', 'TDMGetFilePropertyStr', + 'TDMGetFilePropertyTime', 'TDMGetFilePropertyType', 'TDMGetFileStringPropertyLen', + 'TDMGetGroupPropertyNames', 'TDMGetGroupPropertyNum', 'TDMGetGroupPropertyStr', + 'TDMGetGroupPropertyTime', 'TDMGetGroupPropertyType', 'TDMGetGroups', + 'TDMGetGroupStringPropertyLen', 'TDMGetLibraryErrorDescription', + 'TDMGetNumChannelProperties', 'TDMGetNumChannels', 'TDMGetNumDataValues', + 'TDMGetNumFileProperties', 'TDMGetNumGroupProperties', 'TDMGetNumGroups', + 'TDMGroupPropertyExists', 'TDMOpenFile', 'TDMOpenFileEx', 'TDMRemoveChannel', + 'TDMRemoveGroup', 'TDMReplaceDataValues', 'TDMReplaceDataValuesTime', + 'TDMSaveFile', 'TDMSetChannelPropertyNum', 'TDMSetChannelPropertyStr', + 'TDMSetChannelPropertyTime', 'TDMSetDataValues', 'TDMSetDataValuesTime', + 'TDMSetFilePropertyNum', 'TDMSetFilePropertyStr', 'TDMSetFilePropertyTime', + 'TDMSetGroupPropertyNum', 'TDMSetGroupPropertyStr', 'TDMSetGroupPropertyTime', + 'TextEncodingCode', 'TextEncodingName', 'TextFile', 'ThreadGroupCreate', 'ThreadGroupGetDF', 'ThreadGroupGetDFR', 'ThreadGroupRelease', 'ThreadGroupWait', 'ThreadProcessorCount', 'ThreadReturnValue', 'ticks', 'time', 'TraceFromPixel', - 'TraceInfo', 'TraceNameList', 'TraceNameToWaveRef', 'trunc', 'UniqueName', - 'UnPadString', 'UnsetEnvironmentVariable', 'UpperStr', 'URLDecode', 'URLEncode', - 'VariableList', 'Variance', 'vcsr', 'Voigt', 'VoigtFit', 'VoigtFitBL', - 'VoigtFit1Shape', 'VoigtFit1ShapeBL', 'VoigtFit1Shape1Width', - 'VoigtFit1Shape1WidthBL', 'VoigtFunc', 'WaveCRC', 'WaveDims', 'WaveExists', - 'WaveInfo', 'WaveList', 'WaveMax', 'WaveMin', 'WaveName', 'WaveRefIndexed', + 'TraceInfo', 'TraceNameList', 'TraceNameToWaveRef', 'TrimString', 'trunc', + 'UniqueName', 'UnPadString', 'UnsetEnvironmentVariable', 'UpperStr', 'URLDecode', + 'URLEncode', 'VariableList', 'Variance', 'vcsr', 'viAssertIntrSignal', + 'viAssertTrigger', 'viAssertUtilSignal', 'viClear', 'viClose', 'viDisableEvent', + 'viDiscardEvents', 'viEnableEvent', 'viFindNext', 'viFindRsrc', 'viGetAttribute', + 'viGetAttributeString', 'viGpibCommand', 'viGpibControlATN', 'viGpibControlREN', + 'viGpibPassControl', 'viGpibSendIFC', 'viIn8', 'viIn16', 'viIn32', 'viLock', + 'viMapAddress', 'viMapTrigger', 'viMemAlloc', 'viMemFree', 'viMoveIn8', + 'viMoveIn16', 'viMoveIn32', 'viMoveOut8', 'viMoveOut16', 'viMoveOut32', 'viOpen', + 'viOpenDefaultRM', 'viOut8', 'viOut16', 'viOut32', 'viPeek8', 'viPeek16', + 'viPeek32', 'viPoke8', 'viPoke16', 'viPoke32', 'viRead', 'viReadSTB', + 'viSetAttribute', 'viSetAttributeString', 'viStatusDesc', 'viTerminate', + 'viUnlock', 'viUnmapAddress', 'viUnmapTrigger', 'viUsbControlIn', + 'viUsbControlOut', 'viVxiCommandQuery', 'viWaitOnEvent', 'viWrite', 'VoigtFunc', + 'VoigtPeak', 'WaveCRC', 'WaveDims', 'WaveExists', 'WaveHash', 'WaveInfo', + 'WaveList', 'WaveMax', 'WaveMin', 'WaveName', 'WaveRefIndexed', 'WaveRefIndexedDFR', 'WaveRefsEqual', 'WaveRefWaveToList', 'WaveTextEncoding', 'WaveType', 'WaveUnits', 'WhichListItem', 'WinList', 'WinName', 'WinRecreation', - 'WinType', 'WMFindWholeWord', 'WNoise', 'xcsr', 'XWaveName', 'XWaveRefFromTrace', - 'x2pnt', 'zcsr', 'ZernikeR', 'zeta' + 'WinType', 'wnoise', 'xcsr', 'XWaveName', 'XWaveRefFromTrace', 'x2pnt', 'zcsr', + 'ZernikeR', 'zeromq_client_connect', 'zeromq_client_connect', + 'zeromq_client_recv', 'zeromq_client_recv', 'zeromq_client_send', + 'zeromq_client_send', 'zeromq_handler_start', 'zeromq_handler_start', + 'zeromq_handler_stop', 'zeromq_handler_stop', 'zeromq_server_bind', + 'zeromq_server_bind', 'zeromq_server_recv', 'zeromq_server_recv', + 'zeromq_server_send', 'zeromq_server_send', 'zeromq_set', 'zeromq_set', + 'zeromq_stop', 'zeromq_stop', 'zeromq_test_callfunction', + 'zeromq_test_callfunction', 'zeromq_test_serializeWave', + 'zeromq_test_serializeWave', 'zeta' ) tokens = { diff --git a/pygments/lexers/inferno.py b/pygments/lexers/inferno.py index 5fc5a0ba..f29808cf 100644 --- a/pygments/lexers/inferno.py +++ b/pygments/lexers/inferno.py @@ -5,7 +5,7 @@ Lexers for Inferno os and all the related stuff. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -64,7 +64,7 @@ class LimboLexer(RegexLexer): (r'(byte|int|big|real|string|array|chan|list|adt' r'|fn|ref|of|module|self|type)\b', Keyword.Type), (r'(con|iota|nil)\b', Keyword.Constant), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'statement' : [ include('whitespace'), diff --git a/pygments/lexers/installers.py b/pygments/lexers/installers.py index 0323d140..8c8c39c8 100644 --- a/pygments/lexers/installers.py +++ b/pygments/lexers/installers.py @@ -5,7 +5,7 @@ Lexers for installer/packager DSLs and formats. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/int_fiction.py b/pygments/lexers/int_fiction.py index f280a56d..89d5bccc 100644 --- a/pygments/lexers/int_fiction.py +++ b/pygments/lexers/int_fiction.py @@ -5,7 +5,7 @@ Lexers for interactive fiction languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -911,7 +911,7 @@ class Tads3Lexer(RegexLexer): 'block?/root': [ (r'\{', Punctuation, ('#pop', 'block')), include('whitespace'), - (r'(?=[[\'"<(:])', Text, # It might be a VerbRule macro. + (r'(?=[\[\'"<(:])', Text, # It might be a VerbRule macro. ('#pop', 'object-body/no-braces', 'grammar', 'grammar-rules')), # It might be a macro like DefineAction. default(('#pop', 'object-body/no-braces')) diff --git a/pygments/lexers/iolang.py b/pygments/lexers/iolang.py index bbc17faf..f33c8713 100644 --- a/pygments/lexers/iolang.py +++ b/pygments/lexers/iolang.py @@ -5,7 +5,7 @@ Lexers for the Io language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -49,7 +49,7 @@ class IoLexer(RegexLexer): # names (r'(Object|list|List|Map|args|Sequence|Coroutine|File)\b', Name.Builtin), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), # numbers (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), (r'\d+', Number.Integer) diff --git a/pygments/lexers/j.py b/pygments/lexers/j.py index 434964fe..baec89ef 100644 --- a/pygments/lexers/j.py +++ b/pygments/lexers/j.py @@ -5,7 +5,7 @@ Lexer for the J programming language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -52,13 +52,13 @@ class JLexer(RegexLexer): Name.Function, 'explicitDefinition'), # Flow Control - (words(('for_', 'goto_', 'label_'), suffix=validName+'\.'), Name.Label), + (words(('for_', 'goto_', 'label_'), suffix=validName+r'\.'), Name.Label), (words(( 'assert', 'break', 'case', 'catch', 'catchd', 'catcht', 'continue', 'do', 'else', 'elseif', 'end', 'fcase', 'for', 'if', 'return', 'select', 'throw', 'try', 'while', 'whilst', - ), suffix='\.'), Name.Label), + ), suffix=r'\.'), Name.Label), # Variable Names (validName, Name.Variable), diff --git a/pygments/lexers/javascript.py b/pygments/lexers/javascript.py index 862535c9..e61c451e 100644 --- a/pygments/lexers/javascript.py +++ b/pygments/lexers/javascript.py @@ -5,7 +5,7 @@ Lexers for JavaScript and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -453,6 +453,10 @@ class TypeScriptLexer(RegexLexer): flags = re.DOTALL | re.MULTILINE + # Higher priority than the TypoScriptLexer, as TypeScript is far more + # common these days + priority = 0.5 + tokens = { 'commentsandwhitespace': [ (r'\s+', Text), @@ -478,7 +482,7 @@ class TypeScriptLexer(RegexLexer): (r'[{(\[;,]', Punctuation, 'slashstartsregex'), (r'[})\].]', Punctuation), (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|' - r'throw|try|catch|finally|new|delete|typeof|instanceof|void|' + r'throw|try|catch|finally|new|delete|typeof|instanceof|void|of|' r'this)\b', Keyword, 'slashstartsregex'), (r'(var|let|with|function)\b', Keyword.Declaration, 'slashstartsregex'), (r'(abstract|boolean|byte|char|class|const|debugger|double|enum|export|' @@ -534,12 +538,6 @@ class TypeScriptLexer(RegexLexer): ], } - def analyse_text(text): - if re.search('^(import.+(from\s+)?["\']|' - '(export\s*)?(interface|class|function)\s+)', - text, re.MULTILINE): - return 1.0 - class LassoLexer(RegexLexer): """ @@ -1015,7 +1013,7 @@ class ObjectiveJLexer(RegexLexer): } def analyse_text(text): - if re.search('^\s*@import\s+[<"]', text, re.MULTILINE): + if re.search(r'^\s*@import\s+[<"]', text, re.MULTILINE): # special directive found in most Objective-J files return True return False @@ -1500,8 +1498,10 @@ class JuttleLexer(RegexLexer): (r'^(?=\s|/)', Text, 'slashstartsregex'), include('commentsandwhitespace'), (r':\d{2}:\d{2}:\d{2}(\.\d*)?:', String.Moment), - (r':(now|beginning|end|forever|yesterday|today|tomorrow|(\d+(\.\d*)?|\.\d+)(ms|[smhdwMy])?):', String.Moment), - (r':\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d*)?)?(Z|[+-]\d{2}:\d{2}|[+-]\d{4})?:', String.Moment), + (r':(now|beginning|end|forever|yesterday|today|tomorrow|' + r'(\d+(\.\d*)?|\.\d+)(ms|[smhdwMy])?):', String.Moment), + (r':\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d*)?)?' + r'(Z|[+-]\d{2}:\d{2}|[+-]\d{4})?:', String.Moment), (r':((\d+(\.\d*)?|\.\d+)[ ]+)?(millisecond|second|minute|hour|day|week|month|year)[s]?' r'(([ ]+and[ ]+(\d+[ ]+)?(millisecond|second|minute|hour|day|week|month|year)[s]?)' r'|[ ]+(ago|from[ ]+now))*:', String.Moment), diff --git a/pygments/lexers/julia.py b/pygments/lexers/julia.py index 67453aba..aa1609da 100644 --- a/pygments/lexers/julia.py +++ b/pygments/lexers/julia.py @@ -5,7 +5,7 @@ Lexers for the Julia language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -48,6 +48,7 @@ class JuliaLexer(RegexLexer): # keywords (r'in\b', Keyword.Pseudo), + (r'isa\b', Keyword.Pseudo), (r'(true|false)\b', Keyword.Constant), (r'(local|global|const)\b', Keyword.Declaration), (words([ @@ -55,7 +56,8 @@ class JuliaLexer(RegexLexer): 'baremodule', 'begin', 'bitstype', 'break', 'catch', 'ccall', 'continue', 'do', 'else', 'elseif', 'end', 'export', 'finally', 'for', 'if', 'import', 'importall', 'let', 'macro', 'module', - 'quote', 'return', 'try', 'using', 'while'], + 'mutable', 'primitive', 'quote', 'return', 'struct', 'try', + 'using', 'while'], suffix=r'\b'), Keyword), # NOTE @@ -146,7 +148,7 @@ class JuliaLexer(RegexLexer): (words([ # prec-assignment u'=', u':=', u'+=', u'-=', u'*=', u'/=', u'//=', u'.//=', u'.*=', u'./=', - u'\=', u'.\=', u'^=', u'.^=', u'÷=', u'.÷=', u'%=', u'.%=', u'|=', u'&=', + u'\\=', u'.\\=', u'^=', u'.^=', u'÷=', u'.÷=', u'%=', u'.%=', u'|=', u'&=', u'$=', u'=>', u'<<=', u'>>=', u'>>>=', u'~', u'.+=', u'.-=', # prec-conditional u'?', @@ -181,7 +183,7 @@ class JuliaLexer(RegexLexer): # prec-dot u'.', # unary op - u'+', u'-', u'!', u'~', u'√', u'∛', u'∜' + u'+', u'-', u'!', u'√', u'∛', u'∜' ]), Operator), # chars diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index f4392839..d01b96fc 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -5,7 +5,7 @@ Pygments lexers for JVM languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -21,7 +21,7 @@ from pygments import unistring as uni __all__ = ['JavaLexer', 'ScalaLexer', 'GosuLexer', 'GosuTemplateLexer', 'GroovyLexer', 'IokeLexer', 'ClojureLexer', 'ClojureScriptLexer', 'KotlinLexer', 'XtendLexer', 'AspectJLexer', 'CeylonLexer', - 'PigLexer', 'GoloLexer', 'JasminLexer'] + 'PigLexer', 'GoloLexer', 'JasminLexer', 'SarlLexer'] class JavaLexer(RegexLexer): @@ -257,7 +257,7 @@ class ScalaLexer(RegexLexer): u'\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\uff21-\uff3a]') idrest = u'%s(?:%s|[0-9])*(?:(?<=_)%s)?' % (letter, letter, op) - letter_letter_digit = u'%s(?:%s|\d)*' % (letter, letter) + letter_letter_digit = u'%s(?:%s|\\d)*' % (letter, letter) tokens = { 'root': [ @@ -689,7 +689,7 @@ class IokeLexer(RegexLexer): # functions (u'(generateMatchMethod|aliasMethod|\u03bb|\u028E|fnx|fn|method|' u'dmacro|dlecro|syntax|macro|dlecrox|lecrox|lecro|syntax)' - u'(?![\w!:?])', Name.Function), + u'(?![\\w!:?])', Name.Function), # Numbers (r'-?0[xX][0-9a-fA-F]+', Number.Hex), @@ -801,7 +801,7 @@ class ClojureLexer(RegexLexer): # TODO / should divide keywords/symbols into namespace/rest # but that's hard, so just pretend / is part of the name - valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+' + valid_name = r'(?!#)[\w!$%*+<=>?/.#|-]+' tokens = { 'root': [ @@ -1006,7 +1006,7 @@ class KotlinLexer(RegexLexer): .. versionadded:: 1.5 """ - + name = 'Kotlin' aliases = ['kotlin'] filenames = ['*.kt'] @@ -1017,15 +1017,22 @@ class KotlinLexer(RegexLexer): kt_name = ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' + '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc', 'Cf', 'Mn', 'Mc') + ']*') - kt_id = '(' + kt_name + '|`' + kt_name + '`)' + + kt_space_name = ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' + + '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc', 'Cf', + 'Mn', 'Mc', 'Zs') + ',-]*') + + kt_id = '(' + kt_name + '|`' + kt_space_name + '`)' tokens = { 'root': [ (r'^\s*\[.*?\]', Name.Attribute), (r'[^\S\n]+', Text), + (r'\s+', Text), (r'\\\n', Text), # line continuation (r'//.*?\n', Comment.Single), (r'/[*].*?[*]/', Comment.Multiline), + (r'""".*?"""', String), (r'\n', Text), (r'::|!!|\?[:.]', Operator), (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation), @@ -1035,11 +1042,14 @@ class KotlinLexer(RegexLexer): (r"'\\.'|'[^\\]'", String.Char), (r"[0-9](\.[0-9]*)?([eE][+-][0-9]+)?[flFL]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), - (r'(class)(\s+)(object)', bygroups(Keyword, Text, Keyword)), + (r'(object)(\s+)(:)(\s+)', bygroups(Keyword, Text, Punctuation, Text), 'class'), + (r'(companion)(\s+)(object)', bygroups(Keyword, Text, Keyword)), (r'(class|interface|object)(\s+)', bygroups(Keyword, Text), 'class'), (r'(package|import)(\s+)', bygroups(Keyword, Text), 'package'), + (r'(val|var)(\s+)([(])', bygroups(Keyword, Text, Punctuation), 'property_dec'), (r'(val|var)(\s+)', bygroups(Keyword, Text), 'property'), (r'(fun)(\s+)', bygroups(Keyword, Text), 'function'), + (r'(inline fun)(\s+)', bygroups(Keyword, Text), 'function'), (r'(abstract|annotation|as|break|by|catch|class|companion|const|' r'constructor|continue|crossinline|data|do|dynamic|else|enum|' r'external|false|final|finally|for|fun|get|if|import|in|infix|' @@ -1058,9 +1068,26 @@ class KotlinLexer(RegexLexer): 'property': [ (kt_id, Name.Property, '#pop') ], + 'property_dec': [ + (r'(,)(\s*)', bygroups(Punctuation, Text)), + (r'(:)(\s*)', bygroups(Punctuation, Text)), + (r'<', Punctuation, 'generic'), + (r'([)])', Punctuation, '#pop'), + (kt_id, Name.Property) + ], 'function': [ + (r'<', Punctuation, 'generic'), + (r''+kt_id+'([.])'+kt_id, bygroups(Name.Class, Punctuation, Name.Function), '#pop'), (kt_id, Name.Function, '#pop') ], + 'generic': [ + (r'(>)(\s*)', bygroups(Punctuation, Text), '#pop'), + (r':',Punctuation), + (r'(reified|out|in)\b', Keyword), + (r',',Text), + (r'\s+',Text), + (kt_id,Name) + ] } @@ -1258,7 +1285,7 @@ class GoloLexer(RegexLexer): (r'-?\d[\d_]*L', Number.Integer.Long), (r'-?\d[\d_]*', Number.Integer), - ('`?[a-zA-Z_][\w$]*', Name), + (r'`?[a-zA-Z_][\w$]*', Name), (r'@[a-zA-Z_][\w$.]*', Name.Decorator), (r'"""', String, combined('stringescape', 'triplestring')), @@ -1571,3 +1598,57 @@ class JasminLexer(RegexLexer): re.MULTILINE): score += 0.6 return score + + +class SarlLexer(RegexLexer): + """ + For `SARL <http://www.sarl.io>`_ source code. + + .. versionadded:: 2.4 + """ + + name = 'SARL' + aliases = ['sarl'] + filenames = ['*.sarl'] + mimetypes = ['text/x-sarl'] + + flags = re.MULTILINE | re.DOTALL + + tokens = { + 'root': [ + # method names + (r'^(\s*(?:[a-zA-Z_][\w.\[\]]*\s+)+?)' # return arguments + r'([a-zA-Z_$][\w$]*)' # method name + r'(\s*)(\()', # signature start + bygroups(using(this), Name.Function, Text, Operator)), + (r'[^\S\n]+', Text), + (r'//.*?\n', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline), + (r'@[a-zA-Z_][\w.]*', Name.Decorator), + (r'(as|break|case|catch|default|do|else|extends|extension|finally|fires|for|if|implements|instanceof|new|on|requires|return|super|switch|throw|throws|try|typeof|uses|while|with)\b', + Keyword), + (r'(abstract|def|dispatch|final|native|override|private|protected|public|static|strictfp|synchronized|transient|val|var|volatile)\b', Keyword.Declaration), + (r'(boolean|byte|char|double|float|int|long|short|void)\b', + Keyword.Type), + (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)), + (r'(false|it|null|occurrence|this|true|void)\b', Keyword.Constant), + (r'(agent|annotation|artifact|behavior|capacity|class|enum|event|interface|skill|space)(\s+)', bygroups(Keyword.Declaration, Text), + 'class'), + (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'), + (r'"(\\\\|\\"|[^"])*"', String), + (r"'(\\\\|\\'|[^'])*'", String), + (r'[a-zA-Z_]\w*:', Name.Label), + (r'[a-zA-Z_$]\w*', Name), + (r'[~^*!%&\[\](){}<>\|+=:;,./?-]', Operator), + (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'[0-9]+L?', Number.Integer), + (r'\n', Text) + ], + 'class': [ + (r'[a-zA-Z_]\w*', Name.Class, '#pop') + ], + 'import': [ + (r'[\w.]+\*?', Name.Namespace, '#pop') + ], + } diff --git a/pygments/lexers/lisp.py b/pygments/lexers/lisp.py index e258c347..169d7a9e 100644 --- a/pygments/lexers/lisp.py +++ b/pygments/lexers/lisp.py @@ -5,7 +5,7 @@ Lexers for Lispy languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -19,7 +19,7 @@ from pygments.lexers.python import PythonLexer __all__ = ['SchemeLexer', 'CommonLispLexer', 'HyLexer', 'RacketLexer', 'NewLispLexer', 'EmacsLispLexer', 'ShenLexer', 'CPSALexer', - 'XtlangLexer'] + 'XtlangLexer', 'FennelLexer'] class SchemeLexer(RegexLexer): @@ -139,7 +139,7 @@ class SchemeLexer(RegexLexer): (r"(?<=#\()" + valid_name, Name.Variable), # highlight the builtins - ("(?<=\()(%s)" % '|'.join(re.escape(entry) + ' ' for entry in builtins), + (r"(?<=\()(%s)" % '|'.join(re.escape(entry) + ' ' for entry in builtins), Name.Builtin), # the remaining functions @@ -321,7 +321,7 @@ class CommonLispLexer(RegexLexer): (r'#\d+#', Operator), # read-time comment - (r'#+nil' + terminated + '\s*\(', Comment.Preproc, 'commented-form'), + (r'#+nil' + terminated + r'\s*\(', Comment.Preproc, 'commented-form'), # read-time conditional (r'#[+-]', Operator), @@ -333,7 +333,7 @@ class CommonLispLexer(RegexLexer): (r'(t|nil)' + terminated, Name.Constant), # functions and variables - (r'\*' + symbol + '\*', Name.Variable.Global), + (r'\*' + symbol + r'\*', Name.Variable.Global), (symbol, Name.Variable), # parentheses @@ -382,7 +382,7 @@ class HyLexer(RegexLexer): # valid names for identifiers # well, names can only not consist fully of numbers # but this should be good enough for now - valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+' + valid_name = r'(?!#)[\w!$%*+<=>?/.#-:]+' def _multi_escape(entries): return words(entries, suffix=' ') @@ -1249,7 +1249,7 @@ class RacketLexer(RegexLexer): _opening_parenthesis = r'[([{]' _closing_parenthesis = r'[)\]}]' _delimiters = r'()[\]{}",\'`;\s' - _symbol = r'(?u)(?:\|[^|]*\||\\[\w\W]|[^|\\%s]+)+' % _delimiters + _symbol = r'(?:\|[^|]*\||\\[\w\W]|[^|\\%s]+)+' % _delimiters _exact_decimal_prefix = r'(?:#e)?(?:#d)?(?:#e)?' _exponent = r'(?:[defls][-+]?\d+)' _inexact_simple_no_hashes = r'(?:\d+(?:/\d+|\.\d*)?|\.\d+)' @@ -1301,16 +1301,16 @@ class RacketLexer(RegexLexer): (_inexact_simple, _delimiters), Number.Float, '#pop'), # #b - (r'(?i)(#[ei])?#b%s' % _symbol, Number.Bin, '#pop'), + (r'(?iu)(#[ei])?#b%s' % _symbol, Number.Bin, '#pop'), # #o - (r'(?i)(#[ei])?#o%s' % _symbol, Number.Oct, '#pop'), + (r'(?iu)(#[ei])?#o%s' % _symbol, Number.Oct, '#pop'), # #x - (r'(?i)(#[ei])?#x%s' % _symbol, Number.Hex, '#pop'), + (r'(?iu)(#[ei])?#x%s' % _symbol, Number.Hex, '#pop'), # #i is always inexact, i.e. float - (r'(?i)(#d)?#i%s' % _symbol, Number.Float, '#pop'), + (r'(?iu)(#d)?#i%s' % _symbol, Number.Float, '#pop'), # Strings and characters (r'#?"', String.Double, ('#pop', 'string')), @@ -1323,7 +1323,7 @@ class RacketLexer(RegexLexer): (r'#(true|false|[tTfF])', Name.Constant, '#pop'), # Keyword argument names (e.g. #:keyword) - (r'#:%s' % _symbol, Keyword.Declaration, '#pop'), + (r'(?u)#:%s' % _symbol, Keyword.Declaration, '#pop'), # Reader extensions (r'(#lang |#!)(\S+)', @@ -1400,7 +1400,7 @@ class RacketLexer(RegexLexer): class NewLispLexer(RegexLexer): """ - For `newLISP. <www.newlisp.org>`_ source code (version 10.3.0). + For `newLISP. <http://www.newlisp.org/>`_ source code (version 10.3.0). .. versionadded:: 1.5 """ @@ -2154,7 +2154,7 @@ class EmacsLispLexer(RegexLexer): (r'(t|nil)' + terminated, Name.Constant), # functions and variables - (r'\*' + symbol + '\*', Name.Variable.Global), + (r'\*' + symbol + r'\*', Name.Variable.Global), (symbol, Name.Variable), # parentheses @@ -2327,13 +2327,13 @@ class ShenLexer(RegexLexer): token = Name.Function if token == Literal else token yield index, token, value - raise StopIteration + return def _process_signature(self, tokens): for index, token, value in tokens: if token == Literal and value == '}': yield index, Punctuation, value - raise StopIteration + return elif token in (Literal, Name.Function): token = Name.Variable if value.istitle() else Keyword.Type yield index, token, value @@ -2619,3 +2619,75 @@ class XtlangLexer(RegexLexer): include('scheme') ], } + + +class FennelLexer(RegexLexer): + """A lexer for the `Fennel programming language <https://fennel-lang.org>`_. + + Fennel compiles to Lua, so all the Lua builtins are recognized as well + as the special forms that are particular to the Fennel compiler. + + .. versionadded:: 2.3 + """ + name = 'Fennel' + aliases = ['fennel', 'fnl'] + filenames = ['*.fnl'] + + # these two lists are taken from fennel-mode.el: + # https://gitlab.com/technomancy/fennel-mode + # this list is current as of Fennel version 0.1.0. + special_forms = ( + u'require-macros', u'eval-compiler', + u'do', u'values', u'if', u'when', u'each', u'for', u'fn', u'lambda', + u'λ', u'set', u'global', u'var', u'local', u'let', u'tset', u'doto', + u'set-forcibly!', u'defn', u'partial', u'while', u'or', u'and', u'true', + u'false', u'nil', u'.', u'+', u'..', u'^', u'-', u'*', u'%', u'/', u'>', + u'<', u'>=', u'<=', u'=', u'~=', u'#', u'...', u':', u'->', u'->>', + ) + + # Might be nicer to use the list from _lua_builtins.py but it's unclear how? + builtins = ( + u'_G', u'_VERSION', u'arg', u'assert', u'bit32', u'collectgarbage', + u'coroutine', u'debug', u'dofile', u'error', u'getfenv', + u'getmetatable', u'io', u'ipairs', u'load', u'loadfile', u'loadstring', + u'math', u'next', u'os', u'package', u'pairs', u'pcall', u'print', + u'rawequal', u'rawget', u'rawlen', u'rawset', u'require', u'select', + u'setfenv', u'setmetatable', u'string', u'table', u'tonumber', + u'tostring', u'type', u'unpack', u'xpcall' + ) + + # based on the scheme definition, but disallowing leading digits and commas + valid_name = r'[a-zA-Z_!$%&*+/:<=>?@^~|-][\w!$%&*+/:<=>?@^~|\.-]*' + + tokens = { + 'root': [ + # the only comment form is a semicolon; goes to the end of the line + (r';.*$', Comment.Single), + + (r'[,\s]+', Text), + (r'-?\d+\.\d+', Number.Float), + (r'-?\d+', Number.Integer), + + (r'"(\\\\|\\"|[^"])*"', String), + (r"'(\\\\|\\'|[^'])*'", String), + + # these are technically strings, but it's worth visually + # distinguishing them because their intent is different + # from regular strings. + (r':' + valid_name, String.Symbol), + + # special forms are keywords + (words(special_forms, suffix=' '), Keyword), + # lua standard library are builtins + (words(builtins, suffix=' '), Name.Builtin), + # special-case the vararg symbol + (r'\.\.\.', Name.Variable), + # regular identifiers + (valid_name, Name.Variable), + + # all your normal paired delimiters for your programming enjoyment + (r'(\(|\))', Punctuation), + (r'(\[|\])', Punctuation), + (r'(\{|\})', Punctuation), + ] + } diff --git a/pygments/lexers/make.py b/pygments/lexers/make.py index b222b672..f67f1095 100644 --- a/pygments/lexers/make.py +++ b/pygments/lexers/make.py @@ -5,7 +5,7 @@ Lexers for Makefiles and similar. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -102,8 +102,8 @@ class BaseMakefileLexer(RegexLexer): (r'\$\(', Keyword, 'expansion'), ], 'expansion': [ - (r'[^$a-zA-Z_()]+', Text), - (r'[a-zA-Z_]+', Name.Variable), + (r'[^\w$().-]+', Text), + (r'[\w.-]+', Name.Variable), (r'\$', Keyword), (r'\(', Keyword, '#push'), (r'\)', Keyword, '#pop'), diff --git a/pygments/lexers/markup.py b/pygments/lexers/markup.py index 92dc9e7a..2eb669e4 100644 --- a/pygments/lexers/markup.py +++ b/pygments/lexers/markup.py @@ -5,7 +5,7 @@ Lexers for non-HTML markup languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -536,10 +536,9 @@ class MarkdownLexer(RegexLexer): # no lexer for this language. handle it like it was a code block if lexer is None: yield match.start(4), String, code - return - - for item in do_insertions([], lexer.get_tokens_unprocessed(code)): - yield item + else: + for item in do_insertions([], lexer.get_tokens_unprocessed(code)): + yield item yield match.start(5), String , match.group(5) @@ -583,6 +582,11 @@ class MarkdownLexer(RegexLexer): (r'[@#][\w/:]+', Name.Entity), # (image?) links eg:  (r'(!?\[)([^]]+)(\])(\()([^)]+)(\))', bygroups(Text, Name.Tag, Text, Text, Name.Attribute, Text)), + # reference-style links, e.g.: + # [an example][id] + # [id]: http://example.com/ + (r'(\[)([^]]+)(\])(\[)([^]]*)(\])', bygroups(Text, Name.Tag, Text, Text, Name.Label, Text)), + (r'^(\s*\[)([^]]*)(\]:\s*)(.+)', bygroups(Text, Name.Label, Text, Name.Attribute)), # general text, must come last! (r'[^\\\s]+', Text), diff --git a/pygments/lexers/math.py b/pygments/lexers/math.py index ea0ebee2..73115082 100644 --- a/pygments/lexers/math.py +++ b/pygments/lexers/math.py @@ -5,7 +5,7 @@ Just export lexers that were contained in this module. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/matlab.py b/pygments/lexers/matlab.py index 56a0f6d6..fd3897ad 100644 --- a/pygments/lexers/matlab.py +++ b/pygments/lexers/matlab.py @@ -5,7 +5,7 @@ Lexers for Matlab and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -134,9 +134,9 @@ class MatlabLexer(RegexLexer): } def analyse_text(text): - if re.match('^\s*%', text, re.M): # comment + if re.match(r'^\s*%', text, re.M): # comment return 0.2 - elif re.match('^!\w+', text, re.M): # system cmd + elif re.match(r'^!\w+', text, re.M): # system cmd return 0.2 diff --git a/pygments/lexers/ml.py b/pygments/lexers/ml.py index f80d5bfa..aff8160d 100644 --- a/pygments/lexers/ml.py +++ b/pygments/lexers/ml.py @@ -5,7 +5,7 @@ Lexers for ML family languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -43,7 +43,7 @@ class SMLLexer(RegexLexer): symbolicid_reserved = set(( # Core - ':', '\|', '=', '=>', '->', '#', + ':', r'\|', '=', '=>', '->', '#', # Modules ':>', )) diff --git a/pygments/lexers/modeling.py b/pygments/lexers/modeling.py index b354f1cf..c03907ba 100644 --- a/pygments/lexers/modeling.py +++ b/pygments/lexers/modeling.py @@ -5,7 +5,7 @@ Lexers for modeling languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -13,7 +13,7 @@ import re from pygments.lexer import RegexLexer, include, bygroups, using, default from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ - Number, Punctuation + Number, Punctuation, Whitespace from pygments.lexers.html import HtmlLexer from pygments.lexers import _stan_builtins @@ -284,8 +284,8 @@ class StanLexer(RegexLexer): """Pygments Lexer for Stan models. The Stan modeling language is specified in the *Stan Modeling Language - User's Guide and Reference Manual, v2.8.0*, - `pdf <https://github.com/stan-dev/stan/releases/download/v2.8.8/stan-reference-2.8.0.pdf>`__. + User's Guide and Reference Manual, v2.17.0*, + `pdf <https://github.com/stan-dev/stan/releases/download/v2.17.0/stan-reference-2.17.0.pdf>`__. .. versionadded:: 1.6 """ @@ -316,19 +316,26 @@ class StanLexer(RegexLexer): 'parameters', r'transformed\s+parameters', 'model', r'generated\s+quantities')), bygroups(Keyword.Namespace, Text, Punctuation)), + # target keyword + (r'target\s*\+=', Keyword), # Reserved Words (r'(%s)\b' % r'|'.join(_stan_builtins.KEYWORDS), Keyword), # Truncation (r'T(?=\s*\[)', Keyword), # Data types (r'(%s)\b' % r'|'.join(_stan_builtins.TYPES), Keyword.Type), + # < should be punctuation, but elsewhere I can't tell if it is in + # a range constraint + (r'(<)(\s*)(upper|lower)(\s*)(=)', + bygroups(Operator, Whitespace, Keyword, Whitespace, Punctuation)), + (r'(,)(\s*)(upper)(\s*)(=)', + bygroups(Punctuation, Whitespace, Keyword, Whitespace, Punctuation)), # Punctuation - (r"[;:,\[\]()]", Punctuation), + (r"[;,\[\]()]", Punctuation), # Builtin - (r'(%s)(?=\s*\()' - % r'|'.join(_stan_builtins.FUNCTIONS - + _stan_builtins.DISTRIBUTIONS), - Name.Builtin), + (r'(%s)(?=\s*\()' % '|'.join(_stan_builtins.FUNCTIONS), Name.Builtin), + (r'(~)(\s*)(%s)(?=\s*\()' % '|'.join(_stan_builtins.DISTRIBUTIONS), + bygroups(Operator, Whitespace, Name.Builtin)), # Special names ending in __, like lp__ (r'[A-Za-z]\w*__\b', Name.Builtin.Pseudo), (r'(%s)\b' % r'|'.join(_stan_builtins.RESERVED), Keyword.Reserved), @@ -337,17 +344,18 @@ class StanLexer(RegexLexer): # Regular variable names (r'[A-Za-z]\w*\b', Name), # Real Literals - (r'-?[0-9]+(\.[0-9]+)?[eE]-?[0-9]+', Number.Float), - (r'-?[0-9]*\.[0-9]*', Number.Float), + (r'[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?', Number.Float), + (r'\.[0-9]+([eE][+-]?[0-9]+)?', Number.Float), # Integer Literals - (r'-?[0-9]+', Number.Integer), + (r'[0-9]+', Number.Integer), # Assignment operators - # SLexer makes these tokens Operators. - (r'<-|~', Operator), + (r'<-|(?:\+|-|\.?/|\.?\*|=)?=|~', Operator), # Infix, prefix and postfix operators (and = ) - (r"\+|-|\.?\*|\.?/|\\|'|\^|==?|!=?|<=?|>=?|\|\||&&", Operator), + (r"\+|-|\.?\*|\.?/|\\|'|\^|!=?|<=?|>=?|\|\||&&|%|\?|:", Operator), # Block delimiters (r'[{}]', Punctuation), + # Distribution | + (r'\|', Punctuation) ] } diff --git a/pygments/lexers/modula2.py b/pygments/lexers/modula2.py index c0a69b40..4fd84dab 100644 --- a/pygments/lexers/modula2.py +++ b/pygments/lexers/modula2.py @@ -5,7 +5,7 @@ Multi-Dialect Lexer for Modula-2. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/monte.py b/pygments/lexers/monte.py index ed6e20f8..e181c940 100644 --- a/pygments/lexers/monte.py +++ b/pygments/lexers/monte.py @@ -5,7 +5,7 @@ Lexer for the Monte programming language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/ncl.py b/pygments/lexers/ncl.py index 3ca5135c..e2edd6cc 100644 --- a/pygments/lexers/ncl.py +++ b/pygments/lexers/ncl.py @@ -5,7 +5,7 @@ Lexers for NCAR Command Language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/nimrod.py b/pygments/lexers/nimrod.py index d438c1bf..4c1bccf0 100644 --- a/pygments/lexers/nimrod.py +++ b/pygments/lexers/nimrod.py @@ -5,7 +5,7 @@ Lexer for the Nim language (formerly known as Nimrod). - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/nit.py b/pygments/lexers/nit.py index 21116499..42167ece 100644 --- a/pygments/lexers/nit.py +++ b/pygments/lexers/nit.py @@ -5,7 +5,7 @@ Lexer for the Nit language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/nix.py b/pygments/lexers/nix.py index e148c919..50210c48 100644 --- a/pygments/lexers/nix.py +++ b/pygments/lexers/nix.py @@ -5,7 +5,7 @@ Lexers for the NixOS Nix language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/oberon.py b/pygments/lexers/oberon.py index 3b5fb3e4..1c18488a 100644 --- a/pygments/lexers/oberon.py +++ b/pygments/lexers/oberon.py @@ -5,7 +5,7 @@ Lexers for Oberon family languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/objective.py b/pygments/lexers/objective.py index 7807255e..777d8d4d 100644 --- a/pygments/lexers/objective.py +++ b/pygments/lexers/objective.py @@ -5,7 +5,7 @@ Lexers for Objective-C family languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -66,7 +66,7 @@ def objective(baselexer): 'copy', 'retain', 'assign', 'unsafe_unretained', 'atomic', 'nonatomic', 'readonly', 'readwrite', 'setter', 'getter', 'typeof', 'in', 'out', 'inout', 'release', 'class', '@dynamic', '@optional', - '@required', '@autoreleasepool'), suffix=r'\b'), + '@required', '@autoreleasepool', '@import'), suffix=r'\b'), Keyword), (words(('id', 'instancetype', 'Class', 'IMP', 'SEL', 'BOOL', 'IBOutlet', 'IBAction', 'unichar'), suffix=r'\b'), @@ -87,26 +87,26 @@ def objective(baselexer): ], 'oc_classname': [ # interface definition that inherits - ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)(\{)', + (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)(\{)', bygroups(Name.Class, Text, Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', + (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', bygroups(Name.Class, Text, Name.Class), '#pop'), # interface definition for a category - ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)(\{)', + (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)(\{)', bygroups(Name.Class, Text, Name.Label, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))', + (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))', bygroups(Name.Class, Text, Name.Label), '#pop'), # simple interface / implementation - ('([a-zA-Z$_][\w$]*)(\s*)(\{)', + (r'([a-zA-Z$_][\w$]*)(\s*)(\{)', bygroups(Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop') + (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop') ], 'oc_forward_classname': [ - ('([a-zA-Z$_][\w$]*)(\s*,\s*)', + (r'([a-zA-Z$_][\w$]*)(\s*,\s*)', bygroups(Name.Class, Text), 'oc_forward_classname'), - ('([a-zA-Z$_][\w$]*)(\s*;?)', + (r'([a-zA-Z$_][\w$]*)(\s*;?)', bygroups(Name.Class, Text), '#pop') ], 'oc_ivars': [ @@ -244,17 +244,17 @@ class LogosLexer(ObjectiveCppLexer): inherit, ], 'logos_init_directive': [ - ('\s+', Text), + (r'\s+', Text), (',', Punctuation, ('logos_init_directive', '#pop')), - ('([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)', + (r'([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)', bygroups(Name.Class, Text, Punctuation, Text, Text)), - ('([a-zA-Z$_][\w$]*)', Name.Class), - ('\)', Punctuation, '#pop'), + (r'([a-zA-Z$_][\w$]*)', Name.Class), + (r'\)', Punctuation, '#pop'), ], 'logos_classname': [ - ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', + (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', bygroups(Name.Class, Text, Name.Class), '#pop'), - ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop') + (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop') ], 'root': [ (r'(%subclass)(\s+)', bygroups(Keyword, Text), diff --git a/pygments/lexers/ooc.py b/pygments/lexers/ooc.py index 957b72f1..438719cd 100644 --- a/pygments/lexers/ooc.py +++ b/pygments/lexers/ooc.py @@ -5,7 +5,7 @@ Lexers for the Ooc language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py index bfce4c3c..c3a60cef 100644 --- a/pygments/lexers/other.py +++ b/pygments/lexers/other.py @@ -5,7 +5,7 @@ Just export lexer classes previously contained in this module. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/parasail.py b/pygments/lexers/parasail.py index 53088023..7f8cf073 100644 --- a/pygments/lexers/parasail.py +++ b/pygments/lexers/parasail.py @@ -5,7 +5,7 @@ Lexer for ParaSail. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/parsers.py b/pygments/lexers/parsers.py index 1f3c9b4d..8bcbfc50 100644 --- a/pygments/lexers/parsers.py +++ b/pygments/lexers/parsers.py @@ -5,7 +5,7 @@ Lexers for parser generators. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -364,13 +364,13 @@ class AntlrLexer(RegexLexer): # tokensSpec (r'tokens\b', Keyword, 'tokens'), # attrScope - (r'(scope)(\s*)(' + _id + ')(\s*)(\{)', + (r'(scope)(\s*)(' + _id + r')(\s*)(\{)', bygroups(Keyword, Whitespace, Name.Variable, Whitespace, Punctuation), 'action'), # exception (r'(catch|finally)\b', Keyword, 'exception'), # action - (r'(@' + _id + ')(\s*)(::)?(\s*)(' + _id + ')(\s*)(\{)', + (r'(@' + _id + r')(\s*)(::)?(\s*)(' + _id + r')(\s*)(\{)', bygroups(Name.Label, Whitespace, Punctuation, Whitespace, Name.Label, Whitespace, Punctuation), 'action'), # rule @@ -405,10 +405,10 @@ class AntlrLexer(RegexLexer): # L173 ANTLRv3.g from ANTLR book (r'(scope)(\s+)(\{)', bygroups(Keyword, Whitespace, Punctuation), 'action'), - (r'(scope)(\s+)(' + _id + ')(\s*)(;)', + (r'(scope)(\s+)(' + _id + r')(\s*)(;)', bygroups(Keyword, Whitespace, Name.Label, Whitespace, Punctuation)), # ruleAction - (r'(@' + _id + ')(\s*)(\{)', + (r'(@' + _id + r')(\s*)(\{)', bygroups(Name.Label, Whitespace, Punctuation), 'action'), # finished prelims, go to rule alts! (r':', Punctuation, '#pop') @@ -442,7 +442,7 @@ class AntlrLexer(RegexLexer): include('comments'), (r'\{', Punctuation), (r'(' + _TOKEN_REF + r')(\s*)(=)?(\s*)(' + _STRING_LITERAL - + ')?(\s*)(;)', + + r')?(\s*)(;)', bygroups(Name.Label, Whitespace, Punctuation, Whitespace, String, Whitespace, Punctuation)), (r'\}', Punctuation, '#pop'), @@ -452,7 +452,7 @@ class AntlrLexer(RegexLexer): include('comments'), (r'\{', Punctuation), (r'(' + _id + r')(\s*)(=)(\s*)(' + - '|'.join((_id, _STRING_LITERAL, _INT, '\*')) + ')(\s*)(;)', + '|'.join((_id, _STRING_LITERAL, _INT, r'\*')) + r')(\s*)(;)', bygroups(Name.Variable, Whitespace, Punctuation, Whitespace, Text, Whitespace, Punctuation)), (r'\}', Punctuation, '#pop'), diff --git a/pygments/lexers/pascal.py b/pygments/lexers/pascal.py index 9aa1ac8f..251bca2a 100644 --- a/pygments/lexers/pascal.py +++ b/pygments/lexers/pascal.py @@ -5,7 +5,7 @@ Lexers for Pascal family languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -593,8 +593,8 @@ class AdaLexer(RegexLexer): ], 'end': [ ('(if|case|record|loop|select)', Keyword.Reserved), - ('"[^"]+"|[\w.]+', Name.Function), - ('\s+', Text), + (r'"[^"]+"|[\w.]+', Name.Function), + (r'\s+', Text), (';', Punctuation, '#pop'), ], 'type_def': [ @@ -628,11 +628,11 @@ class AdaLexer(RegexLexer): ], 'package': [ ('body', Keyword.Declaration), - ('is\s+new|renames', Keyword.Reserved), + (r'is\s+new|renames', Keyword.Reserved), ('is', Keyword.Reserved, '#pop'), (';', Punctuation, '#pop'), - ('\(', Punctuation, 'package_instantiation'), - ('([\w.]+)', Name.Class), + (r'\(', Punctuation, 'package_instantiation'), + (r'([\w.]+)', Name.Class), include('root'), ], 'package_instantiation': [ diff --git a/pygments/lexers/pawn.py b/pygments/lexers/pawn.py index f462a883..576df424 100644 --- a/pygments/lexers/pawn.py +++ b/pygments/lexers/pawn.py @@ -5,7 +5,7 @@ Lexers for the Pawn languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -36,7 +36,7 @@ class SourcePawnLexer(RegexLexer): tokens = { 'root': [ # preprocessor directives: without whitespace - ('^#if\s+0', Comment.Preproc, 'if0'), + (r'^#if\s+0', Comment.Preproc, 'if0'), ('^#', Comment.Preproc, 'macro'), # or with whitespace ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'), @@ -62,7 +62,7 @@ class SourcePawnLexer(RegexLexer): r'public|return|sizeof|static|decl|struct|switch)\b', Keyword), (r'(bool|Float)\b', Keyword.Type), (r'(true|false)\b', Keyword.Constant), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'string': [ (r'"', String, '#pop'), @@ -148,7 +148,7 @@ class PawnLexer(RegexLexer): tokens = { 'root': [ # preprocessor directives: without whitespace - ('^#if\s+0', Comment.Preproc, 'if0'), + (r'^#if\s+0', Comment.Preproc, 'if0'), ('^#', Comment.Preproc, 'macro'), # or with whitespace ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'), @@ -174,7 +174,7 @@ class PawnLexer(RegexLexer): r'public|return|sizeof|tagof|state|goto)\b', Keyword), (r'(bool|Float)\b', Keyword.Type), (r'(true|false)\b', Keyword.Constant), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'string': [ (r'"', String, '#pop'), diff --git a/pygments/lexers/perl.py b/pygments/lexers/perl.py index db5a9361..8f30c186 100644 --- a/pygments/lexers/perl.py +++ b/pygments/lexers/perl.py @@ -5,7 +5,7 @@ Lexers for Perl and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -208,7 +208,7 @@ class PerlLexer(RegexLexer): def analyse_text(text): if shebang_matches(text, r'perl'): return True - if re.search('(?:my|our)\s+[$@%(]', text): + if re.search(r'(?:my|our)\s+[$@%(]', text): return 0.9 @@ -226,7 +226,7 @@ class Perl6Lexer(ExtendedRegexLexer): mimetypes = ['text/x-perl6', 'application/x-perl6'] flags = re.MULTILINE | re.DOTALL | re.UNICODE - PERL6_IDENTIFIER_RANGE = "['\w:-]" + PERL6_IDENTIFIER_RANGE = r"['\w:-]" PERL6_KEYWORDS = ( 'BEGIN', 'CATCH', 'CHECK', 'CONTROL', 'END', 'ENTER', 'FIRST', 'INIT', @@ -495,7 +495,7 @@ class Perl6Lexer(ExtendedRegexLexer): (r'^=.*?\n\s*?\n', Comment.Multiline), (r'(regex|token|rule)(\s*' + PERL6_IDENTIFIER_RANGE + '+:sym)', bygroups(Keyword, Name), 'token-sym-brackets'), - (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + ')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?', + (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + r')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?', bygroups(Keyword, Name), 'pre-token'), # deal with a special case in the Perl 6 grammar (role q { ... }) (r'(role)(\s+)(q)(\s*)', bygroups(Keyword, Text, Name, Text)), @@ -591,21 +591,21 @@ class Perl6Lexer(ExtendedRegexLexer): rating = False # check for my/our/has declarations - if re.search("(?:my|our|has)\s+(?:" + Perl6Lexer.PERL6_IDENTIFIER_RANGE + - "+\s+)?[$@%&(]", text): + if re.search(r"(?:my|our|has)\s+(?:" + Perl6Lexer.PERL6_IDENTIFIER_RANGE + + r"+\s+)?[$@%&(]", text): rating = 0.8 saw_perl_decl = True for line in lines: line = re.sub('#.*', '', line) - if re.match('^\s*$', line): + if re.match(r'^\s*$', line): continue # match v6; use v6; use v6.0; use v6.0.0; - if re.match('^\s*(?:use\s+)?v6(?:\.\d(?:\.\d)?)?;', line): + if re.match(r'^\s*(?:use\s+)?v6(?:\.\d(?:\.\d)?)?;', line): return True # match class, module, role, enum, grammar declarations - class_decl = re.match('^\s*(?:(?P<scope>my|our)\s+)?(?:module|class|role|enum|grammar)', line) + class_decl = re.match(r'^\s*(?:(?P<scope>my|our)\s+)?(?:module|class|role|enum|grammar)', line) if class_decl: if saw_perl_decl or class_decl.group('scope') is not None: return True diff --git a/pygments/lexers/php.py b/pygments/lexers/php.py index f618b5fd..bd4a2376 100644 --- a/pygments/lexers/php.py +++ b/pygments/lexers/php.py @@ -5,7 +5,7 @@ Lexers for PHP and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -15,7 +15,8 @@ from pygments.lexer import RegexLexer, include, bygroups, default, using, \ this, words from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Other -from pygments.util import get_bool_opt, get_list_opt, iteritems +from pygments.util import get_bool_opt, get_list_opt, iteritems, \ + shebang_matches __all__ = ['ZephirLexer', 'PhpLexer'] @@ -173,7 +174,7 @@ class PhpLexer(RegexLexer): r'finally)\b', Keyword), (r'(true|false|null)\b', Keyword.Constant), include('magicconstants'), - (r'\$\{\$+' + _ident_inner + '\}', Name.Variable), + (r'\$\{\$+' + _ident_inner + r'\}', Name.Variable), (r'\$+' + _ident_inner, Name.Variable), (_ident_inner, Name.Other), (r'(\d+\.\d*|\d*\.\d+)(e[+-]?[0-9]+)?', Number.Float), @@ -214,7 +215,7 @@ class PhpLexer(RegexLexer): (r'"', String.Double, '#pop'), (r'[^{$"\\]+', String.Double), (r'\\([nrt"$\\]|[0-7]{1,3}|x[0-9a-f]{1,2})', String.Escape), - (r'\$' + _ident_inner + '(\[\S+?\]|->' + _ident_inner + ')?', + (r'\$' + _ident_inner + r'(\[\S+?\]|->' + _ident_inner + ')?', String.Interpol), (r'(\{\$\{)(.*?)(\}\})', bygroups(String.Interpol, using(this, _startinline=True), @@ -261,6 +262,8 @@ class PhpLexer(RegexLexer): yield index, token, value def analyse_text(text): + if shebang_matches(text, r'php'): + return True rv = 0.0 if re.search(r'<\?(?!xml)', text): rv += 0.3 diff --git a/pygments/lexers/pony.py b/pygments/lexers/pony.py new file mode 100644 index 00000000..8f5d4281 --- /dev/null +++ b/pygments/lexers/pony.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.pony + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for Pony and related languages. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['PonyLexer'] + + +class PonyLexer(RegexLexer): + """ + For Pony source code. + + .. versionadded:: 2.4 + """ + + name = 'Pony' + aliases = ['pony'] + filenames = ['*.pony'] + + _caps = r'(iso|trn|ref|val|box|tag)' + + tokens = { + 'root': [ + (r'\n', Text), + (r'[^\S\n]+', Text), + (r'//.*\n', Comment.Single), + (r'/\*', Comment.Multiline, 'nested_comment'), + (r'"""(?:.|\n)*?"""', String.Doc), + (r'"', String, 'string'), + (r'\'.*\'', String.Char), + (r'=>|[]{}:().~;,|&!^?[]', Punctuation), + (words(( + 'addressof', 'and', 'as', 'consume', 'digestof', 'is', 'isnt', + 'not', 'or'), + suffix=r'\b'), + Operator.Word), + (r'!=|==|<<|>>|[-+/*%=<>]', Operator), + (words(( + 'box', 'break', 'compile_error', 'compile_intrinsic', + 'continue', 'do', 'else', 'elseif', 'embed', 'end', 'error', + 'for', 'if', 'ifdef', 'in', 'iso', 'lambda', 'let', 'match', + 'object', 'recover', 'ref', 'repeat', 'return', 'tag', 'then', + 'this', 'trn', 'try', 'until', 'use', 'var', 'val', 'where', + 'while', 'with', '#any', '#read', '#send', '#share'), + suffix=r'\b'), + Keyword), + (r'(actor|class|struct|primitive|interface|trait|type)((?:\s)+)', + bygroups(Keyword, Text), 'typename'), + (r'(new|fun|be)((?:\s)+)', bygroups(Keyword, Text), 'methodname'), + (words(( + 'I8', 'U8', 'I16', 'U16', 'I32', 'U32', 'I64', 'U64', 'I128', + 'U128', 'ILong', 'ULong', 'ISize', 'USize', 'F32', 'F64', + 'Bool', 'Pointer', 'None', 'Any', 'Array', 'String', + 'Iterator'), + suffix=r'\b'), + Name.Builtin.Type), + (r'_?[A-Z]\w*', Name.Type), + (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'\d+', Number.Integer), + (r'(true|false)\b', Name.Builtin), + (r'_\d*', Name), + (r'_?[a-z][\w\'_]*', Name) + ], + 'typename': [ + (_caps + r'?((?:\s)*)(_?[A-Z]\w*)', + bygroups(Keyword, Text, Name.Class), '#pop') + ], + 'methodname': [ + (_caps + r'?((?:\s)*)(_?[a-z]\w*)', + bygroups(Keyword, Text, Name.Function), '#pop') + ], + 'nested_comment': [ + (r'[^*/]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline) + ], + 'string': [ + (r'"', String, '#pop'), + (r'\\"', String), + (r'[^\\"]+', String) + ] + } diff --git a/pygments/lexers/praat.py b/pygments/lexers/praat.py index 1a38a9e8..fa91880f 100644 --- a/pygments/lexers/praat.py +++ b/pygments/lexers/praat.py @@ -5,7 +5,7 @@ Lexer for Praat - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/prolog.py b/pygments/lexers/prolog.py index 90f9529c..8dbbc6d4 100644 --- a/pygments/lexers/prolog.py +++ b/pygments/lexers/prolog.py @@ -5,7 +5,7 @@ Lexers for Prolog and Prolog-like languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -31,7 +31,6 @@ class PrologLexer(RegexLexer): tokens = { 'root': [ - (r'^#.*', Comment.Single), (r'/\*', Comment.Multiline, 'nested-comment'), (r'%.*', Comment.Single), # character literal @@ -57,15 +56,15 @@ class PrologLexer(RegexLexer): (r'_', Keyword), # The don't-care variable (r'([a-z]+)(:)', bygroups(Name.Namespace, Punctuation)), (u'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' + u'[\\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' u'(\\s*)(:-|-->)', bygroups(Name.Function, Text, Operator)), # function defn (u'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' + u'[\\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' u'(\\s*)(\\()', bygroups(Name.Function, Text, Punctuation)), (u'[a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*', + u'[\\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*', String.Atom), # atom, characters # This one includes ! (u'[#&*+\\-./:<=>?@\\\\^~\u00a1-\u00bf\u2010-\u303f]+', @@ -300,7 +299,7 @@ class LogtalkLexer(RegexLexer): return 1.0 elif ':- category(' in text: return 1.0 - elif re.search('^:-\s[a-z]', text, re.M): + elif re.search(r'^:-\s[a-z]', text, re.M): return 0.9 else: return 0.0 diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py index 390eafe8..0f9c4d41 100644 --- a/pygments/lexers/python.py +++ b/pygments/lexers/python.py @@ -5,7 +5,7 @@ Lexers for Python and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -124,10 +124,10 @@ class PythonLexer(RegexLexer): 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError', - 'MemoryError', 'NameError', 'NotImplemented', 'NotImplementedError', + 'MemoryError', 'ModuleNotFoundError', 'NameError', 'NotImplemented', 'NotImplementedError', 'OSError', 'OverflowError', 'OverflowWarning', 'PendingDeprecationWarning', - 'ReferenceError', 'RuntimeError', 'RuntimeWarning', 'StandardError', - 'StopIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', + 'RecursionError', 'ReferenceError', 'RuntimeError', 'RuntimeWarning', 'StandardError', + 'StopIteration', 'StopAsyncIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning', @@ -180,15 +180,15 @@ class PythonLexer(RegexLexer): ], 'name': [ (r'@[\w.]+', Name.Decorator), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'funcname': [ include('magicfuncs'), - ('[a-zA-Z_]\w*', Name.Function, '#pop'), + (r'[a-zA-Z_]\w*', Name.Function, '#pop'), default('#pop'), ], 'classname': [ - ('[a-zA-Z_]\w*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ (r'(?:[ \t]|\\\n)+', Text), @@ -262,13 +262,13 @@ class Python3Lexer(RegexLexer): return [ # the old style '%s' % (...) string formatting (still valid in Py3) (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' - '[hlL]?[E-GXc-giorsux%]', String.Interpol), + '[hlL]?[E-GXc-giorsaux%]', String.Interpol), # the new style '{}'.format(...) string formatting (r'\{' - '((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name - '(\![sra])?' # conversion - '(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?' - '\}', String.Interpol), + r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name + r'(\![sra])?' # conversion + r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?' + r'\}', String.Interpol), # backslashes, quotes and formatting signs must be parsed one at a time (r'[^\\\'"%{\n]+', ttype), @@ -361,12 +361,13 @@ class Python3Lexer(RegexLexer): Name.Variable.Magic), ] tokens['numbers'] = [ - (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), - (r'\d+[eE][+-]?[0-9]+j?', Number.Float), - (r'0[oO][0-7]+', Number.Oct), - (r'0[bB][01]+', Number.Bin), - (r'0[xX][a-fA-F0-9]+', Number.Hex), - (r'\d+', Number.Integer) + (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)' + r'([eE][+-]?\d(?:_?\d)*)?', Number.Float), + (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float), + (r'0[oO](?:_?[0-7])+', Number.Oct), + (r'0[bB](?:_?[01])+', Number.Bin), + (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex), + (r'\d(?:_?\d)*', Number.Integer) ] tokens['backtick'] = [] tokens['name'] = [ @@ -396,6 +397,7 @@ class Python3Lexer(RegexLexer): tokens['strings-single'] = innerstring_rules(String.Single) tokens['strings-double'] = innerstring_rules(String.Double) + def analyse_text(text): return shebang_matches(text, r'pythonw?3(\.\d)?') @@ -671,10 +673,10 @@ class CythonLexer(RegexLexer): ], 'name': [ (r'@\w+', Name.Decorator), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'funcname': [ - ('[a-zA-Z_]\w*', Name.Function, '#pop') + (r'[a-zA-Z_]\w*', Name.Function, '#pop') ], 'cdef': [ (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved), @@ -691,7 +693,7 @@ class CythonLexer(RegexLexer): (r'.', Text), ], 'classname': [ - ('[a-zA-Z_]\w*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)), diff --git a/pygments/lexers/qvt.py b/pygments/lexers/qvt.py index f496d600..b1c1495c 100644 --- a/pygments/lexers/qvt.py +++ b/pygments/lexers/qvt.py @@ -5,7 +5,7 @@ Lexer for QVT Operational language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -18,7 +18,7 @@ __all__ = ['QVToLexer'] class QVToLexer(RegexLexer): - """ + u""" For the `QVT Operational Mapping language <http://www.omg.org/spec/QVT/1.1/>`_. Reference for implementing this: «Meta Object Facility (MOF) 2.0 @@ -126,7 +126,7 @@ class QVToLexer(RegexLexer): (r'[^\\\'"\n]+', String), # quotes, percents and backslashes must be parsed one at a time (r'[\'"\\]', String), - ], + ], 'stringescape': [ (r'\\([\\btnfr"\']|u[0-3][0-7]{2}|u[0-7]{1,2})', String.Escape) ], @@ -134,15 +134,15 @@ class QVToLexer(RegexLexer): (r'"', String, '#pop'), (r'\\\\|\\"', String.Escape), include('strings') - ], + ], 'sqs': [ # single-quoted string (r"'", String, '#pop'), (r"\\\\|\\'", String.Escape), include('strings') - ], + ], 'name': [ - ('[a-zA-Z_]\w*', Name), - ], + (r'[a-zA-Z_]\w*', Name), + ], # numbers: excerpt taken from the python lexer 'numbers': [ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), diff --git a/pygments/lexers/r.py b/pygments/lexers/r.py index dce61969..8e744599 100644 --- a/pygments/lexers/r.py +++ b/pygments/lexers/r.py @@ -5,13 +5,13 @@ Lexers for the R/S languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re -from pygments.lexer import Lexer, RegexLexer, include, words, do_insertions +from pygments.lexer import Lexer, RegexLexer, include, do_insertions, bygroups from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Generic @@ -80,286 +80,25 @@ class SLexer(RegexLexer): mimetypes = ['text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile'] - builtins_base = ( - 'Arg', 'Conj', 'Cstack_info', 'Encoding', 'FALSE', - 'Filter', 'Find', 'I', 'ISOdate', 'ISOdatetime', 'Im', 'Inf', - 'La.svd', 'Map', 'Math.Date', 'Math.POSIXt', 'Math.data.frame', - 'Math.difftime', 'Math.factor', 'Mod', 'NA_character_', - 'NA_complex_', 'NA_real_', 'NCOL', 'NROW', 'NULLNA_integer_', 'NaN', - 'Negate', 'NextMethod', 'Ops.Date', 'Ops.POSIXt', 'Ops.data.frame', - 'Ops.difftime', 'Ops.factor', 'Ops.numeric_version', 'Ops.ordered', - 'Position', 'R.Version', 'R.home', 'R.version', 'R.version.string', - 'RNGkind', 'RNGversion', 'R_system_version', 'Re', 'Recall', - 'Reduce', 'Summary.Date', 'Summary.POSIXct', 'Summary.POSIXlt', - 'Summary.data.frame', 'Summary.difftime', 'Summary.factor', - 'Summary.numeric_version', 'Summary.ordered', 'Sys.Date', - 'Sys.chmod', 'Sys.getenv', 'Sys.getlocale', 'Sys.getpid', - 'Sys.glob', 'Sys.info', 'Sys.localeconv', 'Sys.readlink', - 'Sys.setFileTime', 'Sys.setenv', 'Sys.setlocale', 'Sys.sleep', - 'Sys.time', 'Sys.timezone', 'Sys.umask', 'Sys.unsetenv', - 'Sys.which', 'TRUE', 'UseMethod', 'Vectorize', 'abbreviate', 'abs', - 'acos', 'acosh', 'addNA', 'addTaskCallback', 'agrep', 'alist', - 'all', 'all.equal', 'all.equal.POSIXct', 'all.equal.character', - 'all.equal.default', 'all.equal.factor', 'all.equal.formula', - 'all.equal.language', 'all.equal.list', 'all.equal.numeric', - 'all.equal.raw', 'all.names', 'all.vars', 'any', 'anyDuplicated', - 'anyDuplicated.array', 'anyDuplicated.data.frame', - 'anyDuplicated.default', 'anyDuplicated.matrix', 'aperm', - 'aperm.default', 'aperm.table', 'append', 'apply', 'args', - 'arrayInd', 'as.Date', 'as.Date.POSIXct', 'as.Date.POSIXlt', - 'as.Date.character', 'as.Date.date', 'as.Date.dates', - 'as.Date.default', 'as.Date.factor', 'as.Date.numeric', - 'as.POSIXct', 'as.POSIXct.Date', 'as.POSIXct.POSIXlt', - 'as.POSIXct.date', 'as.POSIXct.dates', 'as.POSIXct.default', - 'as.POSIXct.numeric', 'as.POSIXlt', 'as.POSIXlt.Date', - 'as.POSIXlt.POSIXct', 'as.POSIXlt.character', 'as.POSIXlt.date', - 'as.POSIXlt.dates', 'as.POSIXlt.default', 'as.POSIXlt.factor', - 'as.POSIXlt.numeric', 'as.array', 'as.array.default', 'as.call', - 'as.character', 'as.character.Date', 'as.character.POSIXt', - 'as.character.condition', 'as.character.default', - 'as.character.error', 'as.character.factor', 'as.character.hexmode', - 'as.character.numeric_version', 'as.character.octmode', - 'as.character.srcref', 'as.complex', 'as.data.frame', - 'as.data.frame.AsIs', 'as.data.frame.Date', 'as.data.frame.POSIXct', - 'as.data.frame.POSIXlt', 'as.data.frame.array', - 'as.data.frame.character', 'as.data.frame.complex', - 'as.data.frame.data.frame', 'as.data.frame.default', - 'as.data.frame.difftime', 'as.data.frame.factor', - 'as.data.frame.integer', 'as.data.frame.list', - 'as.data.frame.logical', 'as.data.frame.matrix', - 'as.data.frame.model.matrix', 'as.data.frame.numeric', - 'as.data.frame.numeric_version', 'as.data.frame.ordered', - 'as.data.frame.raw', 'as.data.frame.table', 'as.data.frame.ts', - 'as.data.frame.vector', 'as.difftime', 'as.double', - 'as.double.POSIXlt', 'as.double.difftime', 'as.environment', - 'as.expression', 'as.expression.default', 'as.factor', - 'as.function', 'as.function.default', 'as.hexmode', 'as.integer', - 'as.list', 'as.list.Date', 'as.list.POSIXct', 'as.list.data.frame', - 'as.list.default', 'as.list.environment', 'as.list.factor', - 'as.list.function', 'as.list.numeric_version', 'as.logical', - 'as.logical.factor', 'as.matrix', 'as.matrix.POSIXlt', - 'as.matrix.data.frame', 'as.matrix.default', 'as.matrix.noquote', - 'as.name', 'as.null', 'as.null.default', 'as.numeric', - 'as.numeric_version', 'as.octmode', 'as.ordered', - 'as.package_version', 'as.pairlist', 'as.qr', 'as.raw', 'as.single', - 'as.single.default', 'as.symbol', 'as.table', 'as.table.default', - 'as.vector', 'as.vector.factor', 'asNamespace', 'asS3', 'asS4', - 'asin', 'asinh', 'assign', 'atan', 'atan2', 'atanh', - 'attachNamespace', 'attr', 'attr.all.equal', 'attributes', - 'autoload', 'autoloader', 'backsolve', 'baseenv', 'basename', - 'besselI', 'besselJ', 'besselK', 'besselY', 'beta', - 'bindingIsActive', 'bindingIsLocked', 'bindtextdomain', 'bitwAnd', - 'bitwNot', 'bitwOr', 'bitwShiftL', 'bitwShiftR', 'bitwXor', 'body', - 'bquote', 'browser', 'browserCondition', 'browserSetDebug', - 'browserText', 'builtins', 'by', 'by.data.frame', 'by.default', - 'bzfile', 'c.Date', 'c.POSIXct', 'c.POSIXlt', 'c.noquote', - 'c.numeric_version', 'call', 'callCC', 'capabilities', 'casefold', - 'cat', 'category', 'cbind', 'cbind.data.frame', 'ceiling', - 'char.expand', 'charToRaw', 'charmatch', 'chartr', 'check_tzones', - 'chol', 'chol.default', 'chol2inv', 'choose', 'class', - 'clearPushBack', 'close', 'close.connection', 'close.srcfile', - 'close.srcfilealias', 'closeAllConnections', 'col', 'colMeans', - 'colSums', 'colnames', 'commandArgs', 'comment', 'computeRestarts', - 'conditionCall', 'conditionCall.condition', 'conditionMessage', - 'conditionMessage.condition', 'conflicts', 'contributors', 'cos', - 'cosh', 'crossprod', 'cummax', 'cummin', 'cumprod', 'cumsum', 'cut', - 'cut.Date', 'cut.POSIXt', 'cut.default', 'dQuote', 'data.class', - 'data.matrix', 'date', 'debug', 'debugonce', - 'default.stringsAsFactors', 'delayedAssign', 'deparse', 'det', - 'determinant', 'determinant.matrix', 'dget', 'diag', 'diff', - 'diff.Date', 'diff.POSIXt', 'diff.default', 'difftime', 'digamma', - 'dim', 'dim.data.frame', 'dimnames', 'dimnames.data.frame', 'dir', - 'dir.create', 'dirname', 'do.call', 'dput', 'drop', 'droplevels', - 'droplevels.data.frame', 'droplevels.factor', 'dump', 'duplicated', - 'duplicated.POSIXlt', 'duplicated.array', 'duplicated.data.frame', - 'duplicated.default', 'duplicated.matrix', - 'duplicated.numeric_version', 'dyn.load', 'dyn.unload', 'eapply', - 'eigen', 'else', 'emptyenv', 'enc2native', 'enc2utf8', - 'encodeString', 'enquote', 'env.profile', 'environment', - 'environmentIsLocked', 'environmentName', 'eval', 'eval.parent', - 'evalq', 'exists', 'exp', 'expand.grid', 'expm1', 'expression', - 'factor', 'factorial', 'fifo', 'file', 'file.access', 'file.append', - 'file.choose', 'file.copy', 'file.create', 'file.exists', - 'file.info', 'file.link', 'file.path', 'file.remove', 'file.rename', - 'file.show', 'file.symlink', 'find.package', 'findInterval', - 'findPackageEnv', 'findRestart', 'floor', 'flush', - 'flush.connection', 'force', 'formals', 'format', - 'format.AsIs', 'format.Date', 'format.POSIXct', 'format.POSIXlt', - 'format.data.frame', 'format.default', 'format.difftime', - 'format.factor', 'format.hexmode', 'format.info', - 'format.libraryIQR', 'format.numeric_version', 'format.octmode', - 'format.packageInfo', 'format.pval', 'format.summaryDefault', - 'formatC', 'formatDL', 'forwardsolve', 'gamma', 'gc', 'gc.time', - 'gcinfo', 'gctorture', 'gctorture2', 'get', 'getAllConnections', - 'getCallingDLL', 'getCallingDLLe', 'getConnection', - 'getDLLRegisteredRoutines', 'getDLLRegisteredRoutines.DLLInfo', - 'getDLLRegisteredRoutines.character', 'getElement', - 'getExportedValue', 'getHook', 'getLoadedDLLs', 'getNamespace', - 'getNamespaceExports', 'getNamespaceImports', 'getNamespaceInfo', - 'getNamespaceName', 'getNamespaceUsers', 'getNamespaceVersion', - 'getNativeSymbolInfo', 'getOption', 'getRversion', 'getSrcLines', - 'getTaskCallbackNames', 'geterrmessage', 'gettext', 'gettextf', - 'getwd', 'gl', 'globalenv', 'gregexpr', 'grep', 'grepRaw', 'grepl', - 'gsub', 'gzcon', 'gzfile', 'head', 'iconv', 'iconvlist', - 'icuSetCollate', 'identical', 'identity', 'ifelse', 'importIntoEnv', - 'in', 'inherits', 'intToBits', 'intToUtf8', 'interaction', 'interactive', - 'intersect', 'inverse.rle', 'invisible', 'invokeRestart', - 'invokeRestartInteractively', 'is.R', 'is.array', 'is.atomic', - 'is.call', 'is.character', 'is.complex', 'is.data.frame', - 'is.double', 'is.element', 'is.environment', 'is.expression', - 'is.factor', 'is.finite', 'is.function', 'is.infinite', - 'is.integer', 'is.language', 'is.list', 'is.loaded', 'is.logical', - 'is.matrix', 'is.na', 'is.na.POSIXlt', 'is.na.data.frame', - 'is.na.numeric_version', 'is.name', 'is.nan', 'is.null', - 'is.numeric', 'is.numeric.Date', 'is.numeric.POSIXt', - 'is.numeric.difftime', 'is.numeric_version', 'is.object', - 'is.ordered', 'is.package_version', 'is.pairlist', 'is.primitive', - 'is.qr', 'is.raw', 'is.recursive', 'is.single', 'is.symbol', - 'is.table', 'is.unsorted', 'is.vector', 'isBaseNamespace', - 'isIncomplete', 'isNamespace', 'isOpen', 'isRestart', 'isS4', - 'isSeekable', 'isSymmetric', 'isSymmetric.matrix', 'isTRUE', - 'isatty', 'isdebugged', 'jitter', 'julian', 'julian.Date', - 'julian.POSIXt', 'kappa', 'kappa.default', 'kappa.lm', 'kappa.qr', - 'kronecker', 'l10n_info', 'labels', 'labels.default', 'lapply', - 'lazyLoad', 'lazyLoadDBexec', 'lazyLoadDBfetch', 'lbeta', 'lchoose', - 'length', 'length.POSIXlt', 'letters', 'levels', 'levels.default', - 'lfactorial', 'lgamma', 'library.dynam', 'library.dynam.unload', - 'licence', 'license', 'list.dirs', 'list.files', 'list2env', 'load', - 'loadNamespace', 'loadedNamespaces', 'loadingNamespaceInfo', - 'local', 'lockBinding', 'lockEnvironment', 'log', 'log10', 'log1p', - 'log2', 'logb', 'lower.tri', 'ls', 'make.names', 'make.unique', - 'makeActiveBinding', 'mapply', 'margin.table', 'mat.or.vec', - 'match', 'match.arg', 'match.call', 'match.fun', 'max', 'max.col', - 'mean', 'mean.Date', 'mean.POSIXct', 'mean.POSIXlt', 'mean.default', - 'mean.difftime', 'mem.limits', 'memCompress', 'memDecompress', - 'memory.profile', 'merge', 'merge.data.frame', 'merge.default', - 'message', 'mget', 'min', 'missing', 'mode', 'month.abb', - 'month.name', 'months', 'months.Date', 'months.POSIXt', - 'months.abb', 'months.nameletters', 'names', 'names.POSIXlt', - 'namespaceExport', 'namespaceImport', 'namespaceImportClasses', - 'namespaceImportFrom', 'namespaceImportMethods', 'nargs', 'nchar', - 'ncol', 'new.env', 'ngettext', 'nlevels', 'noquote', 'norm', - 'normalizePath', 'nrow', 'numeric_version', 'nzchar', 'objects', - 'oldClass', 'on.exit', 'open', 'open.connection', 'open.srcfile', - 'open.srcfilealias', 'open.srcfilecopy', 'options', 'order', - 'ordered', 'outer', 'packBits', 'packageEvent', - 'packageHasNamespace', 'packageStartupMessage', 'package_version', - 'pairlist', 'parent.env', 'parent.frame', 'parse', - 'parseNamespaceFile', 'paste', 'paste0', 'path.expand', - 'path.package', 'pipe', 'pmatch', 'pmax', 'pmax.int', 'pmin', - 'pmin.int', 'polyroot', 'pos.to.env', 'pretty', 'pretty.default', - 'prettyNum', 'print', 'print.AsIs', 'print.DLLInfo', - 'print.DLLInfoList', 'print.DLLRegisteredRoutines', 'print.Date', - 'print.NativeRoutineList', 'print.POSIXct', 'print.POSIXlt', - 'print.by', 'print.condition', 'print.connection', - 'print.data.frame', 'print.default', 'print.difftime', - 'print.factor', 'print.function', 'print.hexmode', - 'print.libraryIQR', 'print.listof', 'print.noquote', - 'print.numeric_version', 'print.octmode', 'print.packageInfo', - 'print.proc_time', 'print.restart', 'print.rle', - 'print.simple.list', 'print.srcfile', 'print.srcref', - 'print.summary.table', 'print.summaryDefault', 'print.table', - 'print.warnings', 'prmatrix', 'proc.time', 'prod', 'prop.table', - 'provideDimnames', 'psigamma', 'pushBack', 'pushBackLength', 'q', - 'qr', 'qr.Q', 'qr.R', 'qr.X', 'qr.coef', 'qr.default', 'qr.fitted', - 'qr.qty', 'qr.qy', 'qr.resid', 'qr.solve', 'quarters', - 'quarters.Date', 'quarters.POSIXt', 'quit', 'quote', 'range', - 'range.default', 'rank', 'rapply', 'raw', 'rawConnection', - 'rawConnectionValue', 'rawShift', 'rawToBits', 'rawToChar', 'rbind', - 'rbind.data.frame', 'rcond', 'read.dcf', 'readBin', 'readChar', - 'readLines', 'readRDS', 'readRenviron', 'readline', 'reg.finalizer', - 'regexec', 'regexpr', 'registerS3method', 'registerS3methods', - 'regmatches', 'remove', 'removeTaskCallback', 'rep', 'rep.Date', - 'rep.POSIXct', 'rep.POSIXlt', 'rep.factor', 'rep.int', - 'rep.numeric_version', 'rep_len', 'replace', 'replicate', - 'requireNamespace', 'restartDescription', 'restartFormals', - 'retracemem', 'rev', 'rev.default', 'rle', 'rm', 'round', - 'round.Date', 'round.POSIXt', 'row', 'row.names', - 'row.names.data.frame', 'row.names.default', 'rowMeans', 'rowSums', - 'rownames', 'rowsum', 'rowsum.data.frame', 'rowsum.default', - 'sQuote', 'sample', 'sample.int', 'sapply', 'save', 'save.image', - 'saveRDS', 'scale', 'scale.default', 'scan', 'search', - 'searchpaths', 'seek', 'seek.connection', 'seq', 'seq.Date', - 'seq.POSIXt', 'seq.default', 'seq.int', 'seq_along', 'seq_len', - 'sequence', 'serialize', 'set.seed', 'setHook', 'setNamespaceInfo', - 'setSessionTimeLimit', 'setTimeLimit', 'setdiff', 'setequal', - 'setwd', 'shQuote', 'showConnections', 'sign', 'signalCondition', - 'signif', 'simpleCondition', 'simpleError', 'simpleMessage', - 'simpleWarning', 'simplify2array', 'sin', 'single', - 'sinh', 'sink', 'sink.number', 'slice.index', 'socketConnection', - 'socketSelect', 'solve', 'solve.default', 'solve.qr', 'sort', - 'sort.POSIXlt', 'sort.default', 'sort.int', 'sort.list', 'split', - 'split.Date', 'split.POSIXct', 'split.data.frame', 'split.default', - 'sprintf', 'sqrt', 'srcfile', 'srcfilealias', 'srcfilecopy', - 'srcref', 'standardGeneric', 'stderr', 'stdin', 'stdout', 'stop', - 'stopifnot', 'storage.mode', 'strftime', 'strptime', 'strsplit', - 'strtoi', 'strtrim', 'structure', 'strwrap', 'sub', 'subset', - 'subset.data.frame', 'subset.default', 'subset.matrix', - 'substitute', 'substr', 'substring', 'sum', 'summary', - 'summary.Date', 'summary.POSIXct', 'summary.POSIXlt', - 'summary.connection', 'summary.data.frame', 'summary.default', - 'summary.factor', 'summary.matrix', 'summary.proc_time', - 'summary.srcfile', 'summary.srcref', 'summary.table', - 'suppressMessages', 'suppressPackageStartupMessages', - 'suppressWarnings', 'svd', 'sweep', 'sys.call', 'sys.calls', - 'sys.frame', 'sys.frames', 'sys.function', 'sys.load.image', - 'sys.nframe', 'sys.on.exit', 'sys.parent', 'sys.parents', - 'sys.save.image', 'sys.source', 'sys.status', 'system', - 'system.file', 'system.time', 'system2', 't', 't.data.frame', - 't.default', 'table', 'tabulate', 'tail', 'tan', 'tanh', 'tapply', - 'taskCallbackManager', 'tcrossprod', 'tempdir', 'tempfile', - 'testPlatformEquivalence', 'textConnection', 'textConnectionValue', - 'toString', 'toString.default', 'tolower', 'topenv', 'toupper', - 'trace', 'traceback', 'tracemem', 'tracingState', 'transform', - 'transform.data.frame', 'transform.default', 'trigamma', 'trunc', - 'trunc.Date', 'trunc.POSIXt', 'truncate', 'truncate.connection', - 'try', 'tryCatch', 'typeof', 'unclass', 'undebug', 'union', - 'unique', 'unique.POSIXlt', 'unique.array', 'unique.data.frame', - 'unique.default', 'unique.matrix', 'unique.numeric_version', - 'units', 'units.difftime', 'unix.time', 'unlink', 'unlist', - 'unloadNamespace', 'unlockBinding', 'unname', 'unserialize', - 'unsplit', 'untrace', 'untracemem', 'unz', 'upper.tri', 'url', - 'utf8ToInt', 'vapply', 'version', 'warning', 'warnings', 'weekdays', - 'weekdays.Date', 'weekdays.POSIXt', 'which', 'which.max', - 'which.min', 'with', 'with.default', 'withCallingHandlers', - 'withRestarts', 'withVisible', 'within', 'within.data.frame', - 'within.list', 'write', 'write.dcf', 'writeBin', 'writeChar', - 'writeLines', 'xor', 'xor.hexmode', 'xor.octmode', - 'xpdrows.data.frame', 'xtfrm', 'xtfrm.AsIs', 'xtfrm.Date', - 'xtfrm.POSIXct', 'xtfrm.POSIXlt', 'xtfrm.Surv', 'xtfrm.default', - 'xtfrm.difftime', 'xtfrm.factor', 'xtfrm.numeric_version', 'xzfile', - 'zapsmall' - ) - + valid_name = r'(?:`[^`\\]*(?:\\.[^`\\]*)*`)|(?:(?:[a-zA-z]|[_.][^0-9])[\w_.]*)' tokens = { 'comments': [ (r'#.*$', Comment.Single), ], 'valid_name': [ - (r'[a-zA-Z][\w.]*', Text), - # can begin with ., but not if that is followed by a digit - (r'\.[a-zA-Z_][\w.]*', Text), + (valid_name, Name), ], 'punctuation': [ (r'\[{1,2}|\]{1,2}|\(|\)|;|,', Punctuation), ], 'keywords': [ - (words(builtins_base, suffix=r'(?![\w. =])'), - Keyword.Pseudo), (r'(if|else|for|while|repeat|in|next|break|return|switch|function)' r'(?![\w.])', Keyword.Reserved), - (r'(array|category|character|complex|double|function|integer|list|' - r'logical|matrix|numeric|vector|data.frame|c)' - r'(?![\w.])', - Keyword.Type), - (r'(library|require|attach|detach|source)' - r'(?![\w.])', - Keyword.Namespace) ], 'operators': [ (r'<<?-|->>?|-|==|<=|>=|<|>|&&?|!=|\|\|?|\?', Operator), - (r'\*|\+|\^|/|!|%[^%]*%|=|~|\$|@|:{1,3}', Operator) + (r'\*|\+|\^|/|!|%[^%]*%|=|~|\$|@|:{1,3}', Operator), ], 'builtin_symbols': [ (r'(NULL|NA(_(integer|real|complex|character)_)?|' @@ -379,17 +118,18 @@ class SLexer(RegexLexer): include('comments'), # whitespaces (r'\s+', Text), - (r'`.*?`', String.Backtick), (r'\'', String, 'string_squote'), (r'\"', String, 'string_dquote'), include('builtin_symbols'), + include('valid_name'), include('numbers'), include('keywords'), include('punctuation'), include('operators'), - include('valid_name'), ], 'root': [ + # calls: + (r'(%s)\s*(?=\()' % valid_name, Name.Function), include('statements'), # blocks: (r'\{|\}', Punctuation), @@ -421,7 +161,7 @@ class RdLexer(RegexLexer): This is a very minimal implementation, highlighting little more than the macros. A description of Rd syntax is found in `Writing R Extensions <http://cran.r-project.org/doc/manuals/R-exts.html>`_ - and `Parsing Rd files <developer.r-project.org/parseRd.pdf>`_. + and `Parsing Rd files <http://developer.r-project.org/parseRd.pdf>`_. .. versionadded:: 1.6 """ diff --git a/pygments/lexers/rdf.py b/pygments/lexers/rdf.py index d0f8778a..8682387e 100644 --- a/pygments/lexers/rdf.py +++ b/pygments/lexers/rdf.py @@ -5,7 +5,7 @@ Lexers for semantic web and RDF query languages and markup. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -97,7 +97,7 @@ class SparqlLexer(RegexLexer): 'root': [ (r'\s+', Text), # keywords :: - (r'((?i)select|construct|describe|ask|where|filter|group\s+by|minus|' + (r'(?i)(select|construct|describe|ask|where|filter|group\s+by|minus|' r'distinct|reduced|from\s+named|from|order\s+by|desc|asc|limit|' r'offset|bindings|load|clear|drop|create|add|move|copy|' r'insert\s+data|delete\s+data|delete\s+where|delete|insert|' @@ -111,10 +111,10 @@ class SparqlLexer(RegexLexer): # # variables :: ('[?$]' + VARNAME, Name.Variable), # prefixed names :: - (r'(' + PN_PREFIX + ')?(\:)(' + PN_LOCAL + ')?', + (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + r')?', bygroups(Name.Namespace, Punctuation, Name.Tag)), # function names :: - (r'((?i)str|lang|langmatches|datatype|bound|iri|uri|bnode|rand|abs|' + (r'(?i)(str|lang|langmatches|datatype|bound|iri|uri|bnode|rand|abs|' r'ceil|floor|round|concat|strlen|ucase|lcase|encode_for_uri|' r'contains|strstarts|strends|strbefore|strafter|year|month|day|' r'hours|minutes|seconds|timezone|tz|now|md5|sha1|sha256|sha384|' @@ -125,7 +125,7 @@ class SparqlLexer(RegexLexer): # boolean literals :: (r'(true|false)', Keyword.Constant), # double literals :: - (r'[+\-]?(\d+\.\d*' + EXPONENT + '|\.?\d+' + EXPONENT + ')', Number.Float), + (r'[+\-]?(\d+\.\d*' + EXPONENT + r'|\.?\d+' + EXPONENT + ')', Number.Float), # decimal literals :: (r'[+\-]?(\d+\.\d*|\.\d+)', Number.Float), # integer literals :: @@ -268,3 +268,10 @@ class TurtleLexer(RegexLexer): ], } + + # Turtle and Tera Term macro files share the same file extension + # but each has a recognizable and distinct syntax. + def analyse_text(text): + for t in ('@base ', 'BASE ', '@prefix ', 'PREFIX '): + if re.search(r'^\s*%s' % t, text): + return 0.80 diff --git a/pygments/lexers/rebol.py b/pygments/lexers/rebol.py index f3d00200..1b3d90f5 100644 --- a/pygments/lexers/rebol.py +++ b/pygments/lexers/rebol.py @@ -5,7 +5,7 @@ Lexers for the REBOL and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -102,12 +102,12 @@ class RebolLexer(RegexLexer): yield match.start(), Generic.Heading, word elif re.match("to-.*", word): yield match.start(), Keyword, word - elif re.match('(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|<>|<|>|<=|>=)$', + elif re.match(r'(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|<>|<|>|<=|>=)$', word): yield match.start(), Operator, word - elif re.match(".*\?$", word): + elif re.match(r".*\?$", word): yield match.start(), Keyword, word - elif re.match(".*\!$", word): + elif re.match(r".*\!$", word): yield match.start(), Keyword.Type, word elif re.match("'.*", word): yield match.start(), Name.Variable.Instance, word # lit-word @@ -239,7 +239,7 @@ class RebolLexer(RegexLexer): if re.match(r'^\s*REBOL\s*\[', text, re.IGNORECASE): # The code starts with REBOL header return 1.0 - elif re.search(r'\s*REBOL\s*[', text, re.IGNORECASE): + elif re.search(r'\s*REBOL\s*\[', text, re.IGNORECASE): # The code contains REBOL header but also some text before it return 0.5 @@ -297,10 +297,10 @@ class RedLexer(RegexLexer): yield match.start(), Keyword.Namespace, word elif re.match("to-.*", word): yield match.start(), Keyword, word - elif re.match('(\+|-\*\*|-|\*\*|//|/|\*|and|or|xor|=\?|===|==|=|<>|<=|>=|' - '<<<|>>>|<<|>>|<|>%)$', word): + elif re.match(r'(\+|-\*\*|-|\*\*|//|/|\*|and|or|xor|=\?|===|==|=|<>|<=|>=|' + r'<<<|>>>|<<|>>|<|>%)$', word): yield match.start(), Operator, word - elif re.match(".*\!$", word): + elif re.match(r".*\!$", word): yield match.start(), Keyword.Type, word elif re.match("'.*", word): yield match.start(), Name.Variable.Instance, word # lit-word diff --git a/pygments/lexers/resource.py b/pygments/lexers/resource.py index f7494904..6cc88b95 100644 --- a/pygments/lexers/resource.py +++ b/pygments/lexers/resource.py @@ -5,7 +5,7 @@ Lexer for resource definition files. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/rnc.py b/pygments/lexers/rnc.py index 2f2aacdd..8f0ba5cf 100644 --- a/pygments/lexers/rnc.py +++ b/pygments/lexers/rnc.py @@ -5,7 +5,7 @@ Lexer for Relax-NG Compact syntax - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/roboconf.py b/pygments/lexers/roboconf.py index 8c7df83d..f820fe1e 100644 --- a/pygments/lexers/roboconf.py +++ b/pygments/lexers/roboconf.py @@ -5,7 +5,7 @@ Lexers for Roboconf DSL. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/robotframework.py b/pygments/lexers/robotframework.py index e868127b..1288da8d 100644 --- a/pygments/lexers/robotframework.py +++ b/pygments/lexers/robotframework.py @@ -5,7 +5,7 @@ Lexer for Robot Framework. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -161,7 +161,7 @@ class RowTokenizer(object): class RowSplitter(object): _space_splitter = re.compile('( {2,})') - _pipe_splitter = re.compile('((?:^| +)\|(?: +|$))') + _pipe_splitter = re.compile(r'((?:^| +)\|(?: +|$))') def split(self, row): splitter = (row.startswith('| ') and self._split_from_pipes diff --git a/pygments/lexers/ruby.py b/pygments/lexers/ruby.py index fe750f1a..723895d1 100644 --- a/pygments/lexers/ruby.py +++ b/pygments/lexers/ruby.py @@ -5,7 +5,7 @@ Lexers for Ruby and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -403,8 +403,8 @@ class RubyConsoleLexer(Lexer): aliases = ['rbcon', 'irb'] mimetypes = ['text/x-ruby-shellsession'] - _prompt_re = re.compile('irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] ' - '|>> |\?> ') + _prompt_re = re.compile(r'irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] ' + r'|>> |\?> ') def get_tokens_unprocessed(self, text): rblexer = RubyLexer(**self.options) @@ -498,11 +498,11 @@ class FancyLexer(RegexLexer): (r'[a-zA-Z](\w|[-+?!=*/^><%])*:', Name.Function), # operators, must be below functions (r'[-+*/~,<>=&!?%^\[\].$]+', Operator), - ('[A-Z]\w*', Name.Constant), - ('@[a-zA-Z_]\w*', Name.Variable.Instance), - ('@@[a-zA-Z_]\w*', Name.Variable.Class), + (r'[A-Z]\w*', Name.Constant), + (r'@[a-zA-Z_]\w*', Name.Variable.Instance), + (r'@@[a-zA-Z_]\w*', Name.Variable.Class), ('@@?', Operator), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), # numbers - / checks are necessary to avoid mismarking regexes, # see comment in RubyLexer (r'(0[oO]?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?', diff --git a/pygments/lexers/rust.py b/pygments/lexers/rust.py index 6914f54d..f731785f 100644 --- a/pygments/lexers/rust.py +++ b/pygments/lexers/rust.py @@ -5,7 +5,7 @@ Lexers for the Rust language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -24,12 +24,12 @@ class RustLexer(RegexLexer): """ name = 'Rust' filenames = ['*.rs', '*.rs.in'] - aliases = ['rust'] + aliases = ['rust', 'rs'] mimetypes = ['text/rust'] keyword_types = ( words(('u8', 'u16', 'u32', 'u64', 'i8', 'i16', 'i32', 'i64', - 'usize', 'isize', 'f32', 'f64', 'str', 'bool'), + 'i128', 'u128', 'usize', 'isize', 'f32', 'f64', 'str', 'bool'), suffix=r'\b'), Keyword.Type) @@ -59,7 +59,7 @@ class RustLexer(RegexLexer): tokens = { 'root': [ # rust allows a file to start with a shebang, but if the first line - # starts with #![ then it’s not a shebang but a crate attribute. + # starts with #![ then it's not a shebang but a crate attribute. (r'#![^[\r\n].*$', Comment.Preproc), default('base'), ], @@ -78,10 +78,10 @@ class RustLexer(RegexLexer): (r"""\$([a-zA-Z_]\w*|\(,?|\),?|,?)""", Comment.Preproc), # Keywords (words(( - 'as', 'box', 'const', 'crate', 'else', 'extern', - 'for', 'if', 'impl', 'in', 'loop', 'match', 'move', - 'mut', 'pub', 'ref', 'return', 'static', 'super', - 'trait', 'unsafe', 'use', 'where', 'while'), suffix=r'\b'), + 'as', 'async', 'await', 'box', 'const', 'crate', 'else', + 'extern', 'for', 'if', 'impl', 'in', 'loop', 'match', 'move', + 'mut', 'pub', 'ref', 'return', 'static', 'super', 'trait', + 'try', 'unsafe', 'use', 'where', 'while'), suffix=r'\b'), Keyword), (words(('abstract', 'alignof', 'become', 'do', 'final', 'macro', 'offsetof', 'override', 'priv', 'proc', 'pure', 'sizeof', @@ -95,7 +95,7 @@ class RustLexer(RegexLexer): (r'(default)(\s+)(type|fn)\b', bygroups(Keyword, Text, Keyword)), keyword_types, (r'self\b', Name.Builtin.Pseudo), - # Prelude (taken from Rust’s src/libstd/prelude.rs) + # Prelude (taken from Rust's src/libstd/prelude.rs) builtin_types, # Path seperators, so types don't catch them. (r'::\b', Text), diff --git a/pygments/lexers/sas.py b/pygments/lexers/sas.py index 3747ed9a..38d80774 100644 --- a/pygments/lexers/sas.py +++ b/pygments/lexers/sas.py @@ -5,7 +5,7 @@ Lexer for SAS. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/scripting.py b/pygments/lexers/scripting.py index b3af606e..a340f8e0 100644 --- a/pygments/lexers/scripting.py +++ b/pygments/lexers/scripting.py @@ -5,7 +5,7 @@ Lexer for scripting and embedded languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -104,7 +104,7 @@ class LuaLexer(RegexLexer): (r'%s(?=%s*[.:])' % (_name, _s), Name.Class), (_name, Name.Function, '#pop'), # inline function - ('\(', Punctuation, '#pop'), + (r'\(', Punctuation, '#pop'), ], 'goto': [ @@ -696,8 +696,8 @@ class AppleScriptLexer(RegexLexer): (r'[-+]?\d+', Number.Integer), ], 'comment': [ - ('\(\*', Comment.Multiline, '#push'), - ('\*\)', Comment.Multiline, '#pop'), + (r'\(\*', Comment.Multiline, '#push'), + (r'\*\)', Comment.Multiline, '#pop'), ('[^*(]+', Comment.Multiline), ('[*(]', Comment.Multiline), ], diff --git a/pygments/lexers/sgf.py b/pygments/lexers/sgf.py new file mode 100644 index 00000000..fed864a6 --- /dev/null +++ b/pygments/lexers/sgf.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.sgf + ~~~~~~~~~~~~~~~~~~~ + + Lexer for Smart Game Format (sgf) file format. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups +from pygments.token import Name, Literal, String, Text, Punctuation + +__all__ = ["SmartGameFormatLexer"] + + +class SmartGameFormatLexer(RegexLexer): + """ + Lexer for Smart Game Format (sgf) file format. + + The format is used to store game records of board games for two players + (mainly Go game). + For more information about the definition of the format, see: + https://www.red-bean.com/sgf/ + + .. versionadded:: 2.4 + """ + name = 'SmartGameFormat' + aliases = ['sgf'] + filenames = ['*.sgf'] + + tokens = { + 'root': [ + (r'[\s():;]', Punctuation), + # tokens: + (r'(A[BW]|AE|AN|AP|AR|AS|[BW]L|BM|[BW]R|[BW]S|[BW]T|CA|CH|CP|CR|' + r'DD|DM|DO|DT|EL|EV|EX|FF|FG|G[BW]|GC|GM|GN|HA|HO|ID|IP|IT|IY|KM|' + r'KO|LB|LN|LT|L|MA|MN|M|N|OB|OM|ON|OP|OT|OV|P[BW]|PC|PL|PM|RE|RG|' + r'RO|RU|SO|SC|SE|SI|SL|SO|SQ|ST|SU|SZ|T[BW]|TC|TE|TM|TR|UC|US|VW|' + r'V|[BW]|C)', + Name.Builtin), + # number: + (r'(\[)([0-9.]+)(\])', + bygroups(Punctuation, Literal.Number, Punctuation)), + # date: + (r'(\[)([0-9]{4}-[0-9]{2}-[0-9]{2})(\])', + bygroups(Punctuation, Literal.Date, Punctuation)), + # point: + (r'(\[)([a-z]{2})(\])', + bygroups(Punctuation, String, Punctuation)), + # double points: + (r'(\[)([a-z]{2})(:)([a-z]{2})(\])', + bygroups(Punctuation, String, Punctuation, String, Punctuation)), + + (r'(\[)([\w\s#()+,\-.:?]+)(\])', + bygroups(Punctuation, String, Punctuation)), + (r'(\[)(\s.*)(\])', + bygroups(Punctuation, Text, Punctuation)), + ], + } diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py index ceb6f14d..972c4004 100644 --- a/pygments/lexers/shell.py +++ b/pygments/lexers/shell.py @@ -5,7 +5,7 @@ Lexers for various shells. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -19,7 +19,7 @@ from pygments.util import shebang_matches __all__ = ['BashLexer', 'BashSessionLexer', 'TcshLexer', 'BatchLexer', - 'MSDOSSessionLexer', 'PowerShellLexer', + 'SlurmBashLexer', 'MSDOSSessionLexer', 'PowerShellLexer', 'PowerShellSessionLexer', 'TcshSessionLexer', 'FishShellLexer'] line_re = re.compile('.*?\n') @@ -38,7 +38,7 @@ class BashLexer(RegexLexer): '*.exheres-0', '*.exlib', '*.zsh', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', 'PKGBUILD'] - mimetypes = ['application/x-sh', 'application/x-shellscript'] + mimetypes = ['application/x-sh', 'application/x-shellscript', 'text/x-shellscript'] tokens = { 'root': [ @@ -76,7 +76,7 @@ class BashLexer(RegexLexer): (r'&&|\|\|', Operator), ], 'data': [ - (r'(?s)\$?"(\\\\|\\[0-7]+|\\.|[^"\\$])*"', String.Double), + (r'(?s)\$?"(\\.|[^"\\$])*"', String.Double), (r'"', String.Double, 'string'), (r"(?s)\$'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single), (r"(?s)'.*?'", String.Single), @@ -126,6 +126,28 @@ class BashLexer(RegexLexer): return 0.2 +class SlurmBashLexer(BashLexer): + """ + Lexer for (ba|k|z|)sh Slurm scripts. + + .. versionadded:: 2.4 + """ + + name = 'Slurm' + aliases = ['slurm', 'sbatch'] + filenames = ['*.sl'] + mimetypes = [] + EXTRA_KEYWORDS = {'srun'} + + def get_tokens_unprocessed(self, text): + for index, token, value in BashLexer.get_tokens_unprocessed(self, text): + if token is Text and value in self.EXTRA_KEYWORDS: + yield index, Name.Builtin, value + elif token is Comment.Single and 'SBATCH' in value: + yield index, Keyword.Pseudo, value + else: + yield index, token, value + class ShellSessionBaseLexer(Lexer): """ Base lexer for simplistic shell sessions. @@ -518,7 +540,7 @@ class MSDOSSessionLexer(ShellSessionBaseLexer): mimetypes = [] _innerLexerCls = BatchLexer - _ps1rgx = r'^([^>]+>)(.*\n?)' + _ps1rgx = r'^([^>]*>)(.*\n?)' _ps2 = 'More? ' @@ -638,13 +660,29 @@ class PowerShellLexer(RegexLexer): 'wildcard').split() verbs = ( - 'write where wait use update unregister undo trace test tee take ' - 'suspend stop start split sort skip show set send select scroll resume ' - 'restore restart resolve resize reset rename remove register receive ' - 'read push pop ping out new move measure limit join invoke import ' - 'group get format foreach export expand exit enter enable disconnect ' - 'disable debug cxnew copy convertto convertfrom convert connect ' - 'complete compare clear checkpoint aggregate add').split() + 'write where watch wait use update unregister unpublish unprotect ' + 'unlock uninstall undo unblock trace test tee take sync switch ' + 'suspend submit stop step start split sort skip show set send select ' + 'search scroll save revoke resume restore restart resolve resize ' + 'reset request repair rename remove register redo receive read push ' + 'publish protect pop ping out optimize open new move mount merge ' + 'measure lock limit join invoke install initialize import hide group ' + 'grant get format foreach find export expand exit enter enable edit ' + 'dismount disconnect disable deny debug cxnew copy convertto ' + 'convertfrom convert connect confirm compress complete compare close ' + 'clear checkpoint block backup assert approve aggregate add').split() + + aliases_ = ( + 'ac asnp cat cd cfs chdir clc clear clhy cli clp cls clv cnsn ' + 'compare copy cp cpi cpp curl cvpa dbp del diff dir dnsn ebp echo epal ' + 'epcsv epsn erase etsn exsn fc fhx fl foreach ft fw gal gbp gc gci gcm ' + 'gcs gdr ghy gi gjb gl gm gmo gp gps gpv group gsn gsnp gsv gu gv gwmi ' + 'h history icm iex ihy ii ipal ipcsv ipmo ipsn irm ise iwmi iwr kill lp ' + 'ls man md measure mi mount move mp mv nal ndr ni nmo npssc nsn nv ogv ' + 'oh popd ps pushd pwd r rbp rcjb rcsn rd rdr ren ri rjb rm rmdir rmo ' + 'rni rnp rp rsn rsnp rujb rv rvpa rwmi sajb sal saps sasv sbp sc select ' + 'set shcm si sl sleep sls sort sp spjb spps spsv start sujb sv swmi tee ' + 'trcm type wget where wjb write').split() commenthelp = ( 'component description example externalhelp forwardhelpcategory ' @@ -672,6 +710,7 @@ class PowerShellLexer(RegexLexer): (r'(%s)\b' % '|'.join(keywords), Keyword), (r'-(%s)\b' % '|'.join(operators), Operator), (r'(%s)-[a-z_]\w*\b' % '|'.join(verbs), Name.Builtin), + (r'(%s)\s' % '|'.join(aliases_), Name.Builtin), (r'\[[a-z_\[][\w. `,\[\]]*\]', Name.Constant), # .net [type]s (r'-[a-z_]\w*', Name), (r'\w+', Name), diff --git a/pygments/lexers/slash.py b/pygments/lexers/slash.py new file mode 100644 index 00000000..b6314105 --- /dev/null +++ b/pygments/lexers/slash.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.slash + ~~~~~~~~~~~~~~~~~~~~~ + + Lexer for the `Slash <https://github.com/arturadib/Slash-A>`_ programming + language. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import ExtendedRegexLexer, bygroups, DelegatingLexer +from pygments.token import Name, Number, String, Comment, Punctuation, \ + Other, Keyword, Operator, Whitespace + +__all__ = ['SlashLexer'] + + +class SlashLanguageLexer(ExtendedRegexLexer): + _nkw = r'(?=[^a-zA-Z_0-9])' + + def move_state(new_state): + return ("#pop", new_state) + + def right_angle_bracket(lexer, match, ctx): + if len(ctx.stack) > 1 and ctx.stack[-2] == "string": + ctx.stack.pop() + yield match.start(), String.Interpol, "}" + ctx.pos = match.end() + pass + + tokens = { + "root": [ + (r"<%=", Comment.Preproc, move_state("slash")), + (r"<%!!", Comment.Preproc, move_state("slash")), + (r"<%#.*?%>", Comment.Multiline), + (r"<%", Comment.Preproc, move_state("slash")), + (r".|\n", Other), + ], + "string": [ + (r"\\", String.Escape, move_state("string_e")), + (r"\"", String, move_state("slash")), + (r"#\{", String.Interpol, "slash"), + (r'.|\n', String), + ], + "string_e": [ + (r'n', String.Escape, move_state("string")), + (r't', String.Escape, move_state("string")), + (r'r', String.Escape, move_state("string")), + (r'e', String.Escape, move_state("string")), + (r'x[a-fA-F0-9]{2}', String.Escape, move_state("string")), + (r'.', String.Escape, move_state("string")), + ], + "regexp": [ + (r'}[a-z]*', String.Regex, move_state("slash")), + (r'\\(.|\n)', String.Regex), + (r'{', String.Regex, "regexp_r"), + (r'.|\n', String.Regex), + ], + "regexp_r": [ + (r'}[a-z]*', String.Regex, "#pop"), + (r'\\(.|\n)', String.Regex), + (r'{', String.Regex, "regexp_r"), + ], + "slash": [ + (r"%>", Comment.Preproc, move_state("root")), + (r"\"", String, move_state("string")), + (r"'[a-zA-Z0-9_]+", String), + (r'%r{', String.Regex, move_state("regexp")), + (r'/\*.*?\*/', Comment.Multiline), + (r"(#|//).*?\n", Comment.Single), + (r'-?[0-9]+e[+-]?[0-9]+', Number.Float), + (r'-?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?', Number.Float), + (r'-?[0-9]+', Number.Integer), + (r'nil'+_nkw, Name.Builtin), + (r'true'+_nkw, Name.Builtin), + (r'false'+_nkw, Name.Builtin), + (r'self'+_nkw, Name.Builtin), + (r'(class)(\s+)([A-Z][a-zA-Z0-9_\']*)', + bygroups(Keyword, Whitespace, Name.Class)), + (r'class'+_nkw, Keyword), + (r'extends'+_nkw, Keyword), + (r'(def)(\s+)(self)(\s*)(\.)(\s*)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)', + bygroups(Keyword, Whitespace, Name.Builtin, Whitespace, Punctuation, Whitespace, Name.Function)), + (r'(def)(\s+)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)', + bygroups(Keyword, Whitespace, Name.Function)), + (r'def'+_nkw, Keyword), + (r'if'+_nkw, Keyword), + (r'elsif'+_nkw, Keyword), + (r'else'+_nkw, Keyword), + (r'unless'+_nkw, Keyword), + (r'for'+_nkw, Keyword), + (r'in'+_nkw, Keyword), + (r'while'+_nkw, Keyword), + (r'until'+_nkw, Keyword), + (r'and'+_nkw, Keyword), + (r'or'+_nkw, Keyword), + (r'not'+_nkw, Keyword), + (r'lambda'+_nkw, Keyword), + (r'try'+_nkw, Keyword), + (r'catch'+_nkw, Keyword), + (r'return'+_nkw, Keyword), + (r'next'+_nkw, Keyword), + (r'last'+_nkw, Keyword), + (r'throw'+_nkw, Keyword), + (r'use'+_nkw, Keyword), + (r'switch'+_nkw, Keyword), + (r'\\', Keyword), + (r'λ', Keyword), + (r'__FILE__'+_nkw, Name.Builtin.Pseudo), + (r'__LINE__'+_nkw, Name.Builtin.Pseudo), + (r'[A-Z][a-zA-Z0-9_\']*'+_nkw, Name.Constant), + (r'[a-z_][a-zA-Z0-9_\']*'+_nkw, Name), + (r'@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Instance), + (r'@@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Class), + (r'\(', Punctuation), + (r'\)', Punctuation), + (r'\[', Punctuation), + (r'\]', Punctuation), + (r'\{', Punctuation), + (r'\}', right_angle_bracket), + (r';', Punctuation), + (r',', Punctuation), + (r'<<=', Operator), + (r'>>=', Operator), + (r'<<', Operator), + (r'>>', Operator), + (r'==', Operator), + (r'!=', Operator), + (r'=>', Operator), + (r'=', Operator), + (r'<=>', Operator), + (r'<=', Operator), + (r'>=', Operator), + (r'<', Operator), + (r'>', Operator), + (r'\+\+', Operator), + (r'\+=', Operator), + (r'-=', Operator), + (r'\*\*=', Operator), + (r'\*=', Operator), + (r'\*\*', Operator), + (r'\*', Operator), + (r'/=', Operator), + (r'\+', Operator), + (r'-', Operator), + (r'/', Operator), + (r'%=', Operator), + (r'%', Operator), + (r'^=', Operator), + (r'&&=', Operator), + (r'&=', Operator), + (r'&&', Operator), + (r'&', Operator), + (r'\|\|=', Operator), + (r'\|=', Operator), + (r'\|\|', Operator), + (r'\|', Operator), + (r'!', Operator), + (r'\.\.\.', Operator), + (r'\.\.', Operator), + (r'\.', Operator), + (r'::', Operator), + (r':', Operator), + (r'(\s|\n)+', Whitespace), + (r'[a-z_][a-zA-Z0-9_\']*', Name.Variable), + ], + } + + +class SlashLexer(DelegatingLexer): + """ + Lexer for the Slash programming language. + + .. versionadded:: 2.4 + """ + + name = 'Slash' + aliases = ['slash'] + filenames = ['*.sl'] + + def __init__(self, **options): + from pygments.lexers.web import HtmlLexer + super(SlashLexer, self).__init__(HtmlLexer, SlashLanguageLexer, **options) diff --git a/pygments/lexers/smalltalk.py b/pygments/lexers/smalltalk.py index 79078b66..0e4584f3 100644 --- a/pygments/lexers/smalltalk.py +++ b/pygments/lexers/smalltalk.py @@ -5,7 +5,7 @@ Lexers for Smalltalk and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/smv.py b/pygments/lexers/smv.py index 380a3b70..2d97c2c5 100644 --- a/pygments/lexers/smv.py +++ b/pygments/lexers/smv.py @@ -5,7 +5,7 @@ Lexers for the SMV languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/snobol.py b/pygments/lexers/snobol.py index f6e12fd2..ce52f7c7 100644 --- a/pygments/lexers/snobol.py +++ b/pygments/lexers/snobol.py @@ -5,7 +5,7 @@ Lexers for the SNOBOL language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/special.py b/pygments/lexers/special.py index 6e076b0c..1b3e9724 100644 --- a/pygments/lexers/special.py +++ b/pygments/lexers/special.py @@ -5,7 +5,7 @@ Special lexers. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index 7507c0fc..d789052d 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -34,7 +34,7 @@ The ``tests/examplefiles`` contains a few test files with data to be parsed by these lexers. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -59,7 +59,14 @@ line_re = re.compile('.*?\n') language_re = re.compile(r"\s+LANGUAGE\s+'?(\w+)'?", re.IGNORECASE) -do_re = re.compile(r'\bDO\b', re.IGNORECASE) +do_re = re.compile(r'\bDO\b', re.IGNORECASE) + +# Regular expressions for analyse_text() +name_between_bracket_re = re.compile(r'\[[a-zA-Z_]\w*\]') +name_between_backtick_re = re.compile(r'`[a-zA-Z_]\w*`') +tsql_go_re = re.compile(r'\bgo\b', re.IGNORECASE) +tsql_declare_re = re.compile(r'\bdeclare\s+@', re.IGNORECASE) +tsql_variable_re = re.compile(r'@[a-zA-Z_]\w*\b') def language_callback(lexer, match): @@ -82,7 +89,7 @@ def language_callback(lexer, match): lexer.text[max(0, match.start()-25):match.start()])) if m: l = lexer._get_lexer('plpgsql') - + # 1 = $, 2 = delimiter, 3 = $ yield (match.start(1), String, match.group(1)) yield (match.start(2), String.Delimiter, match.group(2)) @@ -155,7 +162,7 @@ class PostgresLexer(PostgresBase, RegexLexer): (r'\s+', Text), (r'--.*\n?', Comment.Single), (r'/\*', Comment.Multiline, 'multiline-comments'), - (r'(' + '|'.join(s.replace(" ", "\s+") + (r'(' + '|'.join(s.replace(" ", r"\s+") for s in DATATYPES + PSEUDO_TYPES) + r')\b', Name.Builtin), (words(KEYWORDS, suffix=r'\b'), Keyword), @@ -308,14 +315,7 @@ class PostgresConsoleLexer(Lexer): # and continue until the end of command is detected curcode = '' insertions = [] - while 1: - try: - line = next(lines) - except StopIteration: - # allow the emission of partially collected items - # the repl loop will be broken below - break - + for line in lines: # Identify a shell prompt in case of psql commandline example if line.startswith('$') and not curcode: lexer = get_lexer_by_name('console', **self.options) @@ -346,8 +346,7 @@ class PostgresConsoleLexer(Lexer): # Emit the output lines out_token = Generic.Output - while 1: - line = next(lines) + for line in lines: mprompt = re_prompt.match(line) if mprompt is not None: # push the line back to have it processed by the prompt @@ -363,6 +362,8 @@ class PostgresConsoleLexer(Lexer): yield (mmsg.start(2), out_token, mmsg.group(2)) else: yield (0, out_token, line) + else: + return class SqlLexer(RegexLexer): @@ -480,6 +481,9 @@ class SqlLexer(RegexLexer): ] } + def analyse_text(text): + return 0.01 + class TransactSqlLexer(RegexLexer): """ @@ -499,7 +503,7 @@ class TransactSqlLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Whitespace), - (r'--(?m).*?$\n?', Comment.Single), + (r'(?m)--.*?$\n?', Comment.Single), (r'/\*', Comment.Multiline, 'multiline-comments'), (words(_tsql_builtins.OPERATORS), Operator), (words(_tsql_builtins.OPERATOR_WORDS, suffix=r'\b'), Operator.Word), @@ -536,6 +540,33 @@ class TransactSqlLexer(RegexLexer): ] } + def analyse_text(text): + rating = 0 + if tsql_declare_re.search(text): + # Found T-SQL variable declaration. + rating = 1.0 + else: + name_between_backtick_count = len( + name_between_backtick_re.findall((text))) + name_between_bracket_count = len( + name_between_bracket_re.findall(text)) + # We need to check if there are any names using + # backticks or brackets, as otherwise both are 0 + # and 0 >= 2 * 0, so we would always assume it's true + dialect_name_count = name_between_backtick_count + name_between_bracket_count + if dialect_name_count >= 1 and name_between_bracket_count >= 2 * name_between_backtick_count: + # Found at least twice as many [name] as `name`. + rating += 0.5 + elif name_between_bracket_count > name_between_backtick_count: + rating += 0.2 + elif name_between_bracket_count > 0: + rating += 0.1 + if tsql_variable_re.search(text) is not None: + rating += 0.1 + if tsql_go_re.search(text) is not None: + rating += 0.1 + return rating + class MySqlLexer(RegexLexer): """ @@ -609,6 +640,23 @@ class MySqlLexer(RegexLexer): ] } + def analyse_text(text): + rating = 0 + name_between_backtick_count = len( + name_between_backtick_re.findall((text))) + name_between_bracket_count = len( + name_between_bracket_re.findall(text)) + # Same logic as above in the TSQL analysis + dialect_name_count = name_between_backtick_count + name_between_bracket_count + if dialect_name_count >= 1 and name_between_backtick_count >= 2 * name_between_bracket_count: + # Found at least twice as many `name` as [name]. + rating += 0.5 + elif name_between_backtick_count > name_between_bracket_count: + rating += 0.2 + elif name_between_backtick_count > 0: + rating += 0.1 + return rating + class SqliteConsoleLexer(Lexer): """ diff --git a/pygments/lexers/stata.py b/pygments/lexers/stata.py index a015a23e..b7b22edb 100644 --- a/pygments/lexers/stata.py +++ b/pygments/lexers/stata.py @@ -5,10 +5,11 @@ Lexer for Stata - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ +import re from pygments.lexer import RegexLexer, include, words from pygments.token import Comment, Keyword, Name, Number, \ String, Text, Operator @@ -33,56 +34,118 @@ class StataLexer(RegexLexer): aliases = ['stata', 'do'] filenames = ['*.do', '*.ado'] mimetypes = ['text/x-stata', 'text/stata', 'application/x-stata'] + flags = re.MULTILINE | re.DOTALL tokens = { 'root': [ include('comments'), - include('vars-strings'), + include('strings'), + include('macros'), include('numbers'), include('keywords'), + include('operators'), + include('format'), (r'.', Text), ], - # Global and local macros; regular and special strings - 'vars-strings': [ - (r'\$[\w{]', Name.Variable.Global, 'var_validglobal'), - (r'`\w{0,31}\'', Name.Variable), - (r'"', String, 'string_dquote'), - (r'`"', String, 'string_mquote'), - ], - # For either string type, highlight macros as macros - 'string_dquote': [ - (r'"', String, '#pop'), - (r'\\\\|\\"|\\\n', String.Escape), - (r'\$', Name.Variable.Global, 'var_validglobal'), - (r'`', Name.Variable, 'var_validlocal'), - (r'[^$`"\\]+', String), - (r'[$"\\]', String), - ], - 'string_mquote': [ + # Comments are a complicated beast in Stata because they can be + # nested and there are a few corner cases with that. See: + # - github.com/kylebarron/language-stata/issues/90 + # - statalist.org/forums/forum/general-stata-discussion/general/1448244 + 'comments': [ + (r'(^//|(?<=\s)//)(?!/)', Comment.Single, 'comments-double-slash'), + (r'^\s*\*', Comment.Single, 'comments-star'), + (r'/\*', Comment.Multiline, 'comments-block'), + (r'(^///|(?<=\s)///)', Comment.Special, 'comments-triple-slash') + ], + 'comments-block': [ + (r'/\*', Comment.Multiline, '#push'), + # this ends and restarts a comment block. but need to catch this so + # that it doesn\'t start _another_ level of comment blocks + (r'\*/\*', Comment.Multiline), + (r'(\*/\s+\*(?!/)[^\n]*)|(\*/)', Comment.Multiline, '#pop'), + # Match anything else as a character inside the comment + (r'.', Comment.Multiline), + ], + 'comments-star': [ + (r'///.*?\n', Comment.Single, + ('#pop', 'comments-triple-slash')), + (r'(^//|(?<=\s)//)(?!/)', Comment.Single, + ('#pop', 'comments-double-slash')), + (r'/\*', Comment.Multiline, 'comments-block'), + (r'.(?=\n)', Comment.Single, '#pop'), + (r'.', Comment.Single), + ], + 'comments-triple-slash': [ + (r'\n', Comment.Special, '#pop'), + # A // breaks out of a comment for the rest of the line + (r'//.*?(?=\n)', Comment.Single, '#pop'), + (r'.', Comment.Special), + ], + 'comments-double-slash': [ + (r'\n', Text, '#pop'), + (r'.', Comment.Single), + ], + # `"compound string"' and regular "string"; note the former are + # nested. + 'strings': [ + (r'`"', String, 'string-compound'), + (r'(?<!`)"', String, 'string-regular'), + ], + 'string-compound': [ + (r'`"', String, '#push'), (r'"\'', String, '#pop'), - (r'\\\\|\\"|\\\n', String.Escape), - (r'\$', Name.Variable.Global, 'var_validglobal'), - (r'`', Name.Variable, 'var_validlocal'), - (r'[^$`"\\]+', String), - (r'[$"\\]', String), - ], - 'var_validglobal': [ - (r'\{\w{0,32}\}', Name.Variable.Global, '#pop'), - (r'\w{1,32}', Name.Variable.Global, '#pop'), + (r'\\\\|\\"|\\\$|\\`|\\\n', String.Escape), + include('macros'), + (r'.', String) ], - 'var_validlocal': [ - (r'\w{0,31}\'', Name.Variable, '#pop'), + 'string-regular': [ + (r'(")(?!\')|(?=\n)', String, '#pop'), + (r'\\\\|\\"|\\\$|\\`|\\\n', String.Escape), + include('macros'), + (r'.', String) ], - # * only OK at line start, // OK anywhere - 'comments': [ - (r'^\s*\*.*$', Comment), - (r'//.*', Comment.Single), - (r'/\*.*?\*/', Comment.Multiline), - (r'/[*](.|\n)*?[*]/', Comment.Multiline), + # A local is usually + # `\w{0,31}' + # `:extended macro' + # `=expression' + # `[rsen](results)' + # `(++--)scalar(++--)' + # + # However, there are all sorts of weird rules wrt edge + # cases. Instead of writing 27 exceptions, anything inside + # `' is a local. + # + # A global is more restricted, so we do follow rules. Note only + # locals explicitly enclosed ${} can be nested. + 'macros': [ + (r'\$(\{|(?=[\$`]))', Name.Variable.Global, 'macro-global-nested'), + (r'\$', Name.Variable.Global, 'macro-global-name'), + (r'`', Name.Variable, 'macro-local'), + ], + 'macro-local': [ + (r'`', Name.Variable, '#push'), + (r"'", Name.Variable, '#pop'), + (r'\$(\{|(?=[\$`]))', Name.Variable.Global, 'macro-global-nested'), + (r'\$', Name.Variable.Global, 'macro-global-name'), + (r'.', Name.Variable), # fallback + ], + 'macro-global-nested': [ + (r'\$(\{|(?=[\$`]))', Name.Variable.Global, '#push'), + (r'\}', Name.Variable.Global, '#pop'), + (r'\$', Name.Variable.Global, 'macro-global-name'), + (r'`', Name.Variable, 'macro-local'), + (r'\w', Name.Variable.Global), # fallback + (r'(?!\w)', Name.Variable.Global, '#pop'), + ], + 'macro-global-name': [ + (r'\$(\{|(?=[\$`]))', Name.Variable.Global, 'macro-global-nested', '#pop'), + (r'\$', Name.Variable.Global, 'macro-global-name', '#pop'), + (r'`', Name.Variable, 'macro-local', '#pop'), + (r'\w{1,32}', Name.Variable.Global, '#pop'), ], # Built in functions and statements 'keywords': [ - (words(builtins_functions, prefix = r'\b', suffix = r'\('), + (words(builtins_functions, prefix = r'\b', suffix = r'(?=\()'), Name.Function), (words(builtins_base, prefix = r'(^\s*|\s)', suffix = r'\b'), Keyword), @@ -100,9 +163,9 @@ class StataLexer(RegexLexer): ], # Stata formats 'format': [ - (r'%-?\d{1,2}(\.\d{1,2})?[gfe]c?', Name.Variable), - (r'%(21x|16H|16L|8H|8L)', Name.Variable), - (r'%-?(tc|tC|td|tw|tm|tq|th|ty|tg).{0,32}', Name.Variable), - (r'%[-~]?\d{1,4}s', Name.Variable), + (r'%-?\d{1,2}(\.\d{1,2})?[gfe]c?', Name.Other), + (r'%(21x|16H|16L|8H|8L)', Name.Other), + (r'%-?(tc|tC|td|tw|tm|tq|th|ty|tg)\S{0,32}', Name.Other), + (r'%[-~]?\d{1,4}s', Name.Other), ] } diff --git a/pygments/lexers/supercollider.py b/pygments/lexers/supercollider.py index 40ff0aeb..d0d033a0 100644 --- a/pygments/lexers/supercollider.py +++ b/pygments/lexers/supercollider.py @@ -5,7 +5,7 @@ Lexer for SuperCollider - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/tcl.py b/pygments/lexers/tcl.py index 1d1be033..ea37c824 100644 --- a/pygments/lexers/tcl.py +++ b/pygments/lexers/tcl.py @@ -5,7 +5,7 @@ Lexers for Tcl and related languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py index 5e5677fc..498613e0 100644 --- a/pygments/lexers/templates.py +++ b/pygments/lexers/templates.py @@ -5,7 +5,7 @@ Lexers for various template engines' markup. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -187,13 +187,13 @@ class SmartyLexer(RegexLexer): def analyse_text(text): rv = 0.0 - if re.search('\{if\s+.*?\}.*?\{/if\}', text): + if re.search(r'\{if\s+.*?\}.*?\{/if\}', text): rv += 0.15 - if re.search('\{include\s+file=.*?\}', text): + if re.search(r'\{include\s+file=.*?\}', text): rv += 0.15 - if re.search('\{foreach\s+.*?\}.*?\{/foreach\}', text): + if re.search(r'\{foreach\s+.*?\}.*?\{/foreach\}', text): rv += 0.15 - if re.search('\{\$.*?\}', text): + if re.search(r'\{\$.*?\}', text): rv += 0.01 return rv @@ -375,7 +375,7 @@ class DjangoLexer(RegexLexer): (r'\.\w+', Name.Variable), (r':?"(\\\\|\\"|[^"])*"', String.Double), (r":?'(\\\\|\\'|[^'])*'", String.Single), - (r'([{}()\[\]+\-*/,:~]|[><=]=?)', Operator), + (r'([{}()\[\]+\-*/%,:~]|[><=]=?|!=)', Operator), (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), ], @@ -421,18 +421,18 @@ class MyghtyLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (r'(<%(?:def|method))(\s*)(.*?)(>)(.*?)(</%\2\s*>)(?s)', + (r'(?s)(<%(?:def|method))(\s*)(.*?)(>)(.*?)(</%\2\s*>)', bygroups(Name.Tag, Text, Name.Function, Name.Tag, using(this), Name.Tag)), - (r'(<%\w+)(.*?)(>)(.*?)(</%\2\s*>)(?s)', + (r'(?s)(<%\w+)(.*?)(>)(.*?)(</%\2\s*>)', bygroups(Name.Tag, Name.Function, Name.Tag, using(PythonLexer), Name.Tag)), (r'(<&[^|])(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PythonLexer), Name.Tag)), - (r'(<&\|)(.*?)(,.*?)?(&>)(?s)', + (r'(?s)(<&\|)(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PythonLexer), Name.Tag)), (r'</&>', Name.Tag), - (r'(<%!?)(.*?)(%>)(?s)', + (r'(?s)(<%!?)(.*?)(%>)', bygroups(Name.Tag, using(PythonLexer), Name.Tag)), (r'(?<=^)#[^\n]*(\n|\Z)', Comment), (r'(?<=^)(%)([^\n]*)(\n|\Z)', @@ -538,20 +538,20 @@ class MasonLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (r'(<%doc>)(.*?)(</%doc>)(?s)', + (r'(?s)(<%doc>)(.*?)(</%doc>)', bygroups(Name.Tag, Comment.Multiline, Name.Tag)), - (r'(<%(?:def|method))(\s*)(.*?)(>)(.*?)(</%\2\s*>)(?s)', + (r'(?s)(<%(?:def|method))(\s*)(.*?)(>)(.*?)(</%\2\s*>)', bygroups(Name.Tag, Text, Name.Function, Name.Tag, using(this), Name.Tag)), - (r'(<%\w+)(.*?)(>)(.*?)(</%\2\s*>)(?s)', + (r'(?s)(<%\w+)(.*?)(>)(.*?)(</%\2\s*>)', bygroups(Name.Tag, Name.Function, Name.Tag, using(PerlLexer), Name.Tag)), - (r'(<&[^|])(.*?)(,.*?)?(&>)(?s)', + (r'(?s)(<&[^|])(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PerlLexer), Name.Tag)), - (r'(<&\|)(.*?)(,.*?)?(&>)(?s)', + (r'(?s)(<&\|)(.*?)(,.*?)?(&>)', bygroups(Name.Tag, Name.Function, using(PerlLexer), Name.Tag)), (r'</&>', Name.Tag), - (r'(<%!?)(.*?)(%>)(?s)', + (r'(?s)(<%!?)(.*?)(%>)', bygroups(Name.Tag, using(PerlLexer), Name.Tag)), (r'(?<=^)#[^\n]*(\n|\Z)', Comment), (r'(?<=^)(%)([^\n]*)(\n|\Z)', @@ -607,7 +607,7 @@ class MakoLexer(RegexLexer): (r'(</%)([\w.:]+)(>)', bygroups(Comment.Preproc, Name.Builtin, Comment.Preproc)), (r'<%(?=([\w.:]+))', Comment.Preproc, 'ondeftags'), - (r'(<%(?:!?))(.*?)(%>)(?s)', + (r'(?s)(<%(?:!?))(.*?)(%>)', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), (r'(\$\{)(.*?)(\})', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), @@ -759,7 +759,7 @@ class CheetahLexer(RegexLexer): # TODO support other Python syntax like $foo['bar'] (r'(\$)([a-zA-Z_][\w.]*\w)', bygroups(Comment.Preproc, using(CheetahPythonLexer))), - (r'(\$\{!?)(.*?)(\})(?s)', + (r'(?s)(\$\{!?)(.*?)(\})', bygroups(Comment.Preproc, using(CheetahPythonLexer), Comment.Preproc)), (r'''(?sx) @@ -942,9 +942,9 @@ class HtmlGenshiLexer(DelegatingLexer): def analyse_text(text): rv = 0.0 - if re.search('\$\{.*?\}', text) is not None: + if re.search(r'\$\{.*?\}', text) is not None: rv += 0.2 - if re.search('py:(.*?)=["\']', text) is not None: + if re.search(r'py:(.*?)=["\']', text) is not None: rv += 0.2 return rv + HtmlLexer.analyse_text(text) - 0.01 @@ -967,9 +967,9 @@ class GenshiLexer(DelegatingLexer): def analyse_text(text): rv = 0.0 - if re.search('\$\{.*?\}', text) is not None: + if re.search(r'\$\{.*?\}', text) is not None: rv += 0.2 - if re.search('py:(.*?)=["\']', text) is not None: + if re.search(r'py:(.*?)=["\']', text) is not None: rv += 0.2 return rv + XmlLexer.analyse_text(text) - 0.01 @@ -1627,7 +1627,7 @@ class SspLexer(DelegatingLexer): def analyse_text(text): rv = 0.0 - if re.search('val \w+\s*:', text): + if re.search(r'val \w+\s*:', text): rv += 0.6 if looks_like_xml(text): rv += 0.2 @@ -1954,7 +1954,7 @@ class LiquidLexer(RegexLexer): 'output': [ include('whitespace'), - ('\}\}', Punctuation, '#pop'), # end of output + (r'\}\}', Punctuation, '#pop'), # end of output (r'\|', Punctuation, 'filters') ], diff --git a/pygments/lexers/teraterm.py b/pygments/lexers/teraterm.py new file mode 100644 index 00000000..0fd1778a --- /dev/null +++ b/pygments/lexers/teraterm.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.teraterm + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Tera Term macro files. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, bygroups +from pygments.token import Text, Comment, Operator, Name, String, \ + Number, Keyword + +__all__ = ['TeraTermLexer'] + + +class TeraTermLexer(RegexLexer): + """ + For `Tera Term <https://ttssh2.osdn.jp/>`_ macro source code. + + .. versionadded:: 2.4 + """ + name = 'Tera Term macro' + aliases = ['ttl', 'teraterm', 'teratermmacro'] + filenames = ['*.ttl'] + mimetypes = ['text/x-teratermmacro'] + + tokens = { + 'root': [ + include('comments'), + include('labels'), + include('commands'), + include('builtin-variables'), + include('user-variables'), + include('operators'), + include('numeric-literals'), + include('string-literals'), + include('all-whitespace'), + (r'[^\s]', Text), + ], + 'comments': [ + (r';[^\r\n]*', Comment.Single), + (r'/\*', Comment.Multiline, 'in-comment'), + ], + 'in-comment': [ + (r'\*/', Comment.Multiline, '#pop'), + (r'[^*/]+', Comment.Multiline), + (r'[*/]', Comment.Multiline) + ], + 'labels': [ + (r'(?i)^(\s*)(:[0-9a-z_]+)', bygroups(Text, Name.Label)), + ], + 'commands': [ + ( + r'(?i)\b(' + r'basename|beep|bplusrecv|bplussend|break|bringupbox|' + r'callmenu|changedir|checksum16|checksum16file|' + r'checksum32|checksum32file|checksum8|checksum8file|' + r'clearscreen|clipb2var|closesbox|closett|code2str|' + r'connect|continue|crc16|crc16file|crc32|crc32file|' + r'cygconnect|delpassword|dirname|dirnamebox|disconnect|' + r'dispstr|do|else|elseif|enablekeyb|end|endif|enduntil|' + r'endwhile|exec|execcmnd|exit|expandenv|fileclose|' + r'fileconcat|filecopy|filecreate|filedelete|filelock|' + r'filemarkptr|filenamebox|fileopen|fileread|filereadln|' + r'filerename|filesearch|fileseek|fileseekback|filestat|' + r'filestrseek|filestrseek2|filetruncate|fileunlock|' + r'filewrite|filewriteln|findclose|findfirst|findnext|' + r'flushrecv|foldercreate|folderdelete|foldersearch|for|' + r'getdate|getdir|getenv|getfileattr|gethostname|' + r'getipv4addr|getipv6addr|getmodemstatus|getpassword|' + r'getspecialfolder|gettime|gettitle|getttdir|getver|' + r'if|ifdefined|include|inputbox|int2str|intdim|' + r'ispassword|kmtfinish|kmtget|kmtrecv|kmtsend|listbox|' + r'loadkeymap|logautoclosemode|logclose|loginfo|logopen|' + r'logpause|logrotate|logstart|logwrite|loop|makepath|' + r'messagebox|mpause|next|passwordbox|pause|quickvanrecv|' + r'quickvansend|random|recvln|regexoption|restoresetup|' + r'return|rotateleft|rotateright|scprecv|scpsend|send|' + r'sendbreak|sendbroadcast|sendfile|sendkcode|sendln|' + r'sendlnbroadcast|sendlnmulticast|sendmulticast|setbaud|' + r'setdate|setdebug|setdir|setdlgpos|setdtr|setecho|' + r'setenv|setexitcode|setfileattr|setflowctrl|' + r'setmulticastname|setpassword|setrts|setsync|settime|' + r'settitle|show|showtt|sprintf|sprintf2|statusbox|' + r'str2code|str2int|strcompare|strconcat|strcopy|strdim|' + r'strinsert|strjoin|strlen|strmatch|strremove|' + r'strreplace|strscan|strspecial|strsplit|strtrim|' + r'testlink|then|tolower|toupper|unlink|until|uptime|' + r'var2clipb|wait|wait4all|waitevent|waitln|waitn|' + r'waitrecv|waitregex|while|xmodemrecv|xmodemsend|' + r'yesnobox|ymodemrecv|ymodemsend|zmodemrecv|zmodemsend' + r')\b', + Keyword, + ), + ( + r'(?i)(call|goto)([ \t]+)([0-9a-z_]+)', + bygroups(Keyword, Text, Name.Label), + ) + ], + 'builtin-variables': [ + ( + r'(?i)(' + r'groupmatchstr1|groupmatchstr2|groupmatchstr3|' + r'groupmatchstr4|groupmatchstr5|groupmatchstr6|' + r'groupmatchstr7|groupmatchstr8|groupmatchstr9|' + r'param1|param2|param3|param4|param5|param6|' + r'param7|param8|param9|paramcnt|params|' + r'inputstr|matchstr|mtimeout|result|timeout' + r')\b', + Name.Builtin + ), + ], + 'user-variables': [ + (r'(?i)[A-Z_][A-Z0-9_]*', Name.Variable), + ], + 'numeric-literals': [ + (r'(-?)([0-9]+)', bygroups(Operator, Number.Integer)), + (r'(?i)\$[0-9a-f]+', Number.Hex), + ], + 'string-literals': [ + (r'(?i)#(?:[0-9]+|\$[0-9a-f]+)', String.Char), + (r"'", String.Single, 'in-single-string'), + (r'"', String.Double, 'in-double-string'), + ], + 'in-general-string': [ + (r'[\\][\\nt]', String.Escape), # Only three escapes are supported. + (r'.', String), + ], + 'in-single-string': [ + (r"'", String.Single, '#pop'), + include('in-general-string'), + ], + 'in-double-string': [ + (r'"', String.Double, '#pop'), + include('in-general-string'), + ], + 'operators': [ + (r'and|not|or|xor', Operator.Word), + (r'[!%&*+<=>^~\|\/-]+', Operator), + (r'[()]', String.Symbol), + ], + 'all-whitespace': [ + (r'[\s]+', Text), + ], + } + + # Turtle and Tera Term macro files share the same file extension + # but each has a recognizable and distinct syntax. + def analyse_text(text): + result = 0.0 + if re.search(TeraTermLexer.tokens['commands'][0][0], text): + result += 0.60 + return result diff --git a/pygments/lexers/testing.py b/pygments/lexers/testing.py index 1e0795b1..9288b5b4 100644 --- a/pygments/lexers/testing.py +++ b/pygments/lexers/testing.py @@ -5,7 +5,7 @@ Lexers for testing languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -29,7 +29,7 @@ class GherkinLexer(RegexLexer): feature_keywords = u'^(기능|機能|功能|フィーチャ|خاصية|תכונה|Функціонал|Функционалност|Функционал|Фича|Особина|Могућност|Özellik|Właściwość|Tính năng|Trajto|Savybė|Požiadavka|Požadavek|Osobina|Ominaisuus|Omadus|OH HAI|Mogućnost|Mogucnost|Jellemző|Fīča|Funzionalità|Funktionalität|Funkcionalnost|Funkcionalitāte|Funcționalitate|Functionaliteit|Functionalitate|Funcionalitat|Funcionalidade|Fonctionnalité|Fitur|Feature|Egenskap|Egenskab|Crikey|Característica|Arwedd)(:)(.*)$' feature_element_keywords = u'^(\\s*)(시나리오 개요|시나리오|배경|背景|場景大綱|場景|场景大纲|场景|劇本大綱|劇本|剧本大纲|剧本|テンプレ|シナリオテンプレート|シナリオテンプレ|シナリオアウトライン|シナリオ|سيناريو مخطط|سيناريو|الخلفية|תרחיש|תבנית תרחיש|רקע|Тарих|Сценарій|Сценарио|Сценарий структураси|Сценарий|Структура сценарію|Структура сценарија|Структура сценария|Скица|Рамка на сценарий|Пример|Предыстория|Предистория|Позадина|Передумова|Основа|Концепт|Контекст|Założenia|Wharrimean is|Tình huống|The thing of it is|Tausta|Taust|Tapausaihio|Tapaus|Szenariogrundriss|Szenario|Szablon scenariusza|Stsenaarium|Struktura scenarija|Skica|Skenario konsep|Skenario|Situācija|Senaryo taslağı|Senaryo|Scénář|Scénario|Schema dello scenario|Scenārijs pēc parauga|Scenārijs|Scenár|Scenaro|Scenariusz|Scenariul de şablon|Scenariul de sablon|Scenariu|Scenario Outline|Scenario Amlinellol|Scenario|Scenarijus|Scenarijaus šablonas|Scenarij|Scenarie|Rerefons|Raamstsenaarium|Primer|Pozadí|Pozadina|Pozadie|Plan du scénario|Plan du Scénario|Osnova scénáře|Osnova|Náčrt Scénáře|Náčrt Scenáru|Mate|MISHUN SRSLY|MISHUN|Kịch bản|Konturo de la scenaro|Kontext|Konteksts|Kontekstas|Kontekst|Koncept|Khung tình huống|Khung kịch bản|Háttér|Grundlage|Geçmiş|Forgatókönyv vázlat|Forgatókönyv|Fono|Esquema do Cenário|Esquema do Cenario|Esquema del escenario|Esquema de l\'escenari|Escenario|Escenari|Dis is what went down|Dasar|Contexto|Contexte|Contesto|Condiţii|Conditii|Cenário|Cenario|Cefndir|Bối cảnh|Blokes|Bakgrunn|Bakgrund|Baggrund|Background|B4|Antecedents|Antecedentes|All y\'all|Achtergrond|Abstrakt Scenario|Abstract Scenario)(:)(.*)$' examples_keywords = u'^(\\s*)(예|例子|例|サンプル|امثلة|דוגמאות|Сценарији|Примери|Приклади|Мисоллар|Значения|Örnekler|Voorbeelden|Variantai|Tapaukset|Scenarios|Scenariji|Scenarijai|Příklady|Példák|Príklady|Przykłady|Primjeri|Primeri|Piemēri|Pavyzdžiai|Paraugs|Juhtumid|Exemplos|Exemples|Exemplele|Exempel|Examples|Esempi|Enghreifftiau|Ekzemploj|Eksempler|Ejemplos|EXAMPLZ|Dữ liệu|Contoh|Cobber|Beispiele)(:)(.*)$' - step_keywords = u'^(\\s*)(하지만|조건|먼저|만일|만약|단|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假设|假如|假定|但是|但し|並且|并且|同時|同时|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Якщо |Унда |То |Припустимо, що |Припустимо |Онда |Но |Нехай |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Але |Агар |А |І |Și |És |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Youse know when youse got |Youse know like when |Yna |Ya know how |Ya gotta |Y |Wun |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Stel |Soit |Siis |Si |Sed |Se |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kaj |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givun |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Dun |Donitaĵo |Donat |Donada |Do |Diyelim ki |Dengan |Den youse gotta |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |Cal |But y\'all |But |Buh |Biết |Bet |BUT |Atès |Atunci |Atesa |Anrhegedig a |Angenommen |And y\'all |And |An |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\* )' + step_keywords = u'^(\\s*)(하지만|조건|먼저|만일|만약|단|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假设|假如|假定|但是|但し|並且|并且|同時|同时|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Якщо |Унда |То |Припустимо, що |Припустимо |Онда |Но |Нехай |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Але |Агар |А |І |Și |És |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Youse know when youse got |Youse know like when |Yna |Ya know how |Ya gotta |Y |Wun |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Stel |Soit |Siis |Si |Sed |Se |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kaj |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givun |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Dun |Donitaĵo |Donat |Donada |Do |Diyelim ki |Dengan |Den youse gotta |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |Cal |But y\'all |But |Buh |Biết |Bet |BUT |Atès |Atunci |Atesa |Anrhegedig a |Angenommen |And y\'all |And |An |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\\* )' tokens = { 'comments': [ diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py index 9b3b5fea..6b431f69 100644 --- a/pygments/lexers/text.py +++ b/pygments/lexers/text.py @@ -5,7 +5,7 @@ Lexers for non-source code file types. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -18,6 +18,7 @@ from pygments.lexers.markup import BBCodeLexer, MoinWikiLexer, RstLexer, \ from pygments.lexers.installers import DebianControlLexer, SourcesListLexer from pygments.lexers.make import MakefileLexer, BaseMakefileLexer, CMakeLexer from pygments.lexers.haxe import HxmlLexer +from pygments.lexers.sgf import SmartGameFormatLexer from pygments.lexers.diff import DiffLexer, DarcsPatchLexer from pygments.lexers.data import YamlLexer from pygments.lexers.textfmts import IrcLogsLexer, GettextLexer, HttpLexer diff --git a/pygments/lexers/textedit.py b/pygments/lexers/textedit.py index e8856dbd..66b36924 100644 --- a/pygments/lexers/textedit.py +++ b/pygments/lexers/textedit.py @@ -5,7 +5,7 @@ Lexers for languages related to text processing. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/textfmts.py b/pygments/lexers/textfmts.py index bb8124ef..a3aed0c0 100644 --- a/pygments/lexers/textfmts.py +++ b/pygments/lexers/textfmts.py @@ -5,7 +5,7 @@ Lexers for various text formats. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -266,7 +266,7 @@ class TodotxtLexer(RegexLexer): # 5. Leading project (project_regex, Project, 'incomplete'), # 6. Non-whitespace catch-all - ('\S+', IncompleteTaskText, 'incomplete'), + (r'\S+', IncompleteTaskText, 'incomplete'), ], # Parse a complete task @@ -277,9 +277,9 @@ class TodotxtLexer(RegexLexer): (context_regex, Context), (project_regex, Project), # Tokenize non-whitespace text - ('\S+', CompleteTaskText), + (r'\S+', CompleteTaskText), # Tokenize whitespace not containing a newline - ('\s+', CompleteTaskText), + (r'\s+', CompleteTaskText), ], # Parse an incomplete task @@ -290,8 +290,8 @@ class TodotxtLexer(RegexLexer): (context_regex, Context), (project_regex, Project), # Tokenize non-whitespace text - ('\S+', IncompleteTaskText), + (r'\S+', IncompleteTaskText), # Tokenize whitespace not containing a newline - ('\s+', IncompleteTaskText), + (r'\s+', IncompleteTaskText), ], } diff --git a/pygments/lexers/theorem.py b/pygments/lexers/theorem.py index e84a398b..a26a173f 100644 --- a/pygments/lexers/theorem.py +++ b/pygments/lexers/theorem.py @@ -5,7 +5,7 @@ Lexers for theorem-proving languages. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -98,7 +98,6 @@ class CoqLexer(RegexLexer): operators = r'[!$%&*+\./:<=>?@^|~-]' prefix_syms = r'[!?~]' infix_syms = r'[=<>@^|&+\*/$%-]' - primitives = ('unit', 'nat', 'bool', 'string', 'ascii', 'list') tokens = { 'root': [ @@ -115,7 +114,6 @@ class CoqLexer(RegexLexer): (r'\b([A-Z][\w\']*)', Name), (r'(%s)' % '|'.join(keyopts[::-1]), Operator), (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator), - (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type), (r"[^\W\d][\w']*", Name), diff --git a/pygments/lexers/trafficscript.py b/pygments/lexers/trafficscript.py index 42542280..9b767253 100644 --- a/pygments/lexers/trafficscript.py +++ b/pygments/lexers/trafficscript.py @@ -5,7 +5,7 @@ Lexer for RiverBed's TrafficScript (RTS) language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/typoscript.py b/pygments/lexers/typoscript.py index e358af07..745292bd 100644 --- a/pygments/lexers/typoscript.py +++ b/pygments/lexers/typoscript.py @@ -14,7 +14,7 @@ `TypoScriptHtmlDataLexer` Lexer that highlights markers, constants and registers within html tags. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -108,14 +108,11 @@ class TypoScriptLexer(RegexLexer): name = 'TypoScript' aliases = ['typoscript'] - filenames = ['*.ts', '*.txt'] + filenames = ['*.typoscript'] mimetypes = ['text/x-typoscript'] flags = re.DOTALL | re.MULTILINE - # Slightly higher than TypeScript (which is 0). - priority = 0.1 - tokens = { 'root': [ include('comment'), @@ -132,7 +129,7 @@ class TypoScriptLexer(RegexLexer): ], 'keywords': [ # Conditions - (r'(\[)(?i)(browser|compatVersion|dayofmonth|dayofweek|dayofyear|' + (r'(?i)(\[)(browser|compatVersion|dayofmonth|dayofweek|dayofyear|' r'device|ELSE|END|GLOBAL|globalString|globalVar|hostname|hour|IP|' r'language|loginUser|loginuser|minute|month|page|PIDinRootline|' r'PIDupinRootline|system|treeLevel|useragent|userFunc|usergroup|' @@ -172,7 +169,7 @@ class TypoScriptLexer(RegexLexer): 'html': [ (r'<\S[^\n>]*>', using(TypoScriptHtmlDataLexer)), (r'&[^;\n]*;', String), - (r'(_CSS_DEFAULT_STYLE)(\s*)(\()(?s)(.*(?=\n\)))', + (r'(?s)(_CSS_DEFAULT_STYLE)(\s*)(\()(.*(?=\n\)))', bygroups(Name.Class, Text, String.Symbol, using(TypoScriptCssDataLexer))), ], 'literal': [ @@ -220,7 +217,3 @@ class TypoScriptLexer(RegexLexer): (r'[\w"\-!/&;]+', Text), ], } - - def analyse_text(text): - if '<INCLUDE_TYPOSCRIPT:' in text: - return 1.0 diff --git a/pygments/lexers/unicon.py b/pygments/lexers/unicon.py new file mode 100644 index 00000000..b67c6895 --- /dev/null +++ b/pygments/lexers/unicon.py @@ -0,0 +1,390 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.unicon + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for the Icon and Unicon languages, including ucode VM. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, bygroups, words, using, this +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['IconLexer', 'UcodeLexer', 'UniconLexer'] + + +class UniconLexer(RegexLexer): + """ + For Unicon source code. + + .. versionadded:: 2.4 + """ + + name = 'Unicon' + aliases = ['unicon'] + filenames = ['*.icn'] + mimetypes = ['text/unicon'] + + flags = re.MULTILINE + + tokens = { + 'root': [ + (r'[^\S\n]+', Text), + (r'#.*?\n', Comment.Single), + (r'[^\S\n]+', Text), + (r'class|method|procedure', Keyword.Declaration, 'subprogram'), + (r'(record)(\s+)(\w+)', + bygroups(Keyword.Declaration, Text, Keyword.Type), 'type_def'), + (r'(#line|\$C|\$Cend|\$define|\$else|\$endif|\$error|\$ifdef|' + r'\$ifndef|\$include|\$line|\$undef)\b', Keyword.PreProc), + (r'(&null|&fail)\b', Keyword.Constant), + (r'&allocated|&ascii|&clock|&collections|&column|&col|&control|' + r'&cset|¤t|&dateline|&date|&digits|&dump|' + r'&errno|&errornumber|&errortext|&errorvalue|&error|&errout|' + r'&eventcode|&eventvalue|&eventsource|&e|' + r'&features|&file|&host|&input|&interval|&lcase|&letters|' + r'&level|&line|&ldrag|&lpress|&lrelease|' + r'&main|&mdrag|&meta|&mpress|&mrelease|&now|&output|' + r'&phi|&pick|&pi|&pos|&progname|' + r'&random|&rdrag|®ions|&resize|&row|&rpress|&rrelease|' + r'&shift|&source|&storage|&subject|' + r'&time|&trace|&ucase|&version|' + r'&window|&x|&y', Keyword.Reserved), + (r'(by|of|not|to)\b', Keyword.Reserved), + (r'(global|local|static|abstract)\b', Keyword.Reserved), + (r'package|link|import', Keyword.Declaration), + (words(( + 'break', 'case', 'create', 'critical', 'default', 'end', 'all', + 'do', 'else', 'every', 'fail', 'if', 'import', 'initial', + 'initially', 'invocable', 'next', + 'repeat', 'return', 'suspend', + 'then', 'thread', 'until', 'while'), prefix=r'\b', suffix=r'\b'), + Keyword.Reserved), + (words(( + 'Abort', 'abs', 'acos', 'Active', 'Alert', 'any', 'Any', 'Arb', + 'Arbno', 'args', 'array', 'asin', 'atan', 'atanh', 'Attrib', + 'Bal', 'bal', 'Bg', 'Break', 'Breakx', + 'callout', 'center', 'char', 'chdir', 'chmod', 'chown', 'chroot', + 'classname', 'Clip', 'Clone', 'close', 'cofail', 'collect', + 'Color', 'ColorValue', 'condvar', 'constructor', 'copy', + 'CopyArea', 'cos', 'Couple', 'crypt', 'cset', 'ctime', + 'dbcolumns', 'dbdriver', 'dbkeys', 'dblimits', 'dbproduct', + 'dbtables', 'delay', 'delete', 'detab', 'display', 'DrawArc', + 'DrawCircle', 'DrawCube', 'DrawCurve', 'DrawCylinder', + 'DrawDisk', 'DrawImage', 'DrawLine', 'DrawPoint', 'DrawPolygon', + 'DrawRectangle', 'DrawSegment', 'DrawSphere', 'DrawString', + 'DrawTorus', 'dtor', + 'entab', 'EraseArea', 'errorclear', 'Event', 'eventmask', + 'EvGet', 'EvSend', 'exec', 'exit', 'exp', 'Eye', + 'Fail', 'fcntl', 'fdup', 'Fence', 'fetch', 'Fg', 'fieldnames', + 'filepair', 'FillArc', 'FillCircle', 'FillPolygon', + 'FillRectangle', 'find', 'flock', 'flush', 'Font', 'fork', + 'FreeColor', 'FreeSpace', 'function', + 'get', 'getch', 'getche', 'getegid', 'getenv', 'geteuid', + 'getgid', 'getgr', 'gethost', 'getpgrp', 'getpid', 'getppid', + 'getpw', 'getrusage', 'getserv', 'GetSpace', 'gettimeofday', + 'getuid', 'globalnames', 'GotoRC', 'GotoXY', 'gtime', 'hardlink', + 'iand', 'icom', 'IdentityMatrix', 'image', 'InPort', 'insert', + 'Int86', 'integer', 'ioctl', 'ior', 'ishift', 'istate', 'ixor', + 'kbhit', 'key', 'keyword', 'kill', + 'left', 'Len', 'list', 'load', 'loadfunc', 'localnames', + 'lock', 'log', 'Lower', 'lstat', + 'many', 'map', 'match', 'MatrixMode', 'max', 'member', + 'membernames', 'methodnames', 'methods', 'min', 'mkdir', 'move', + 'MultMatrix', 'mutex', + 'name', 'NewColor', 'Normals', 'NotAny', 'numeric', + 'open', 'opencl', 'oprec', 'ord', 'OutPort', + 'PaletteChars', 'PaletteColor', 'PaletteKey', 'paramnames', + 'parent', 'Pattern', 'Peek', 'Pending', 'pipe', 'Pixel', + 'PlayAudio', 'Poke', 'pop', 'PopMatrix', 'Pos', 'pos', + 'proc', 'pull', 'push', 'PushMatrix', 'PushRotate', 'PushScale', + 'PushTranslate', 'put', + 'QueryPointer', + 'Raise', 'read', 'ReadImage', 'readlink', 'reads', 'ready', + 'real', 'receive', 'Refresh', 'Rem', 'remove', 'rename', + 'repl', 'reverse', 'right', 'rmdir', 'Rotate', 'Rpos', + 'Rtab', 'rtod', 'runerr', + 'save', 'Scale', 'seek', 'select', 'send', 'seq', + 'serial', 'set', 'setenv', 'setgid', 'setgrent', + 'sethostent', 'setpgrp', 'setpwent', 'setservent', + 'setuid', 'signal', 'sin', 'sort', 'sortf', 'Span', + 'spawn', 'sql', 'sqrt', 'stat', 'staticnames', 'stop', + 'StopAudio', 'string', 'structure', 'Succeed', 'Swi', + 'symlink', 'sys_errstr', 'system', 'syswrite', + 'Tab', 'tab', 'table', 'tan', + 'Texcoord', 'Texture', 'TextWidth', 'Translate', + 'trap', 'trim', 'truncate', 'trylock', 'type', + 'umask', 'Uncouple', 'unlock', 'upto', 'utime', + 'variable', 'VAttrib', + 'wait', 'WAttrib', 'WDefault', 'WFlush', 'where', + 'WinAssociate', 'WinButton', 'WinColorDialog', 'WindowContents', + 'WinEditRegion', 'WinFontDialog', 'WinMenuBar', 'WinOpenDialog', + 'WinPlayMedia', 'WinSaveDialog', 'WinScrollBar', 'WinSelectDialog', + 'write', 'WriteImage', 'writes', 'WSection', + 'WSync'), prefix=r'\b', suffix=r'\b'), + Name.Function), + include('numbers'), + (r'<@|<<@|>@|>>@|\.>|\->', Operator), + (r'\*\*|\+\+|\-\-|\.|\=|\~\=|<\=|>\=|\=\=|\~\=\=|<<|<<\=|>>|>>\=', Operator), + (r':\=|:\=:|\->|<\->|\+:\=|\|', Operator), + (r'\=\=\=|\~\=\=\=', Operator), + (r'"(?:[^\\"]|\\.)*"', String), + (r"'(?:[^\\']|\\.)*'", String.Character), + (r'[*<>+=/&!?@~\\-]', Operator), + (r'\^', Operator), + (r'(\w+)(\s*|[(,])', bygroups(Name, using(this))), + (r"([\[\]])", Punctuation), + (r"(<>|=>|[()|:;,.'`]|[{}]|[%]|[&?])", Punctuation), + (r'\n+', Text), + ], + 'numbers': [ + (r'\b([+-]?([2-9]|[12][0-9]|3[0-6])[rR][0-9a-zA-Z]+)\b', Number.Hex), + (r'[+-]?[0-9]*\.([0-9]*)([Ee][+-]?[0-9]*)?', Number.Float), + (r'\b([+-]?[0-9]+[KMGTPkmgtp]?)\b', Number.Integer), + ], + 'subprogram': [ + (r'\(', Punctuation, ('#pop', 'formal_part')), + (r';', Punctuation, '#pop'), + (r'"[^"]+"|\w+', Name.Function), + include('root'), + ], + 'type_def': [ + (r'\(', Punctuation, 'formal_part'), + ], + 'formal_part': [ + (r'\)', Punctuation, '#pop'), + (r'\w+', Name.Variable), + (r',', Punctuation), + (r'(:string|:integer|:real)\b', Keyword.Reserved), + include('root'), + ], + } + + +class IconLexer(RegexLexer): + """ + Lexer for Icon. + + .. versionadded:: 1.6 + """ + name = 'Icon' + aliases = ['icon'] + filenames = ['*.icon', '*.ICON'] + mimetypes = [] + flags = re.MULTILINE + + tokens = { + 'root': [ + (r'[^\S\n]+', Text), + (r'#.*?\n', Comment.Single), + (r'[^\S\n]+', Text), + (r'class|method|procedure', Keyword.Declaration, 'subprogram'), + (r'(record)(\s+)(\w+)', + bygroups(Keyword.Declaration, Text, Keyword.Type), 'type_def'), + (r'(#line|\$C|\$Cend|\$define|\$else|\$endif|\$error|\$ifdef|' + r'\$ifndef|\$include|\$line|\$undef)\b', Keyword.PreProc), + (r'(&null|&fail)\b', Keyword.Constant), + (r'&allocated|&ascii|&clock|&collections|&column|&col|&control|' + r'&cset|¤t|&dateline|&date|&digits|&dump|' + r'&errno|&errornumber|&errortext|&errorvalue|&error|&errout|' + r'&eventcode|&eventvalue|&eventsource|&e|' + r'&features|&file|&host|&input|&interval|&lcase|&letters|' + r'&level|&line|&ldrag|&lpress|&lrelease|' + r'&main|&mdrag|&meta|&mpress|&mrelease|&now|&output|' + r'&phi|&pick|&pi|&pos|&progname|' + r'&random|&rdrag|®ions|&resize|&row|&rpress|&rrelease|' + r'&shift|&source|&storage|&subject|' + r'&time|&trace|&ucase|&version|' + r'&window|&x|&y', Keyword.Reserved), + (r'(by|of|not|to)\b', Keyword.Reserved), + (r'(global|local|static)\b', Keyword.Reserved), + (r'link', Keyword.Declaration), + (words(( + 'break', 'case', 'create', 'default', 'end', 'all', + 'do', 'else', 'every', 'fail', 'if', 'initial', + 'invocable', 'next', + 'repeat', 'return', 'suspend', + 'then', 'until', 'while'), prefix=r'\b', suffix=r'\b'), + Keyword.Reserved), + (words(( + 'abs', 'acos', 'Active', 'Alert', 'any', + 'args', 'array', 'asin', 'atan', 'atanh', 'Attrib', + 'bal', 'Bg', + 'callout', 'center', 'char', 'chdir', 'chmod', 'chown', 'chroot', + 'Clip', 'Clone', 'close', 'cofail', 'collect', + 'Color', 'ColorValue', 'condvar', 'copy', + 'CopyArea', 'cos', 'Couple', 'crypt', 'cset', 'ctime', + 'delay', 'delete', 'detab', 'display', 'DrawArc', + 'DrawCircle', 'DrawCube', 'DrawCurve', 'DrawCylinder', + 'DrawDisk', 'DrawImage', 'DrawLine', 'DrawPoint', 'DrawPolygon', + 'DrawRectangle', 'DrawSegment', 'DrawSphere', 'DrawString', + 'DrawTorus', 'dtor', + 'entab', 'EraseArea', 'errorclear', 'Event', 'eventmask', + 'EvGet', 'EvSend', 'exec', 'exit', 'exp', 'Eye', + 'fcntl', 'fdup', 'fetch', 'Fg', 'fieldnames', + 'FillArc', 'FillCircle', 'FillPolygon', + 'FillRectangle', 'find', 'flock', 'flush', 'Font', + 'FreeColor', 'FreeSpace', 'function', + 'get', 'getch', 'getche', 'getenv', + 'GetSpace', 'gettimeofday', + 'getuid', 'globalnames', 'GotoRC', 'GotoXY', 'gtime', 'hardlink', + 'iand', 'icom', 'IdentityMatrix', 'image', 'InPort', 'insert', + 'Int86', 'integer', 'ioctl', 'ior', 'ishift', 'istate', 'ixor', + 'kbhit', 'key', 'keyword', 'kill', + 'left', 'Len', 'list', 'load', 'loadfunc', 'localnames', + 'lock', 'log', 'Lower', 'lstat', + 'many', 'map', 'match', 'MatrixMode', 'max', 'member', + 'membernames', 'methodnames', 'methods', 'min', 'mkdir', 'move', + 'MultMatrix', 'mutex', + 'name', 'NewColor', 'Normals', 'numeric', + 'open', 'opencl', 'oprec', 'ord', 'OutPort', + 'PaletteChars', 'PaletteColor', 'PaletteKey', 'paramnames', + 'parent', 'Pattern', 'Peek', 'Pending', 'pipe', 'Pixel', + 'Poke', 'pop', 'PopMatrix', 'Pos', 'pos', + 'proc', 'pull', 'push', 'PushMatrix', 'PushRotate', 'PushScale', + 'PushTranslate', 'put', + 'QueryPointer', + 'Raise', 'read', 'ReadImage', 'readlink', 'reads', 'ready', + 'real', 'receive', 'Refresh', 'Rem', 'remove', 'rename', + 'repl', 'reverse', 'right', 'rmdir', 'Rotate', 'Rpos', + 'rtod', 'runerr', + 'save', 'Scale', 'seek', 'select', 'send', 'seq', + 'serial', 'set', 'setenv', + 'setuid', 'signal', 'sin', 'sort', 'sortf', + 'spawn', 'sql', 'sqrt', 'stat', 'staticnames', 'stop', + 'string', 'structure', 'Swi', + 'symlink', 'sys_errstr', 'system', 'syswrite', + 'tab', 'table', 'tan', + 'Texcoord', 'Texture', 'TextWidth', 'Translate', + 'trap', 'trim', 'truncate', 'trylock', 'type', + 'umask', 'Uncouple', 'unlock', 'upto', 'utime', + 'variable', + 'wait', 'WAttrib', 'WDefault', 'WFlush', 'where', + 'WinAssociate', 'WinButton', 'WinColorDialog', 'WindowContents', + 'WinEditRegion', 'WinFontDialog', 'WinMenuBar', 'WinOpenDialog', + 'WinPlayMedia', 'WinSaveDialog', 'WinScrollBar', 'WinSelectDialog', + 'write', 'WriteImage', 'writes', 'WSection', + 'WSync'), prefix=r'\b', suffix=r'\b'), + Name.Function), + include('numbers'), + (r'\*\*|\+\+|\-\-|\.|\=|\~\=|<\=|>\=|\=\=|\~\=\=|<<|<<\=|>>|>>\=', Operator), + (r':\=|:\=:|<\-|<\->|\+:\=|\||\|\|', Operator), + (r'\=\=\=|\~\=\=\=', Operator), + (r'"(?:[^\\"]|\\.)*"', String), + (r"'(?:[^\\']|\\.)*'", String.Character), + (r'[*<>+=/&!?@~\\-]', Operator), + (r'(\w+)(\s*|[(,])', bygroups(Name, using(this))), + (r"([\[\]])", Punctuation), + (r"(<>|=>|[()|:;,.'`]|[{}]|[%^]|[&?])", Punctuation), + (r'\n+', Text), + ], + 'numbers': [ + (r'\b([+-]?([2-9]|[12][0-9]|3[0-6])[rR][0-9a-zA-Z]+)\b', Number.Hex), + (r'[+-]?[0-9]*\.([0-9]*)([Ee][+-]?[0-9]*)?', Number.Float), + (r'\b([+-]?[0-9]+[KMGTPkmgtp]?)\b', Number.Integer), + ], + 'subprogram': [ + (r'\(', Punctuation, ('#pop', 'formal_part')), + (r';', Punctuation, '#pop'), + (r'"[^"]+"|\w+', Name.Function), + include('root'), + ], + 'type_def': [ + (r'\(', Punctuation, 'formal_part'), + ], + 'formal_part': [ + (r'\)', Punctuation, '#pop'), + (r'\w+', Name.Variable), + (r',', Punctuation), + (r'(:string|:integer|:real)\b', Keyword.Reserved), + include('root'), + ], + } + + +class UcodeLexer(RegexLexer): + """ + Lexer for Icon ucode files. + + .. versionadded:: 2.4 + """ + name = 'ucode' + aliases = ['ucode'] + filenames = ['*.u', '*.u1', '*.u2'] + mimetypes = [] + flags = re.MULTILINE + + tokens = { + 'root': [ + (r'(#.*\n)', Comment), + (words(( + 'con', 'declend', 'end', + 'global', + 'impl', 'invocable', + 'lab', 'link', 'local', + 'record', + 'uid', 'unions', + 'version'), + prefix=r'\b', suffix=r'\b'), + Name.Function), + (words(( + 'colm', 'filen', 'line', 'synt'), + prefix=r'\b', suffix=r'\b'), + Comment), + (words(( + 'asgn', + 'bang', 'bscan', + 'cat', 'ccase', 'chfail', + 'coact', 'cofail', 'compl', + 'coret', 'create', 'cset', + 'diff', 'div', 'dup', + 'efail', 'einit', 'end', 'eqv', 'eret', + 'error', 'escan', 'esusp', + 'field', + 'goto', + 'init', 'int', 'inter', + 'invoke', + 'keywd', + 'lconcat', 'lexeq', 'lexge', + 'lexgt', 'lexle', 'lexlt', 'lexne', + 'limit', 'llist', 'lsusp', + 'mark', 'mark0', 'minus', 'mod', 'mult', + 'neg', 'neqv', 'nonnull', 'noop', 'null', + 'number', 'numeq', 'numge', 'numgt', + 'numle', 'numlt', 'numne', + 'pfail', 'plus', 'pnull', 'pop', 'power', + 'pret', 'proc', 'psusp', 'push1', 'pushn1', + 'random', 'rasgn', 'rcv', 'rcvbk', 'real', + 'refresh', 'rswap', + 'sdup', 'sect', 'size', 'snd', 'sndbk', + 'str', 'subsc', 'swap', + 'tabmat', 'tally', 'toby', 'trace', + 'unmark', + 'value', 'var'), prefix=r'\b', suffix=r'\b'), + Keyword.Declaration), + (words(( + 'any', + 'case', + 'endcase', 'endevery', 'endif', + 'endifelse', 'endrepeat', 'endsuspend', + 'enduntil', 'endwhile', 'every', + 'if', 'ifelse', + 'repeat', + 'suspend', + 'until', + 'while'), + prefix=r'\b', suffix=r'\b'), + Name.Constant), + (r'\d+(\s*|\.$|$)', Number.Integer), + (r'[+-]?\d*\.\d+(E[-+]?\d+)?', Number.Float), + (r'[+-]?\d+\.\d*(E[-+]?\d+)?', Number.Float), + (r"(<>|=>|[()|:;,.'`]|[{}]|[%^]|[&?])", Punctuation), + (r'\s+\b', Text), + (r'[\w-]+', Text), + ], + } diff --git a/pygments/lexers/urbi.py b/pygments/lexers/urbi.py index 7aaba90c..72349cbd 100644 --- a/pygments/lexers/urbi.py +++ b/pygments/lexers/urbi.py @@ -5,7 +5,7 @@ Lexers for UrbiScript language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/varnish.py b/pygments/lexers/varnish.py index 44521422..4db74e3c 100644 --- a/pygments/lexers/varnish.py +++ b/pygments/lexers/varnish.py @@ -5,7 +5,7 @@ Lexers for Varnish configuration - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -36,7 +36,7 @@ class VCLLexer(RegexLexer): # Skip over comments and blank lines # This is accurate enough that returning 0.9 is reasonable. # Almost no VCL files start without some comments. - elif '\nvcl 4\.0;' in text[:1000]: + elif '\nvcl 4.0;' in text[:1000]: return 0.9 tokens = { @@ -120,7 +120,7 @@ class VCLLexer(RegexLexer): r'([a-zA-Z_]\w*)' r'(\s*\(.*\))', bygroups(Name.Function, Punctuation, Name.Function, using(this))), - ('[a-zA-Z_]\w*', Name), + (r'[a-zA-Z_]\w*', Name), ], 'comment': [ (r'[^*/]+', Comment.Multiline), diff --git a/pygments/lexers/verification.py b/pygments/lexers/verification.py index 5322e17f..f6530726 100644 --- a/pygments/lexers/verification.py +++ b/pygments/lexers/verification.py @@ -5,7 +5,7 @@ Lexer for Intermediate Verification Languages (IVLs). - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py index 6e9c4f92..587b0cde 100644 --- a/pygments/lexers/web.py +++ b/pygments/lexers/web.py @@ -5,7 +5,7 @@ Just export previously exported lexers. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/webmisc.py b/pygments/lexers/webmisc.py index 712c8246..b39334bc 100644 --- a/pygments/lexers/webmisc.py +++ b/pygments/lexers/webmisc.py @@ -5,7 +5,7 @@ Lexers for misc. web stuff. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -438,7 +438,7 @@ class XQueryLexer(ExtendedRegexLexer): ], 'varname': [ (r'\(:', Comment, 'comment'), - (r'(' + qname + ')(\()?', bygroups(Name, Punctuation), 'operator'), + (r'(' + qname + r')(\()?', bygroups(Name, Punctuation), 'operator'), ], 'singletype': [ include('whitespace'), @@ -643,9 +643,9 @@ class XQueryLexer(ExtendedRegexLexer): bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'operator'), (r'(declare)(\s+)(context)(\s+)(item)', bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'operator'), - (ncname + ':\*', Name, 'operator'), - ('\*:'+ncname, Name.Tag, 'operator'), - ('\*', Name.Tag, 'operator'), + (ncname + r':\*', Name, 'operator'), + (r'\*:'+ncname, Name.Tag, 'operator'), + (r'\*', Name.Tag, 'operator'), (stringdouble, String.Double, 'operator'), (stringsingle, String.Single, 'operator'), @@ -661,7 +661,8 @@ class XQueryLexer(ExtendedRegexLexer): # NAMESPACE KEYWORD (r'(declare)(\s+)(default)(\s+)(element|function)', - bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'namespacekeyword'), + bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), + 'namespacekeyword'), (r'(import)(\s+)(schema|module)', bygroups(Keyword.Pseudo, Text, Keyword.Pseudo), 'namespacekeyword'), (r'(declare)(\s+)(copy-namespaces)', @@ -861,7 +862,7 @@ class QmlLexer(RegexLexer): class CirruLexer(RegexLexer): - """ + r""" Syntax rules of Cirru can be found at: http://cirru.org/ diff --git a/pygments/lexers/whiley.py b/pygments/lexers/whiley.py index 0d0e8ab8..feeeb3b7 100644 --- a/pygments/lexers/whiley.py +++ b/pygments/lexers/whiley.py @@ -5,7 +5,7 @@ Lexers for the Whiley language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/x10.py b/pygments/lexers/x10.py index 1c63326d..eac87b1c 100644 --- a/pygments/lexers/x10.py +++ b/pygments/lexers/x10.py @@ -5,7 +5,7 @@ Lexers for the X10 programming language. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/xorg.py b/pygments/lexers/xorg.py index 89475a80..eadcfe76 100644 --- a/pygments/lexers/xorg.py +++ b/pygments/lexers/xorg.py @@ -5,7 +5,7 @@ Lexers for Xorg configs. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -16,6 +16,7 @@ __all__ = ['XorgLexer'] class XorgLexer(RegexLexer): + """Lexer for xorg.conf file.""" name = 'Xorg' aliases = ['xorg.conf'] filenames = ['xorg.conf'] @@ -26,8 +27,8 @@ class XorgLexer(RegexLexer): (r'\s+', Text), (r'#.*$', Comment), - (r'((|Sub)Section)(\s+)("\w+")', - bygroups(String.Escape, String.Escape, Text, String.Escape)), + (r'((?:Sub)?Section)(\s+)("\w+")', + bygroups(String.Escape, Text, String.Escape)), (r'(End(|Sub)Section)', String.Escape), (r'(\w+)(\s+)([^\n#]+)', diff --git a/pygments/lexers/zig.py b/pygments/lexers/zig.py new file mode 100644 index 00000000..7850fdf0 --- /dev/null +++ b/pygments/lexers/zig.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.zig + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for Zig. + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re +from pygments.lexer import RegexLexer, bygroups, include, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Error, Whitespace + +__all__ = ['ZigLexer'] + +class ZigLexer(RegexLexer): + """ + For `Zig <http://www.ziglang.org>`_ source code. + + grammar: https://ziglang.org/documentation/master/#Grammar + """ + name = 'Zig' + aliases = ['zig'] + filenames = ['*.zig'] + mimetypes = ['text/zig'] + + type_keywords = ( + words(('bool', 'f16', 'f32', 'f64', 'f128', 'void', 'noreturn', 'type', 'anyerror', 'promise', + 'i0', 'u0', 'isize', 'usize', 'comptime_int', 'comptime_float', + 'c_short', 'c_ushort', 'c_int', 'c_uint', 'c_long', 'c_ulong', 'c_longlong', 'c_ulonglong', 'c_longdouble', 'c_void' + 'i8', 'u8', 'i16', 'u16', 'i32', 'u32', 'i64', 'u64', 'i128', 'u128' + ), suffix=r'\b'), + Keyword.Type) + + storage_keywords = ( + words(('const', 'var', 'extern', 'packed', 'export', 'pub', 'noalias', + 'inline', 'comptime', 'nakedcc', 'stdcallcc', 'volatile', 'allowzero', + 'align', 'linksection', 'threadlocal'), suffix=r'\b'), + Keyword.Reserved) + + structure_keywords = ( + words(('struct', 'enum', 'union', 'error'), suffix=r'\b'), + Keyword) + + statement_keywords = ( + words(('break', 'return', 'continue', 'asm', 'defer', 'errdefer', + 'unreachable', 'try', 'catch', 'async', 'await', 'suspend', + 'resume', 'cancel'), suffix=r'\b'), + Keyword) + + conditional_keywords = ( + words(('if', 'else', 'switch', 'and', 'or', 'orelse'), suffix=r'\b'), + Keyword) + + repeat_keywords = ( + words(('while', 'for'), suffix=r'\b'), + Keyword) + + other_keywords = ( + words(('fn', 'usingnamespace', 'test'), suffix=r'\b'), + Keyword) + + constant_keywords = ( + words(('true', 'false', 'null', 'undefined'), suffix=r'\b'), + Keyword.Constant) + + tokens = { + 'root': [ + (r'\n', Whitespace), + (r'\s+', Whitespace), + (r'//.*?\n', Comment.Single), + + # Keywords + statement_keywords, + storage_keywords, + structure_keywords, + repeat_keywords, + type_keywords, + constant_keywords, + conditional_keywords, + other_keywords, + + # Floats + (r'0x[0-9a-fA-F]+\.[0-9a-fA-F]+([pP][\-+]?[0-9a-fA-F]+)?', Number.Float), + (r'0x[0-9a-fA-F]+\.?[pP][\-+]?[0-9a-fA-F]+', Number.Float), + (r'[0-9]+\.[0-9]+([eE][-+]?[0-9]+)?', Number.Float), + (r'[0-9]+\.?[eE][-+]?[0-9]+', Number.Float), + + # Integers + (r'0b[01]+', Number.Bin), + (r'0o[0-7]+', Number.Oct), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'[0-9]+', Number.Integer), + + # Identifier + (r'@[a-zA-Z_]\w*',Name.Builtin), + (r'[a-zA-Z_]\w*', Name), + + # Characters + (r'\'\\\'\'', String.Escape), + (r'\'\\(|x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\'"])\'', String.Escape), + (r'\'[^\\\']\'', String), + + # Strings + (r'\\\\[^\n]*', String.Heredoc), + (r'c\\\\[^\n]*', String.Heredoc), + (r'c?"',String, 'string'), + + # Operators, Punctuation + (r'[+%=><|^!?/\-*&~:]', Operator), + (r'[{}()\[\],.;]', Punctuation) + ], + 'string': [ + (r'\\(x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{6}|[nr\\t\'"])', String.Escape), + (r'[^\\"\n]+', String), + (r'"', String, '#pop') + ] + } + + def get_tokens_unprocessed(self, text): + for index, token, value in \ + RegexLexer.get_tokens_unprocessed(self, text): + yield index, token, value diff --git a/pygments/modeline.py b/pygments/modeline.py index 9f8d5dab..31b2e7fb 100644 --- a/pygments/modeline.py +++ b/pygments/modeline.py @@ -5,7 +5,7 @@ A simple modeline parser (based on pymodeline). - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/plugin.py b/pygments/plugin.py index 7987d646..3d185efc 100644 --- a/pygments/plugin.py +++ b/pygments/plugin.py @@ -32,7 +32,7 @@ yourfilter = yourfilter:YourFilter - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ LEXER_ENTRY_POINT = 'pygments.lexers' @@ -40,14 +40,16 @@ FORMATTER_ENTRY_POINT = 'pygments.formatters' STYLE_ENTRY_POINT = 'pygments.styles' FILTER_ENTRY_POINT = 'pygments.filters' + def iter_entry_points(group_name): try: import pkg_resources - except ImportError: + except (ImportError, IOError): return [] return pkg_resources.iter_entry_points(group_name) + def find_plugin_lexers(): for entrypoint in iter_entry_points(LEXER_ENTRY_POINT): yield entrypoint.load() diff --git a/pygments/regexopt.py b/pygments/regexopt.py index dcfae2fd..59d77ee0 100644 --- a/pygments/regexopt.py +++ b/pygments/regexopt.py @@ -6,7 +6,7 @@ An algorithm that generates optimized regexes for matching long lists of literal strings. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/scanner.py b/pygments/scanner.py index 3350ac8e..bcb19ed9 100644 --- a/pygments/scanner.py +++ b/pygments/scanner.py @@ -12,7 +12,7 @@ Have a look at the `DelphiLexer` to get an idea of how to use this scanner. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re diff --git a/pygments/sphinxext.py b/pygments/sphinxext.py index f962f8c6..282693e7 100644 --- a/pygments/sphinxext.py +++ b/pygments/sphinxext.py @@ -6,7 +6,7 @@ Sphinx extension to generate automatic documentation of lexers, formatters and filters. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -16,7 +16,7 @@ import sys from docutils import nodes from docutils.statemachine import ViewList -from sphinx.util.compat import Directive +from docutils.parsers.rst import Directive from sphinx.util.nodes import nested_parse_with_titles diff --git a/pygments/style.py b/pygments/style.py index 879c4e05..73264577 100644 --- a/pygments/style.py +++ b/pygments/style.py @@ -5,33 +5,54 @@ Basic style object. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ from pygments.token import Token, STANDARD_TYPES from pygments.util import add_metaclass -# Default mapping of #ansixxx to RGB colors. +# Default mapping of ansixxx to RGB colors. _ansimap = { # dark - '#ansiblack': '000000', - '#ansidarkred': '7f0000', - '#ansidarkgreen': '007f00', - '#ansibrown': '7f7fe0', - '#ansidarkblue': '00007f', - '#ansipurple': '7f007f', - '#ansiteal': '007f7f', - '#ansilightgray': 'e5e5e5', + 'ansiblack': '000000', + 'ansired': '7f0000', + 'ansigreen': '007f00', + 'ansiyellow': '7f7fe0', + 'ansiblue': '00007f', + 'ansimagenta': '7f007f', + 'ansicyan': '007f7f', + 'ansigray': 'e5e5e5', # normal - '#ansidarkgray': '555555', - '#ansired': 'ff0000', - '#ansigreen': '00ff00', - '#ansiyellow': 'ffff00', - '#ansiblue': '0000ff', - '#ansifuchsia': 'ff00ff', - '#ansiturquoise': '00ffff', - '#ansiwhite': 'ffffff', + 'ansibrightblack': '555555', + 'ansibrightred': 'ff0000', + 'ansibrightgreen': '00ff00', + 'ansibrightyellow': 'ffff00', + 'ansibrightblue': '0000ff', + 'ansibrightmagenta': 'ff00ff', + 'ansibrightcyan': '00ffff', + 'ansiwhite': 'ffffff', +} +# mapping of deprecated #ansixxx colors to new color names +_deprecated_ansicolors = { + # dark + '#ansiblack': 'ansiblack', + '#ansidarkred': 'ansired', + '#ansidarkgreen': 'ansigreen', + '#ansibrown': 'ansiyellow', + '#ansidarkblue': 'ansiblue', + '#ansipurple': 'ansimagenta', + '#ansiteal': 'ansicyan', + '#ansilightgray': 'ansigray', + # normal + '#ansidarkgray': 'ansibrightblack', + '#ansired': 'ansibrightred', + '#ansigreen': 'ansibrightgreen', + '#ansiyellow': 'ansibrightyellow', + '#ansiblue': 'ansibrightblue', + '#ansifuchsia': 'ansibrightmagenta', + '#ansiturquoise': 'ansibrightcyan', + '#ansiwhite': 'ansiwhite', } ansicolors = set(_ansimap) @@ -52,9 +73,11 @@ class StyleMeta(type): if len(col) == 6: return col elif len(col) == 3: - return col[0]*2 + col[1]*2 + col[2]*2 + return col[0] * 2 + col[1] * 2 + col[2] * 2 elif text == '': return '' + elif text.startswith('var') or text.startswith('calc'): + return text assert False, "wrong color format %r" % text _styles = obj._styles = {} @@ -106,11 +129,15 @@ class StyleMeta(type): t = cls._styles[token] ansicolor = bgansicolor = None color = t[0] - if color.startswith('#ansi'): + if color in _deprecated_ansicolors: + color = _deprecated_ansicolors[color] + if color in ansicolors: ansicolor = color color = _ansimap[color] bgcolor = t[4] - if bgcolor.startswith('#ansi'): + if bgcolor in _deprecated_ansicolors: + bgcolor = _deprecated_ansicolors[color] + if bgcolor in ansicolors: bgansicolor = bgcolor bgcolor = _ansimap[bgcolor] diff --git a/pygments/styles/__init__.py b/pygments/styles/__init__.py index 839a9b78..8de9e43b 100644 --- a/pygments/styles/__init__.py +++ b/pygments/styles/__init__.py @@ -5,7 +5,7 @@ Contains built-in styles. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -44,6 +44,12 @@ STYLE_MAP = { 'arduino': 'arduino::ArduinoStyle', 'rainbow_dash': 'rainbow_dash::RainbowDashStyle', 'abap': 'abap::AbapStyle', + 'solarized-dark': 'solarized::SolarizedDarkStyle', + 'solarized-light': 'solarized::SolarizedLightStyle', + 'sas': 'sas::SasStyle', + 'stata': 'stata_light::StataLightStyle', + 'stata-light': 'stata_light::StataLightStyle', + 'stata-dark': 'stata_dark::StataDarkStyle', } diff --git a/pygments/styles/abap.py b/pygments/styles/abap.py index 91286a3a..d0622d29 100644 --- a/pygments/styles/abap.py +++ b/pygments/styles/abap.py @@ -5,7 +5,7 @@ ABAP workbench like style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/algol.py b/pygments/styles/algol.py index 16461e0b..d4d91870 100644 --- a/pygments/styles/algol.py +++ b/pygments/styles/algol.py @@ -26,7 +26,7 @@ [1] `Revised Report on the Algorithmic Language Algol-60 <http://www.masswerk.at/algol60/report.htm>` - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/algol_nu.py b/pygments/styles/algol_nu.py index 366ae215..09d69452 100644 --- a/pygments/styles/algol_nu.py +++ b/pygments/styles/algol_nu.py @@ -26,7 +26,7 @@ [1] `Revised Report on the Algorithmic Language Algol-60 <http://www.masswerk.at/algol60/report.htm>` - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/arduino.py b/pygments/styles/arduino.py index 57e3809e..160ca094 100644 --- a/pygments/styles/arduino.py +++ b/pygments/styles/arduino.py @@ -5,17 +5,17 @@ Arduino® Syntax highlighting style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic, Whitespace + Number, Operator, Generic, Whitespace class ArduinoStyle(Style): - """ + u""" The Arduino® language style. This style is designed to highlight the Arduino source code, so exepect the best results with it. """ diff --git a/pygments/styles/autumn.py b/pygments/styles/autumn.py index 71b93b1e..d5ee045d 100644 --- a/pygments/styles/autumn.py +++ b/pygments/styles/autumn.py @@ -5,7 +5,7 @@ A colorful style, inspired by the terminal highlighting style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/borland.py b/pygments/styles/borland.py index 0d13d1aa..e8ec1229 100644 --- a/pygments/styles/borland.py +++ b/pygments/styles/borland.py @@ -5,7 +5,7 @@ Style similar to the style used in the Borland IDEs. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/bw.py b/pygments/styles/bw.py index f0a6b148..54a653b6 100644 --- a/pygments/styles/bw.py +++ b/pygments/styles/bw.py @@ -5,7 +5,7 @@ Simple black/white only style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/colorful.py b/pygments/styles/colorful.py index bfc0b502..6aa493c9 100644 --- a/pygments/styles/colorful.py +++ b/pygments/styles/colorful.py @@ -5,7 +5,7 @@ A colorful style, inspired by CodeRay. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/default.py b/pygments/styles/default.py index 6b9bd446..c93d29fc 100644 --- a/pygments/styles/default.py +++ b/pygments/styles/default.py @@ -5,7 +5,7 @@ The default highlighting style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/emacs.py b/pygments/styles/emacs.py index af15f30d..8408b09a 100644 --- a/pygments/styles/emacs.py +++ b/pygments/styles/emacs.py @@ -5,7 +5,7 @@ A highlighting style for Pygments, inspired by Emacs. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/friendly.py b/pygments/styles/friendly.py index b2d1c0ce..4184a8e3 100644 --- a/pygments/styles/friendly.py +++ b/pygments/styles/friendly.py @@ -5,7 +5,7 @@ A modern style based on the VIM pyte theme. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/fruity.py b/pygments/styles/fruity.py index 1bbe0316..7f5c0e3f 100644 --- a/pygments/styles/fruity.py +++ b/pygments/styles/fruity.py @@ -5,7 +5,7 @@ pygments version of my "fruity" vim theme. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/igor.py b/pygments/styles/igor.py index d4620a42..70ee38bc 100644 --- a/pygments/styles/igor.py +++ b/pygments/styles/igor.py @@ -5,7 +5,7 @@ Igor Pro default style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/lovelace.py b/pygments/styles/lovelace.py index 861f778d..65b641ec 100644 --- a/pygments/styles/lovelace.py +++ b/pygments/styles/lovelace.py @@ -9,7 +9,7 @@ A desaturated, somewhat subdued style created for the Lovelace interactive learning environment. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/manni.py b/pygments/styles/manni.py index f0a325af..bd14f6a1 100644 --- a/pygments/styles/manni.py +++ b/pygments/styles/manni.py @@ -8,7 +8,7 @@ This is a port of the style used in the `php port`_ of pygments by Manni. The style is called 'default' there. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/monokai.py b/pygments/styles/monokai.py index 337e2f89..4580df2b 100644 --- a/pygments/styles/monokai.py +++ b/pygments/styles/monokai.py @@ -7,7 +7,7 @@ http://www.monokai.nl/blog/2006/07/15/textmate-color-theme/ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/murphy.py b/pygments/styles/murphy.py index c8270065..e9b50d7a 100644 --- a/pygments/styles/murphy.py +++ b/pygments/styles/murphy.py @@ -5,7 +5,7 @@ Murphy's style from CodeRay. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/native.py b/pygments/styles/native.py index 921a58d9..a41852ac 100644 --- a/pygments/styles/native.py +++ b/pygments/styles/native.py @@ -5,7 +5,7 @@ pygments version of my "native" vim theme. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/paraiso_dark.py b/pygments/styles/paraiso_dark.py index 5f334bb9..50afc37e 100644 --- a/pygments/styles/paraiso_dark.py +++ b/pygments/styles/paraiso_dark.py @@ -9,7 +9,7 @@ Created with Base16 Builder by Chris Kempson (https://github.com/chriskempson/base16-builder). - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/paraiso_light.py b/pygments/styles/paraiso_light.py index a8112819..a5f357ba 100644 --- a/pygments/styles/paraiso_light.py +++ b/pygments/styles/paraiso_light.py @@ -9,7 +9,7 @@ Created with Base16 Builder by Chris Kempson (https://github.com/chriskempson/base16-builder). - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/pastie.py b/pygments/styles/pastie.py index d6142908..6b022025 100644 --- a/pygments/styles/pastie.py +++ b/pygments/styles/pastie.py @@ -7,7 +7,7 @@ .. _pastie: http://pastie.caboo.se/ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/perldoc.py b/pygments/styles/perldoc.py index 24af2df6..a6c21e90 100644 --- a/pygments/styles/perldoc.py +++ b/pygments/styles/perldoc.py @@ -7,7 +7,7 @@ .. _perldoc: http://perldoc.perl.org/ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/rainbow_dash.py b/pygments/styles/rainbow_dash.py index 7cf5c9d7..c2c2c7c5 100644 --- a/pygments/styles/rainbow_dash.py +++ b/pygments/styles/rainbow_dash.py @@ -7,7 +7,7 @@ .. _theme: http://sanssecours.github.io/Rainbow-Dash.tmbundle - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/rrt.py b/pygments/styles/rrt.py index 96f9490c..d61085e0 100644 --- a/pygments/styles/rrt.py +++ b/pygments/styles/rrt.py @@ -5,7 +5,7 @@ pygments "rrt" theme, based on Zap and Emacs defaults. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/sas.py b/pygments/styles/sas.py index 78686fc2..3121aa53 100644 --- a/pygments/styles/sas.py +++ b/pygments/styles/sas.py @@ -7,7 +7,7 @@ meant to be a complete style. It's merely meant to mimic SAS' program editor syntax highlighting. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/solarized.py b/pygments/styles/solarized.py new file mode 100644 index 00000000..7d4b113d --- /dev/null +++ b/pygments/styles/solarized.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +""" + pygments.styles.solarized + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Solarized by Camil Staps + + A Pygments style for the Solarized themes (licensed under MIT). + See: https://github.com/altercation/solarized + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import Comment, Error, Generic, Keyword, Name, Number, \ + Operator, String, Token + + +def make_style(colors): + return { + Token: colors['base0'], + + Comment: 'italic ' + colors['base01'], + Comment.Hashbang: colors['base01'], + Comment.Multiline: colors['base01'], + Comment.Preproc: 'noitalic ' + colors['magenta'], + Comment.PreprocFile: 'noitalic ' + colors['base01'], + + Keyword: colors['green'], + Keyword.Constant: colors['cyan'], + Keyword.Declaration: colors['cyan'], + Keyword.Namespace: colors['orange'], + Keyword.Type: colors['yellow'], + + Operator: colors['base01'], + Operator.Word: colors['green'], + + Name.Builtin: colors['blue'], + Name.Builtin.Pseudo: colors['blue'], + Name.Class: colors['blue'], + Name.Constant: colors['blue'], + Name.Decorator: colors['blue'], + Name.Entity: colors['blue'], + Name.Exception: colors['blue'], + Name.Function: colors['blue'], + Name.Function.Magic: colors['blue'], + Name.Label: colors['blue'], + Name.Namespace: colors['blue'], + Name.Tag: colors['blue'], + Name.Variable: colors['blue'], + Name.Variable.Global:colors['blue'], + Name.Variable.Magic: colors['blue'], + + String: colors['cyan'], + String.Doc: colors['base01'], + String.Regex: colors['orange'], + + Number: colors['cyan'], + + Generic.Deleted: colors['red'], + Generic.Emph: 'italic', + Generic.Error: colors['red'], + Generic.Heading: 'bold', + Generic.Subheading: 'underline', + Generic.Inserted: colors['green'], + Generic.Strong: 'bold', + Generic.Traceback: colors['blue'], + + Error: 'bg:' + colors['red'], + } + + +DARK_COLORS = { + 'base03': '#002b36', + 'base02': '#073642', + 'base01': '#586e75', + 'base00': '#657b83', + 'base0': '#839496', + 'base1': '#93a1a1', + 'base2': '#eee8d5', + 'base3': '#fdf6e3', + 'yellow': '#b58900', + 'orange': '#cb4b16', + 'red': '#dc322f', + 'magenta': '#d33682', + 'violet': '#6c71c4', + 'blue': '#268bd2', + 'cyan': '#2aa198', + 'green': '#859900', +} + +LIGHT_COLORS = { + 'base3': '#002b36', + 'base2': '#073642', + 'base1': '#586e75', + 'base0': '#657b83', + 'base00': '#839496', + 'base01': '#93a1a1', + 'base02': '#eee8d5', + 'base03': '#fdf6e3', + 'yellow': '#b58900', + 'orange': '#cb4b16', + 'red': '#dc322f', + 'magenta': '#d33682', + 'violet': '#6c71c4', + 'blue': '#268bd2', + 'cyan': '#2aa198', + 'green': '#859900', +} + + +class SolarizedDarkStyle(Style): + """ + The solarized style, dark. + """ + + styles = make_style(DARK_COLORS) + background_color = DARK_COLORS['base03'] + highlight_color = DARK_COLORS['base02'] + + +class SolarizedLightStyle(SolarizedDarkStyle): + """ + The solarized style, light. + """ + + styles = make_style(LIGHT_COLORS) + background_color = LIGHT_COLORS['base03'] + highlight_color = LIGHT_COLORS['base02'] diff --git a/pygments/styles/stata_dark.py b/pygments/styles/stata_dark.py new file mode 100644 index 00000000..122b3d77 --- /dev/null +++ b/pygments/styles/stata_dark.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +""" + pygments.styles.stata_dark + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Dark style inspired by Stata's do-file editor. Note this is not + meant to be a complete style, just for Stata's file formats. + + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import Keyword, Name, Comment, String, Error, \ + Number, Operator, Whitespace, Generic, Text + + +class StataDarkStyle(Style): + + default_style = '' + + background_color = "#232629" + highlight_color = "#49483e" + + styles = { + Whitespace: '#bbbbbb', + Error: 'bg:#e3d2d2 #a61717', + Text: '#cccccc', + String: '#51cc99', + Number: '#4FB8CC', + Operator: '', + Name.Function: '#6a6aff', + Name.Other: '#e2828e', + Keyword: 'bold #7686bb', + Keyword.Constant: '', + Comment: 'italic #777777', + Name.Variable: 'bold #7AB4DB', + Name.Variable.Global: 'bold #BE646C', + Generic.Prompt: '#ffffff', + } diff --git a/pygments/styles/stata.py b/pygments/styles/stata_light.py index 2b5f5edd..41009a18 100644 --- a/pygments/styles/stata.py +++ b/pygments/styles/stata_light.py @@ -1,40 +1,39 @@ # -*- coding: utf-8 -*- """ - pygments.styles.stata - ~~~~~~~~~~~~~~~~~~~~~ + pygments.styles.stata_light + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Style inspired by Stata's do-file editor. Note this is not meant - to be a complete style. It's merely meant to mimic Stata's do file - editor syntax highlighting. + Light Style inspired by Stata's do-file editor. Note this is not + meant to be a complete style, just for Stata's file formats. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Whitespace + Number, Operator, Whitespace, Text -class StataStyle(Style): +class StataLightStyle(Style): """ - Style inspired by Stata's do-file editor. Note this is not meant - to be a complete style. It's merely meant to mimic Stata's do file - editor syntax highlighting. + Light mode style inspired by Stata's do-file editor. This is not + meant to be a complete style, just for use with Stata. """ default_style = '' - styles = { + Text: '#111111', Whitespace: '#bbbbbb', - Comment: 'italic #008800', + Error: 'bg:#e3d2d2 #a61717', String: '#7a2424', Number: '#2c2cff', Operator: '', + Name.Function: '#2c2cff', + Name.Other: '#be646c', Keyword: 'bold #353580', Keyword.Constant: '', - Name.Function: '#2c2cff', + Comment: 'italic #008800', Name.Variable: 'bold #35baba', Name.Variable.Global: 'bold #b5565e', - Error: 'bg:#e3d2d2 #a61717' } diff --git a/pygments/styles/tango.py b/pygments/styles/tango.py index 2abc8c61..5205244c 100644 --- a/pygments/styles/tango.py +++ b/pygments/styles/tango.py @@ -33,7 +33,7 @@ have been chosen to have the same style. Similarly, keywords (Keyword.*), and Operator.Word (and, or, in) have been assigned the same style. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/trac.py b/pygments/styles/trac.py index aff39fd4..1e96584d 100644 --- a/pygments/styles/trac.py +++ b/pygments/styles/trac.py @@ -5,7 +5,7 @@ Port of the default trac highlighter design. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/vim.py b/pygments/styles/vim.py index 888088b1..954e91d5 100644 --- a/pygments/styles/vim.py +++ b/pygments/styles/vim.py @@ -5,7 +5,7 @@ A highlighting style for Pygments, inspired by vim. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/vs.py b/pygments/styles/vs.py index bc3ed2b5..111e4aed 100644 --- a/pygments/styles/vs.py +++ b/pygments/styles/vs.py @@ -5,7 +5,7 @@ Simple style with MS Visual Studio colors. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/xcode.py b/pygments/styles/xcode.py index 64bfcf03..7e87d083 100644 --- a/pygments/styles/xcode.py +++ b/pygments/styles/xcode.py @@ -5,7 +5,7 @@ Style similar to the `Xcode` default theme. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/token.py b/pygments/token.py index 43f73c85..5c30eb46 100644 --- a/pygments/token.py +++ b/pygments/token.py @@ -5,7 +5,7 @@ Basic token types and the standard tokens. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/unistring.py b/pygments/unistring.py index 6096d110..dd011cf0 100644 --- a/pygments/unistring.py +++ b/pygments/unistring.py @@ -8,7 +8,7 @@ Inspired by chartypes_create.py from the MoinMoin project. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -16,9 +16,9 @@ import sys Cc = u'\x00-\x1f\x7f-\x9f' -Cf = u'\xad\u0600-\u0604\u061c\u06dd\u070f\u180e\u200b-\u200f\u202a-\u202e\u2060-\u2064\u2066-\u206f\ufeff\ufff9-\ufffb' +Cf = u'\xad\u0600-\u0605\u061c\u06dd\u070f\u08e2\u180e\u200b-\u200f\u202a-\u202e\u2060-\u2064\u2066-\u206f\ufeff\ufff9-\ufffb' -Cn = u'\u0378-\u0379\u037f-\u0383\u038b\u038d\u03a2\u0528-\u0530\u0557-\u0558\u0560\u0588\u058b-\u058e\u0590\u05c8-\u05cf\u05eb-\u05ef\u05f5-\u05ff\u0605\u061d\u070e\u074b-\u074c\u07b2-\u07bf\u07fb-\u07ff\u082e-\u082f\u083f\u085c-\u085d\u085f-\u089f\u08a1\u08ad-\u08e3\u08ff\u0978\u0980\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09c5-\u09c6\u09c9-\u09ca\u09cf-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09fc-\u0a00\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a50\u0a52-\u0a58\u0a5d\u0a5f-\u0a65\u0a76-\u0a80\u0a84\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0acf\u0ad1-\u0adf\u0ae4-\u0ae5\u0af2-\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34\u0b3a-\u0b3b\u0b45-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b64-\u0b65\u0b78-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bcf\u0bd1-\u0bd6\u0bd8-\u0be5\u0bfb-\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c3c\u0c45\u0c49\u0c4e-\u0c54\u0c57\u0c5a-\u0c5f\u0c64-\u0c65\u0c70-\u0c77\u0c80-\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbb\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce4-\u0ce5\u0cf0\u0cf3-\u0d01\u0d04\u0d0d\u0d11\u0d3b-\u0d3c\u0d45\u0d49\u0d4f-\u0d56\u0d58-\u0d5f\u0d64-\u0d65\u0d76-\u0d78\u0d80-\u0d81\u0d84\u0d97-\u0d99\u0db2\u0dbc\u0dbe-\u0dbf\u0dc7-\u0dc9\u0dcb-\u0dce\u0dd5\u0dd7\u0de0-\u0df1\u0df5-\u0e00\u0e3b-\u0e3e\u0e5c-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0edb\u0ee0-\u0eff\u0f48\u0f6d-\u0f70\u0f98\u0fbd\u0fcd\u0fdb-\u0fff\u10c6\u10c8-\u10cc\u10ce-\u10cf\u1249\u124e-\u124f\u1257\u1259\u125e-\u125f\u1289\u128e-\u128f\u12b1\u12b6-\u12b7\u12bf\u12c1\u12c6-\u12c7\u12d7\u1311\u1316-\u1317\u135b-\u135c\u137d-\u137f\u139a-\u139f\u13f5-\u13ff\u169d-\u169f\u16f1-\u16ff\u170d\u1715-\u171f\u1737-\u173f\u1754-\u175f\u176d\u1771\u1774-\u177f\u17de-\u17df\u17ea-\u17ef\u17fa-\u17ff\u180f\u181a-\u181f\u1878-\u187f\u18ab-\u18af\u18f6-\u18ff\u191d-\u191f\u192c-\u192f\u193c-\u193f\u1941-\u1943\u196e-\u196f\u1975-\u197f\u19ac-\u19af\u19ca-\u19cf\u19db-\u19dd\u1a1c-\u1a1d\u1a5f\u1a7d-\u1a7e\u1a8a-\u1a8f\u1a9a-\u1a9f\u1aae-\u1aff\u1b4c-\u1b4f\u1b7d-\u1b7f\u1bf4-\u1bfb\u1c38-\u1c3a\u1c4a-\u1c4c\u1c80-\u1cbf\u1cc8-\u1ccf\u1cf7-\u1cff\u1de7-\u1dfb\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fc5\u1fd4-\u1fd5\u1fdc\u1ff0-\u1ff1\u1ff5\u1fff\u2065\u2072-\u2073\u208f\u209d-\u209f\u20bb-\u20cf\u20f1-\u20ff\u218a-\u218f\u23f4-\u23ff\u2427-\u243f\u244b-\u245f\u2700\u2b4d-\u2b4f\u2b5a-\u2bff\u2c2f\u2c5f\u2cf4-\u2cf8\u2d26\u2d28-\u2d2c\u2d2e-\u2d2f\u2d68-\u2d6e\u2d71-\u2d7e\u2d97-\u2d9f\u2da7\u2daf\u2db7\u2dbf\u2dc7\u2dcf\u2dd7\u2ddf\u2e3c-\u2e7f\u2e9a\u2ef4-\u2eff\u2fd6-\u2fef\u2ffc-\u2fff\u3040\u3097-\u3098\u3100-\u3104\u312e-\u3130\u318f\u31bb-\u31bf\u31e4-\u31ef\u321f\u32ff\u4db6-\u4dbf\u9fcd-\u9fff\ua48d-\ua48f\ua4c7-\ua4cf\ua62c-\ua63f\ua698-\ua69e\ua6f8-\ua6ff\ua78f\ua794-\ua79f\ua7ab-\ua7f7\ua82c-\ua82f\ua83a-\ua83f\ua878-\ua87f\ua8c5-\ua8cd\ua8da-\ua8df\ua8fc-\ua8ff\ua954-\ua95e\ua97d-\ua97f\ua9ce\ua9da-\ua9dd\ua9e0-\ua9ff\uaa37-\uaa3f\uaa4e-\uaa4f\uaa5a-\uaa5b\uaa7c-\uaa7f\uaac3-\uaada\uaaf7-\uab00\uab07-\uab08\uab0f-\uab10\uab17-\uab1f\uab27\uab2f-\uabbf\uabee-\uabef\uabfa-\uabff\ud7a4-\ud7af\ud7c7-\ud7ca\ud7fc-\ud7ff\ufa6e-\ufa6f\ufada-\ufaff\ufb07-\ufb12\ufb18-\ufb1c\ufb37\ufb3d\ufb3f\ufb42\ufb45\ufbc2-\ufbd2\ufd40-\ufd4f\ufd90-\ufd91\ufdc8-\ufdef\ufdfe-\ufdff\ufe1a-\ufe1f\ufe27-\ufe2f\ufe53\ufe67\ufe6c-\ufe6f\ufe75\ufefd-\ufefe\uff00\uffbf-\uffc1\uffc8-\uffc9\uffd0-\uffd1\uffd8-\uffd9\uffdd-\uffdf\uffe7\uffef-\ufff8\ufffe-\uffff' +Cn = u'\u0378-\u0379\u0380-\u0383\u038b\u038d\u03a2\u0530\u0557-\u0558\u058b-\u058c\u0590\u05c8-\u05cf\u05eb-\u05ee\u05f5-\u05ff\u061d\u070e\u074b-\u074c\u07b2-\u07bf\u07fb-\u07fc\u082e-\u082f\u083f\u085c-\u085d\u085f\u086b-\u089f\u08b5\u08be-\u08d2\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09c5-\u09c6\u09c9-\u09ca\u09cf-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09ff-\u0a00\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a50\u0a52-\u0a58\u0a5d\u0a5f-\u0a65\u0a77-\u0a80\u0a84\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0acf\u0ad1-\u0adf\u0ae4-\u0ae5\u0af2-\u0af8\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34\u0b3a-\u0b3b\u0b45-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b64-\u0b65\u0b78-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bcf\u0bd1-\u0bd6\u0bd8-\u0be5\u0bfb-\u0bff\u0c0d\u0c11\u0c29\u0c3a-\u0c3c\u0c45\u0c49\u0c4e-\u0c54\u0c57\u0c5b-\u0c5f\u0c64-\u0c65\u0c70-\u0c77\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbb\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce4-\u0ce5\u0cf0\u0cf3-\u0cff\u0d04\u0d0d\u0d11\u0d45\u0d49\u0d50-\u0d53\u0d64-\u0d65\u0d80-\u0d81\u0d84\u0d97-\u0d99\u0db2\u0dbc\u0dbe-\u0dbf\u0dc7-\u0dc9\u0dcb-\u0dce\u0dd5\u0dd7\u0de0-\u0de5\u0df0-\u0df1\u0df5-\u0e00\u0e3b-\u0e3e\u0e5c-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0edb\u0ee0-\u0eff\u0f48\u0f6d-\u0f70\u0f98\u0fbd\u0fcd\u0fdb-\u0fff\u10c6\u10c8-\u10cc\u10ce-\u10cf\u1249\u124e-\u124f\u1257\u1259\u125e-\u125f\u1289\u128e-\u128f\u12b1\u12b6-\u12b7\u12bf\u12c1\u12c6-\u12c7\u12d7\u1311\u1316-\u1317\u135b-\u135c\u137d-\u137f\u139a-\u139f\u13f6-\u13f7\u13fe-\u13ff\u169d-\u169f\u16f9-\u16ff\u170d\u1715-\u171f\u1737-\u173f\u1754-\u175f\u176d\u1771\u1774-\u177f\u17de-\u17df\u17ea-\u17ef\u17fa-\u17ff\u180f\u181a-\u181f\u1879-\u187f\u18ab-\u18af\u18f6-\u18ff\u191f\u192c-\u192f\u193c-\u193f\u1941-\u1943\u196e-\u196f\u1975-\u197f\u19ac-\u19af\u19ca-\u19cf\u19db-\u19dd\u1a1c-\u1a1d\u1a5f\u1a7d-\u1a7e\u1a8a-\u1a8f\u1a9a-\u1a9f\u1aae-\u1aaf\u1abf-\u1aff\u1b4c-\u1b4f\u1b7d-\u1b7f\u1bf4-\u1bfb\u1c38-\u1c3a\u1c4a-\u1c4c\u1c89-\u1c8f\u1cbb-\u1cbc\u1cc8-\u1ccf\u1cfa-\u1cff\u1dfa\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fc5\u1fd4-\u1fd5\u1fdc\u1ff0-\u1ff1\u1ff5\u1fff\u2065\u2072-\u2073\u208f\u209d-\u209f\u20c0-\u20cf\u20f1-\u20ff\u218c-\u218f\u2427-\u243f\u244b-\u245f\u2b74-\u2b75\u2b96-\u2b97\u2bc9\u2bff\u2c2f\u2c5f\u2cf4-\u2cf8\u2d26\u2d28-\u2d2c\u2d2e-\u2d2f\u2d68-\u2d6e\u2d71-\u2d7e\u2d97-\u2d9f\u2da7\u2daf\u2db7\u2dbf\u2dc7\u2dcf\u2dd7\u2ddf\u2e4f-\u2e7f\u2e9a\u2ef4-\u2eff\u2fd6-\u2fef\u2ffc-\u2fff\u3040\u3097-\u3098\u3100-\u3104\u3130\u318f\u31bb-\u31bf\u31e4-\u31ef\u321f\u32ff\u4db6-\u4dbf\u9ff0-\u9fff\ua48d-\ua48f\ua4c7-\ua4cf\ua62c-\ua63f\ua6f8-\ua6ff\ua7ba-\ua7f6\ua82c-\ua82f\ua83a-\ua83f\ua878-\ua87f\ua8c6-\ua8cd\ua8da-\ua8df\ua954-\ua95e\ua97d-\ua97f\ua9ce\ua9da-\ua9dd\ua9ff\uaa37-\uaa3f\uaa4e-\uaa4f\uaa5a-\uaa5b\uaac3-\uaada\uaaf7-\uab00\uab07-\uab08\uab0f-\uab10\uab17-\uab1f\uab27\uab2f\uab66-\uab6f\uabee-\uabef\uabfa-\uabff\ud7a4-\ud7af\ud7c7-\ud7ca\ud7fc-\ud7ff\ufa6e-\ufa6f\ufada-\ufaff\ufb07-\ufb12\ufb18-\ufb1c\ufb37\ufb3d\ufb3f\ufb42\ufb45\ufbc2-\ufbd2\ufd40-\ufd4f\ufd90-\ufd91\ufdc8-\ufdef\ufdfe-\ufdff\ufe1a-\ufe1f\ufe53\ufe67\ufe6c-\ufe6f\ufe75\ufefd-\ufefe\uff00\uffbf-\uffc1\uffc8-\uffc9\uffd0-\uffd1\uffd8-\uffd9\uffdd-\uffdf\uffe7\uffef-\ufff8\ufffe-\uffff' Co = u'\ue000-\uf8ff' @@ -27,49 +27,49 @@ try: except UnicodeDecodeError: Cs = '' # Jython can't handle isolated surrogates -Ll = u'a-z\xb5\xdf-\xf6\xf8-\xff\u0101\u0103\u0105\u0107\u0109\u010b\u010d\u010f\u0111\u0113\u0115\u0117\u0119\u011b\u011d\u011f\u0121\u0123\u0125\u0127\u0129\u012b\u012d\u012f\u0131\u0133\u0135\u0137-\u0138\u013a\u013c\u013e\u0140\u0142\u0144\u0146\u0148-\u0149\u014b\u014d\u014f\u0151\u0153\u0155\u0157\u0159\u015b\u015d\u015f\u0161\u0163\u0165\u0167\u0169\u016b\u016d\u016f\u0171\u0173\u0175\u0177\u017a\u017c\u017e-\u0180\u0183\u0185\u0188\u018c-\u018d\u0192\u0195\u0199-\u019b\u019e\u01a1\u01a3\u01a5\u01a8\u01aa-\u01ab\u01ad\u01b0\u01b4\u01b6\u01b9-\u01ba\u01bd-\u01bf\u01c6\u01c9\u01cc\u01ce\u01d0\u01d2\u01d4\u01d6\u01d8\u01da\u01dc-\u01dd\u01df\u01e1\u01e3\u01e5\u01e7\u01e9\u01eb\u01ed\u01ef-\u01f0\u01f3\u01f5\u01f9\u01fb\u01fd\u01ff\u0201\u0203\u0205\u0207\u0209\u020b\u020d\u020f\u0211\u0213\u0215\u0217\u0219\u021b\u021d\u021f\u0221\u0223\u0225\u0227\u0229\u022b\u022d\u022f\u0231\u0233-\u0239\u023c\u023f-\u0240\u0242\u0247\u0249\u024b\u024d\u024f-\u0293\u0295-\u02af\u0371\u0373\u0377\u037b-\u037d\u0390\u03ac-\u03ce\u03d0-\u03d1\u03d5-\u03d7\u03d9\u03db\u03dd\u03df\u03e1\u03e3\u03e5\u03e7\u03e9\u03eb\u03ed\u03ef-\u03f3\u03f5\u03f8\u03fb-\u03fc\u0430-\u045f\u0461\u0463\u0465\u0467\u0469\u046b\u046d\u046f\u0471\u0473\u0475\u0477\u0479\u047b\u047d\u047f\u0481\u048b\u048d\u048f\u0491\u0493\u0495\u0497\u0499\u049b\u049d\u049f\u04a1\u04a3\u04a5\u04a7\u04a9\u04ab\u04ad\u04af\u04b1\u04b3\u04b5\u04b7\u04b9\u04bb\u04bd\u04bf\u04c2\u04c4\u04c6\u04c8\u04ca\u04cc\u04ce-\u04cf\u04d1\u04d3\u04d5\u04d7\u04d9\u04db\u04dd\u04df\u04e1\u04e3\u04e5\u04e7\u04e9\u04eb\u04ed\u04ef\u04f1\u04f3\u04f5\u04f7\u04f9\u04fb\u04fd\u04ff\u0501\u0503\u0505\u0507\u0509\u050b\u050d\u050f\u0511\u0513\u0515\u0517\u0519\u051b\u051d\u051f\u0521\u0523\u0525\u0527\u0561-\u0587\u1d00-\u1d2b\u1d6b-\u1d77\u1d79-\u1d9a\u1e01\u1e03\u1e05\u1e07\u1e09\u1e0b\u1e0d\u1e0f\u1e11\u1e13\u1e15\u1e17\u1e19\u1e1b\u1e1d\u1e1f\u1e21\u1e23\u1e25\u1e27\u1e29\u1e2b\u1e2d\u1e2f\u1e31\u1e33\u1e35\u1e37\u1e39\u1e3b\u1e3d\u1e3f\u1e41\u1e43\u1e45\u1e47\u1e49\u1e4b\u1e4d\u1e4f\u1e51\u1e53\u1e55\u1e57\u1e59\u1e5b\u1e5d\u1e5f\u1e61\u1e63\u1e65\u1e67\u1e69\u1e6b\u1e6d\u1e6f\u1e71\u1e73\u1e75\u1e77\u1e79\u1e7b\u1e7d\u1e7f\u1e81\u1e83\u1e85\u1e87\u1e89\u1e8b\u1e8d\u1e8f\u1e91\u1e93\u1e95-\u1e9d\u1e9f\u1ea1\u1ea3\u1ea5\u1ea7\u1ea9\u1eab\u1ead\u1eaf\u1eb1\u1eb3\u1eb5\u1eb7\u1eb9\u1ebb\u1ebd\u1ebf\u1ec1\u1ec3\u1ec5\u1ec7\u1ec9\u1ecb\u1ecd\u1ecf\u1ed1\u1ed3\u1ed5\u1ed7\u1ed9\u1edb\u1edd\u1edf\u1ee1\u1ee3\u1ee5\u1ee7\u1ee9\u1eeb\u1eed\u1eef\u1ef1\u1ef3\u1ef5\u1ef7\u1ef9\u1efb\u1efd\u1eff-\u1f07\u1f10-\u1f15\u1f20-\u1f27\u1f30-\u1f37\u1f40-\u1f45\u1f50-\u1f57\u1f60-\u1f67\u1f70-\u1f7d\u1f80-\u1f87\u1f90-\u1f97\u1fa0-\u1fa7\u1fb0-\u1fb4\u1fb6-\u1fb7\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fc7\u1fd0-\u1fd3\u1fd6-\u1fd7\u1fe0-\u1fe7\u1ff2-\u1ff4\u1ff6-\u1ff7\u210a\u210e-\u210f\u2113\u212f\u2134\u2139\u213c-\u213d\u2146-\u2149\u214e\u2184\u2c30-\u2c5e\u2c61\u2c65-\u2c66\u2c68\u2c6a\u2c6c\u2c71\u2c73-\u2c74\u2c76-\u2c7b\u2c81\u2c83\u2c85\u2c87\u2c89\u2c8b\u2c8d\u2c8f\u2c91\u2c93\u2c95\u2c97\u2c99\u2c9b\u2c9d\u2c9f\u2ca1\u2ca3\u2ca5\u2ca7\u2ca9\u2cab\u2cad\u2caf\u2cb1\u2cb3\u2cb5\u2cb7\u2cb9\u2cbb\u2cbd\u2cbf\u2cc1\u2cc3\u2cc5\u2cc7\u2cc9\u2ccb\u2ccd\u2ccf\u2cd1\u2cd3\u2cd5\u2cd7\u2cd9\u2cdb\u2cdd\u2cdf\u2ce1\u2ce3-\u2ce4\u2cec\u2cee\u2cf3\u2d00-\u2d25\u2d27\u2d2d\ua641\ua643\ua645\ua647\ua649\ua64b\ua64d\ua64f\ua651\ua653\ua655\ua657\ua659\ua65b\ua65d\ua65f\ua661\ua663\ua665\ua667\ua669\ua66b\ua66d\ua681\ua683\ua685\ua687\ua689\ua68b\ua68d\ua68f\ua691\ua693\ua695\ua697\ua723\ua725\ua727\ua729\ua72b\ua72d\ua72f-\ua731\ua733\ua735\ua737\ua739\ua73b\ua73d\ua73f\ua741\ua743\ua745\ua747\ua749\ua74b\ua74d\ua74f\ua751\ua753\ua755\ua757\ua759\ua75b\ua75d\ua75f\ua761\ua763\ua765\ua767\ua769\ua76b\ua76d\ua76f\ua771-\ua778\ua77a\ua77c\ua77f\ua781\ua783\ua785\ua787\ua78c\ua78e\ua791\ua793\ua7a1\ua7a3\ua7a5\ua7a7\ua7a9\ua7fa\ufb00-\ufb06\ufb13-\ufb17\uff41-\uff5a' +Ll = u'a-z\xb5\xdf-\xf6\xf8-\xff\u0101\u0103\u0105\u0107\u0109\u010b\u010d\u010f\u0111\u0113\u0115\u0117\u0119\u011b\u011d\u011f\u0121\u0123\u0125\u0127\u0129\u012b\u012d\u012f\u0131\u0133\u0135\u0137-\u0138\u013a\u013c\u013e\u0140\u0142\u0144\u0146\u0148-\u0149\u014b\u014d\u014f\u0151\u0153\u0155\u0157\u0159\u015b\u015d\u015f\u0161\u0163\u0165\u0167\u0169\u016b\u016d\u016f\u0171\u0173\u0175\u0177\u017a\u017c\u017e-\u0180\u0183\u0185\u0188\u018c-\u018d\u0192\u0195\u0199-\u019b\u019e\u01a1\u01a3\u01a5\u01a8\u01aa-\u01ab\u01ad\u01b0\u01b4\u01b6\u01b9-\u01ba\u01bd-\u01bf\u01c6\u01c9\u01cc\u01ce\u01d0\u01d2\u01d4\u01d6\u01d8\u01da\u01dc-\u01dd\u01df\u01e1\u01e3\u01e5\u01e7\u01e9\u01eb\u01ed\u01ef-\u01f0\u01f3\u01f5\u01f9\u01fb\u01fd\u01ff\u0201\u0203\u0205\u0207\u0209\u020b\u020d\u020f\u0211\u0213\u0215\u0217\u0219\u021b\u021d\u021f\u0221\u0223\u0225\u0227\u0229\u022b\u022d\u022f\u0231\u0233-\u0239\u023c\u023f-\u0240\u0242\u0247\u0249\u024b\u024d\u024f-\u0293\u0295-\u02af\u0371\u0373\u0377\u037b-\u037d\u0390\u03ac-\u03ce\u03d0-\u03d1\u03d5-\u03d7\u03d9\u03db\u03dd\u03df\u03e1\u03e3\u03e5\u03e7\u03e9\u03eb\u03ed\u03ef-\u03f3\u03f5\u03f8\u03fb-\u03fc\u0430-\u045f\u0461\u0463\u0465\u0467\u0469\u046b\u046d\u046f\u0471\u0473\u0475\u0477\u0479\u047b\u047d\u047f\u0481\u048b\u048d\u048f\u0491\u0493\u0495\u0497\u0499\u049b\u049d\u049f\u04a1\u04a3\u04a5\u04a7\u04a9\u04ab\u04ad\u04af\u04b1\u04b3\u04b5\u04b7\u04b9\u04bb\u04bd\u04bf\u04c2\u04c4\u04c6\u04c8\u04ca\u04cc\u04ce-\u04cf\u04d1\u04d3\u04d5\u04d7\u04d9\u04db\u04dd\u04df\u04e1\u04e3\u04e5\u04e7\u04e9\u04eb\u04ed\u04ef\u04f1\u04f3\u04f5\u04f7\u04f9\u04fb\u04fd\u04ff\u0501\u0503\u0505\u0507\u0509\u050b\u050d\u050f\u0511\u0513\u0515\u0517\u0519\u051b\u051d\u051f\u0521\u0523\u0525\u0527\u0529\u052b\u052d\u052f\u0560-\u0588\u10d0-\u10fa\u10fd-\u10ff\u13f8-\u13fd\u1c80-\u1c88\u1d00-\u1d2b\u1d6b-\u1d77\u1d79-\u1d9a\u1e01\u1e03\u1e05\u1e07\u1e09\u1e0b\u1e0d\u1e0f\u1e11\u1e13\u1e15\u1e17\u1e19\u1e1b\u1e1d\u1e1f\u1e21\u1e23\u1e25\u1e27\u1e29\u1e2b\u1e2d\u1e2f\u1e31\u1e33\u1e35\u1e37\u1e39\u1e3b\u1e3d\u1e3f\u1e41\u1e43\u1e45\u1e47\u1e49\u1e4b\u1e4d\u1e4f\u1e51\u1e53\u1e55\u1e57\u1e59\u1e5b\u1e5d\u1e5f\u1e61\u1e63\u1e65\u1e67\u1e69\u1e6b\u1e6d\u1e6f\u1e71\u1e73\u1e75\u1e77\u1e79\u1e7b\u1e7d\u1e7f\u1e81\u1e83\u1e85\u1e87\u1e89\u1e8b\u1e8d\u1e8f\u1e91\u1e93\u1e95-\u1e9d\u1e9f\u1ea1\u1ea3\u1ea5\u1ea7\u1ea9\u1eab\u1ead\u1eaf\u1eb1\u1eb3\u1eb5\u1eb7\u1eb9\u1ebb\u1ebd\u1ebf\u1ec1\u1ec3\u1ec5\u1ec7\u1ec9\u1ecb\u1ecd\u1ecf\u1ed1\u1ed3\u1ed5\u1ed7\u1ed9\u1edb\u1edd\u1edf\u1ee1\u1ee3\u1ee5\u1ee7\u1ee9\u1eeb\u1eed\u1eef\u1ef1\u1ef3\u1ef5\u1ef7\u1ef9\u1efb\u1efd\u1eff-\u1f07\u1f10-\u1f15\u1f20-\u1f27\u1f30-\u1f37\u1f40-\u1f45\u1f50-\u1f57\u1f60-\u1f67\u1f70-\u1f7d\u1f80-\u1f87\u1f90-\u1f97\u1fa0-\u1fa7\u1fb0-\u1fb4\u1fb6-\u1fb7\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fc7\u1fd0-\u1fd3\u1fd6-\u1fd7\u1fe0-\u1fe7\u1ff2-\u1ff4\u1ff6-\u1ff7\u210a\u210e-\u210f\u2113\u212f\u2134\u2139\u213c-\u213d\u2146-\u2149\u214e\u2184\u2c30-\u2c5e\u2c61\u2c65-\u2c66\u2c68\u2c6a\u2c6c\u2c71\u2c73-\u2c74\u2c76-\u2c7b\u2c81\u2c83\u2c85\u2c87\u2c89\u2c8b\u2c8d\u2c8f\u2c91\u2c93\u2c95\u2c97\u2c99\u2c9b\u2c9d\u2c9f\u2ca1\u2ca3\u2ca5\u2ca7\u2ca9\u2cab\u2cad\u2caf\u2cb1\u2cb3\u2cb5\u2cb7\u2cb9\u2cbb\u2cbd\u2cbf\u2cc1\u2cc3\u2cc5\u2cc7\u2cc9\u2ccb\u2ccd\u2ccf\u2cd1\u2cd3\u2cd5\u2cd7\u2cd9\u2cdb\u2cdd\u2cdf\u2ce1\u2ce3-\u2ce4\u2cec\u2cee\u2cf3\u2d00-\u2d25\u2d27\u2d2d\ua641\ua643\ua645\ua647\ua649\ua64b\ua64d\ua64f\ua651\ua653\ua655\ua657\ua659\ua65b\ua65d\ua65f\ua661\ua663\ua665\ua667\ua669\ua66b\ua66d\ua681\ua683\ua685\ua687\ua689\ua68b\ua68d\ua68f\ua691\ua693\ua695\ua697\ua699\ua69b\ua723\ua725\ua727\ua729\ua72b\ua72d\ua72f-\ua731\ua733\ua735\ua737\ua739\ua73b\ua73d\ua73f\ua741\ua743\ua745\ua747\ua749\ua74b\ua74d\ua74f\ua751\ua753\ua755\ua757\ua759\ua75b\ua75d\ua75f\ua761\ua763\ua765\ua767\ua769\ua76b\ua76d\ua76f\ua771-\ua778\ua77a\ua77c\ua77f\ua781\ua783\ua785\ua787\ua78c\ua78e\ua791\ua793-\ua795\ua797\ua799\ua79b\ua79d\ua79f\ua7a1\ua7a3\ua7a5\ua7a7\ua7a9\ua7af\ua7b5\ua7b7\ua7b9\ua7fa\uab30-\uab5a\uab60-\uab65\uab70-\uabbf\ufb00-\ufb06\ufb13-\ufb17\uff41-\uff5a' -Lm = u'\u02b0-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0374\u037a\u0559\u0640\u06e5-\u06e6\u07f4-\u07f5\u07fa\u081a\u0824\u0828\u0971\u0e46\u0ec6\u10fc\u17d7\u1843\u1aa7\u1c78-\u1c7d\u1d2c-\u1d6a\u1d78\u1d9b-\u1dbf\u2071\u207f\u2090-\u209c\u2c7c-\u2c7d\u2d6f\u2e2f\u3005\u3031-\u3035\u303b\u309d-\u309e\u30fc-\u30fe\ua015\ua4f8-\ua4fd\ua60c\ua67f\ua717-\ua71f\ua770\ua788\ua7f8-\ua7f9\ua9cf\uaa70\uaadd\uaaf3-\uaaf4\uff70\uff9e-\uff9f' +Lm = u'\u02b0-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0374\u037a\u0559\u0640\u06e5-\u06e6\u07f4-\u07f5\u07fa\u081a\u0824\u0828\u0971\u0e46\u0ec6\u10fc\u17d7\u1843\u1aa7\u1c78-\u1c7d\u1d2c-\u1d6a\u1d78\u1d9b-\u1dbf\u2071\u207f\u2090-\u209c\u2c7c-\u2c7d\u2d6f\u2e2f\u3005\u3031-\u3035\u303b\u309d-\u309e\u30fc-\u30fe\ua015\ua4f8-\ua4fd\ua60c\ua67f\ua69c-\ua69d\ua717-\ua71f\ua770\ua788\ua7f8-\ua7f9\ua9cf\ua9e6\uaa70\uaadd\uaaf3-\uaaf4\uab5c-\uab5f\uff70\uff9e-\uff9f' -Lo = u'\xaa\xba\u01bb\u01c0-\u01c3\u0294\u05d0-\u05ea\u05f0-\u05f2\u0620-\u063f\u0641-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u0800-\u0815\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0972-\u0977\u0979-\u097f\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58-\u0c59\u0c60-\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32-\u0e33\u0e40-\u0e45\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2-\u0eb3\u0ebd\u0ec0-\u0ec4\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10d0-\u10fa\u10fd-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17dc\u1820-\u1842\u1844-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c77\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u2135-\u2138\u2d30-\u2d67\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3006\u303c\u3041-\u3096\u309f\u30a1-\u30fa\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua014\ua016-\ua48c\ua4d0-\ua4f7\ua500-\ua60b\ua610-\ua61f\ua62a-\ua62b\ua66e\ua6a0-\ua6e5\ua7fb-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa6f\uaa71-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadc\uaae0-\uaaea\uaaf2\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff66-\uff6f\uff71-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' +Lo = u'\xaa\xba\u01bb\u01c0-\u01c3\u0294\u05d0-\u05ea\u05ef-\u05f2\u0620-\u063f\u0641-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u0800-\u0815\u0840-\u0858\u0860-\u086a\u08a0-\u08b4\u08b6-\u08bd\u0904-\u0939\u093d\u0950\u0958-\u0961\u0972-\u0980\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u09fc\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0af9\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c39\u0c3d\u0c58-\u0c5a\u0c60-\u0c61\u0c80\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d54-\u0d56\u0d5f-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32-\u0e33\u0e40-\u0e45\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2-\u0eb3\u0ebd\u0ec0-\u0ec4\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u1100-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16f1-\u16f8\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17dc\u1820-\u1842\u1844-\u1878\u1880-\u1884\u1887-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191e\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u1a00-\u1a16\u1a20-\u1a54\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c77\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u2135-\u2138\u2d30-\u2d67\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3006\u303c\u3041-\u3096\u309f\u30a1-\u30fa\u30ff\u3105-\u312f\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fef\ua000-\ua014\ua016-\ua48c\ua4d0-\ua4f7\ua500-\ua60b\ua610-\ua61f\ua62a-\ua62b\ua66e\ua6a0-\ua6e5\ua78f\ua7f7\ua7fb-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua8fd-\ua8fe\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9e0-\ua9e4\ua9e7-\ua9ef\ua9fa-\ua9fe\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa6f\uaa71-\uaa76\uaa7a\uaa7e-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadc\uaae0-\uaaea\uaaf2\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff66-\uff6f\uff71-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' Lt = u'\u01c5\u01c8\u01cb\u01f2\u1f88-\u1f8f\u1f98-\u1f9f\u1fa8-\u1faf\u1fbc\u1fcc\u1ffc' -Lu = u'A-Z\xc0-\xd6\xd8-\xde\u0100\u0102\u0104\u0106\u0108\u010a\u010c\u010e\u0110\u0112\u0114\u0116\u0118\u011a\u011c\u011e\u0120\u0122\u0124\u0126\u0128\u012a\u012c\u012e\u0130\u0132\u0134\u0136\u0139\u013b\u013d\u013f\u0141\u0143\u0145\u0147\u014a\u014c\u014e\u0150\u0152\u0154\u0156\u0158\u015a\u015c\u015e\u0160\u0162\u0164\u0166\u0168\u016a\u016c\u016e\u0170\u0172\u0174\u0176\u0178-\u0179\u017b\u017d\u0181-\u0182\u0184\u0186-\u0187\u0189-\u018b\u018e-\u0191\u0193-\u0194\u0196-\u0198\u019c-\u019d\u019f-\u01a0\u01a2\u01a4\u01a6-\u01a7\u01a9\u01ac\u01ae-\u01af\u01b1-\u01b3\u01b5\u01b7-\u01b8\u01bc\u01c4\u01c7\u01ca\u01cd\u01cf\u01d1\u01d3\u01d5\u01d7\u01d9\u01db\u01de\u01e0\u01e2\u01e4\u01e6\u01e8\u01ea\u01ec\u01ee\u01f1\u01f4\u01f6-\u01f8\u01fa\u01fc\u01fe\u0200\u0202\u0204\u0206\u0208\u020a\u020c\u020e\u0210\u0212\u0214\u0216\u0218\u021a\u021c\u021e\u0220\u0222\u0224\u0226\u0228\u022a\u022c\u022e\u0230\u0232\u023a-\u023b\u023d-\u023e\u0241\u0243-\u0246\u0248\u024a\u024c\u024e\u0370\u0372\u0376\u0386\u0388-\u038a\u038c\u038e-\u038f\u0391-\u03a1\u03a3-\u03ab\u03cf\u03d2-\u03d4\u03d8\u03da\u03dc\u03de\u03e0\u03e2\u03e4\u03e6\u03e8\u03ea\u03ec\u03ee\u03f4\u03f7\u03f9-\u03fa\u03fd-\u042f\u0460\u0462\u0464\u0466\u0468\u046a\u046c\u046e\u0470\u0472\u0474\u0476\u0478\u047a\u047c\u047e\u0480\u048a\u048c\u048e\u0490\u0492\u0494\u0496\u0498\u049a\u049c\u049e\u04a0\u04a2\u04a4\u04a6\u04a8\u04aa\u04ac\u04ae\u04b0\u04b2\u04b4\u04b6\u04b8\u04ba\u04bc\u04be\u04c0-\u04c1\u04c3\u04c5\u04c7\u04c9\u04cb\u04cd\u04d0\u04d2\u04d4\u04d6\u04d8\u04da\u04dc\u04de\u04e0\u04e2\u04e4\u04e6\u04e8\u04ea\u04ec\u04ee\u04f0\u04f2\u04f4\u04f6\u04f8\u04fa\u04fc\u04fe\u0500\u0502\u0504\u0506\u0508\u050a\u050c\u050e\u0510\u0512\u0514\u0516\u0518\u051a\u051c\u051e\u0520\u0522\u0524\u0526\u0531-\u0556\u10a0-\u10c5\u10c7\u10cd\u1e00\u1e02\u1e04\u1e06\u1e08\u1e0a\u1e0c\u1e0e\u1e10\u1e12\u1e14\u1e16\u1e18\u1e1a\u1e1c\u1e1e\u1e20\u1e22\u1e24\u1e26\u1e28\u1e2a\u1e2c\u1e2e\u1e30\u1e32\u1e34\u1e36\u1e38\u1e3a\u1e3c\u1e3e\u1e40\u1e42\u1e44\u1e46\u1e48\u1e4a\u1e4c\u1e4e\u1e50\u1e52\u1e54\u1e56\u1e58\u1e5a\u1e5c\u1e5e\u1e60\u1e62\u1e64\u1e66\u1e68\u1e6a\u1e6c\u1e6e\u1e70\u1e72\u1e74\u1e76\u1e78\u1e7a\u1e7c\u1e7e\u1e80\u1e82\u1e84\u1e86\u1e88\u1e8a\u1e8c\u1e8e\u1e90\u1e92\u1e94\u1e9e\u1ea0\u1ea2\u1ea4\u1ea6\u1ea8\u1eaa\u1eac\u1eae\u1eb0\u1eb2\u1eb4\u1eb6\u1eb8\u1eba\u1ebc\u1ebe\u1ec0\u1ec2\u1ec4\u1ec6\u1ec8\u1eca\u1ecc\u1ece\u1ed0\u1ed2\u1ed4\u1ed6\u1ed8\u1eda\u1edc\u1ede\u1ee0\u1ee2\u1ee4\u1ee6\u1ee8\u1eea\u1eec\u1eee\u1ef0\u1ef2\u1ef4\u1ef6\u1ef8\u1efa\u1efc\u1efe\u1f08-\u1f0f\u1f18-\u1f1d\u1f28-\u1f2f\u1f38-\u1f3f\u1f48-\u1f4d\u1f59\u1f5b\u1f5d\u1f5f\u1f68-\u1f6f\u1fb8-\u1fbb\u1fc8-\u1fcb\u1fd8-\u1fdb\u1fe8-\u1fec\u1ff8-\u1ffb\u2102\u2107\u210b-\u210d\u2110-\u2112\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u2130-\u2133\u213e-\u213f\u2145\u2183\u2c00-\u2c2e\u2c60\u2c62-\u2c64\u2c67\u2c69\u2c6b\u2c6d-\u2c70\u2c72\u2c75\u2c7e-\u2c80\u2c82\u2c84\u2c86\u2c88\u2c8a\u2c8c\u2c8e\u2c90\u2c92\u2c94\u2c96\u2c98\u2c9a\u2c9c\u2c9e\u2ca0\u2ca2\u2ca4\u2ca6\u2ca8\u2caa\u2cac\u2cae\u2cb0\u2cb2\u2cb4\u2cb6\u2cb8\u2cba\u2cbc\u2cbe\u2cc0\u2cc2\u2cc4\u2cc6\u2cc8\u2cca\u2ccc\u2cce\u2cd0\u2cd2\u2cd4\u2cd6\u2cd8\u2cda\u2cdc\u2cde\u2ce0\u2ce2\u2ceb\u2ced\u2cf2\ua640\ua642\ua644\ua646\ua648\ua64a\ua64c\ua64e\ua650\ua652\ua654\ua656\ua658\ua65a\ua65c\ua65e\ua660\ua662\ua664\ua666\ua668\ua66a\ua66c\ua680\ua682\ua684\ua686\ua688\ua68a\ua68c\ua68e\ua690\ua692\ua694\ua696\ua722\ua724\ua726\ua728\ua72a\ua72c\ua72e\ua732\ua734\ua736\ua738\ua73a\ua73c\ua73e\ua740\ua742\ua744\ua746\ua748\ua74a\ua74c\ua74e\ua750\ua752\ua754\ua756\ua758\ua75a\ua75c\ua75e\ua760\ua762\ua764\ua766\ua768\ua76a\ua76c\ua76e\ua779\ua77b\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\ua78d\ua790\ua792\ua7a0\ua7a2\ua7a4\ua7a6\ua7a8\ua7aa\uff21-\uff3a' +Lu = u'A-Z\xc0-\xd6\xd8-\xde\u0100\u0102\u0104\u0106\u0108\u010a\u010c\u010e\u0110\u0112\u0114\u0116\u0118\u011a\u011c\u011e\u0120\u0122\u0124\u0126\u0128\u012a\u012c\u012e\u0130\u0132\u0134\u0136\u0139\u013b\u013d\u013f\u0141\u0143\u0145\u0147\u014a\u014c\u014e\u0150\u0152\u0154\u0156\u0158\u015a\u015c\u015e\u0160\u0162\u0164\u0166\u0168\u016a\u016c\u016e\u0170\u0172\u0174\u0176\u0178-\u0179\u017b\u017d\u0181-\u0182\u0184\u0186-\u0187\u0189-\u018b\u018e-\u0191\u0193-\u0194\u0196-\u0198\u019c-\u019d\u019f-\u01a0\u01a2\u01a4\u01a6-\u01a7\u01a9\u01ac\u01ae-\u01af\u01b1-\u01b3\u01b5\u01b7-\u01b8\u01bc\u01c4\u01c7\u01ca\u01cd\u01cf\u01d1\u01d3\u01d5\u01d7\u01d9\u01db\u01de\u01e0\u01e2\u01e4\u01e6\u01e8\u01ea\u01ec\u01ee\u01f1\u01f4\u01f6-\u01f8\u01fa\u01fc\u01fe\u0200\u0202\u0204\u0206\u0208\u020a\u020c\u020e\u0210\u0212\u0214\u0216\u0218\u021a\u021c\u021e\u0220\u0222\u0224\u0226\u0228\u022a\u022c\u022e\u0230\u0232\u023a-\u023b\u023d-\u023e\u0241\u0243-\u0246\u0248\u024a\u024c\u024e\u0370\u0372\u0376\u037f\u0386\u0388-\u038a\u038c\u038e-\u038f\u0391-\u03a1\u03a3-\u03ab\u03cf\u03d2-\u03d4\u03d8\u03da\u03dc\u03de\u03e0\u03e2\u03e4\u03e6\u03e8\u03ea\u03ec\u03ee\u03f4\u03f7\u03f9-\u03fa\u03fd-\u042f\u0460\u0462\u0464\u0466\u0468\u046a\u046c\u046e\u0470\u0472\u0474\u0476\u0478\u047a\u047c\u047e\u0480\u048a\u048c\u048e\u0490\u0492\u0494\u0496\u0498\u049a\u049c\u049e\u04a0\u04a2\u04a4\u04a6\u04a8\u04aa\u04ac\u04ae\u04b0\u04b2\u04b4\u04b6\u04b8\u04ba\u04bc\u04be\u04c0-\u04c1\u04c3\u04c5\u04c7\u04c9\u04cb\u04cd\u04d0\u04d2\u04d4\u04d6\u04d8\u04da\u04dc\u04de\u04e0\u04e2\u04e4\u04e6\u04e8\u04ea\u04ec\u04ee\u04f0\u04f2\u04f4\u04f6\u04f8\u04fa\u04fc\u04fe\u0500\u0502\u0504\u0506\u0508\u050a\u050c\u050e\u0510\u0512\u0514\u0516\u0518\u051a\u051c\u051e\u0520\u0522\u0524\u0526\u0528\u052a\u052c\u052e\u0531-\u0556\u10a0-\u10c5\u10c7\u10cd\u13a0-\u13f5\u1c90-\u1cba\u1cbd-\u1cbf\u1e00\u1e02\u1e04\u1e06\u1e08\u1e0a\u1e0c\u1e0e\u1e10\u1e12\u1e14\u1e16\u1e18\u1e1a\u1e1c\u1e1e\u1e20\u1e22\u1e24\u1e26\u1e28\u1e2a\u1e2c\u1e2e\u1e30\u1e32\u1e34\u1e36\u1e38\u1e3a\u1e3c\u1e3e\u1e40\u1e42\u1e44\u1e46\u1e48\u1e4a\u1e4c\u1e4e\u1e50\u1e52\u1e54\u1e56\u1e58\u1e5a\u1e5c\u1e5e\u1e60\u1e62\u1e64\u1e66\u1e68\u1e6a\u1e6c\u1e6e\u1e70\u1e72\u1e74\u1e76\u1e78\u1e7a\u1e7c\u1e7e\u1e80\u1e82\u1e84\u1e86\u1e88\u1e8a\u1e8c\u1e8e\u1e90\u1e92\u1e94\u1e9e\u1ea0\u1ea2\u1ea4\u1ea6\u1ea8\u1eaa\u1eac\u1eae\u1eb0\u1eb2\u1eb4\u1eb6\u1eb8\u1eba\u1ebc\u1ebe\u1ec0\u1ec2\u1ec4\u1ec6\u1ec8\u1eca\u1ecc\u1ece\u1ed0\u1ed2\u1ed4\u1ed6\u1ed8\u1eda\u1edc\u1ede\u1ee0\u1ee2\u1ee4\u1ee6\u1ee8\u1eea\u1eec\u1eee\u1ef0\u1ef2\u1ef4\u1ef6\u1ef8\u1efa\u1efc\u1efe\u1f08-\u1f0f\u1f18-\u1f1d\u1f28-\u1f2f\u1f38-\u1f3f\u1f48-\u1f4d\u1f59\u1f5b\u1f5d\u1f5f\u1f68-\u1f6f\u1fb8-\u1fbb\u1fc8-\u1fcb\u1fd8-\u1fdb\u1fe8-\u1fec\u1ff8-\u1ffb\u2102\u2107\u210b-\u210d\u2110-\u2112\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u2130-\u2133\u213e-\u213f\u2145\u2183\u2c00-\u2c2e\u2c60\u2c62-\u2c64\u2c67\u2c69\u2c6b\u2c6d-\u2c70\u2c72\u2c75\u2c7e-\u2c80\u2c82\u2c84\u2c86\u2c88\u2c8a\u2c8c\u2c8e\u2c90\u2c92\u2c94\u2c96\u2c98\u2c9a\u2c9c\u2c9e\u2ca0\u2ca2\u2ca4\u2ca6\u2ca8\u2caa\u2cac\u2cae\u2cb0\u2cb2\u2cb4\u2cb6\u2cb8\u2cba\u2cbc\u2cbe\u2cc0\u2cc2\u2cc4\u2cc6\u2cc8\u2cca\u2ccc\u2cce\u2cd0\u2cd2\u2cd4\u2cd6\u2cd8\u2cda\u2cdc\u2cde\u2ce0\u2ce2\u2ceb\u2ced\u2cf2\ua640\ua642\ua644\ua646\ua648\ua64a\ua64c\ua64e\ua650\ua652\ua654\ua656\ua658\ua65a\ua65c\ua65e\ua660\ua662\ua664\ua666\ua668\ua66a\ua66c\ua680\ua682\ua684\ua686\ua688\ua68a\ua68c\ua68e\ua690\ua692\ua694\ua696\ua698\ua69a\ua722\ua724\ua726\ua728\ua72a\ua72c\ua72e\ua732\ua734\ua736\ua738\ua73a\ua73c\ua73e\ua740\ua742\ua744\ua746\ua748\ua74a\ua74c\ua74e\ua750\ua752\ua754\ua756\ua758\ua75a\ua75c\ua75e\ua760\ua762\ua764\ua766\ua768\ua76a\ua76c\ua76e\ua779\ua77b\ua77d-\ua77e\ua780\ua782\ua784\ua786\ua78b\ua78d\ua790\ua792\ua796\ua798\ua79a\ua79c\ua79e\ua7a0\ua7a2\ua7a4\ua7a6\ua7a8\ua7aa-\ua7ae\ua7b0-\ua7b4\ua7b6\ua7b8\uff21-\uff3a' -Mc = u'\u0903\u093b\u093e-\u0940\u0949-\u094c\u094e-\u094f\u0982-\u0983\u09be-\u09c0\u09c7-\u09c8\u09cb-\u09cc\u09d7\u0a03\u0a3e-\u0a40\u0a83\u0abe-\u0ac0\u0ac9\u0acb-\u0acc\u0b02-\u0b03\u0b3e\u0b40\u0b47-\u0b48\u0b4b-\u0b4c\u0b57\u0bbe-\u0bbf\u0bc1-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcc\u0bd7\u0c01-\u0c03\u0c41-\u0c44\u0c82-\u0c83\u0cbe\u0cc0-\u0cc4\u0cc7-\u0cc8\u0cca-\u0ccb\u0cd5-\u0cd6\u0d02-\u0d03\u0d3e-\u0d40\u0d46-\u0d48\u0d4a-\u0d4c\u0d57\u0d82-\u0d83\u0dcf-\u0dd1\u0dd8-\u0ddf\u0df2-\u0df3\u0f3e-\u0f3f\u0f7f\u102b-\u102c\u1031\u1038\u103b-\u103c\u1056-\u1057\u1062-\u1064\u1067-\u106d\u1083-\u1084\u1087-\u108c\u108f\u109a-\u109c\u17b6\u17be-\u17c5\u17c7-\u17c8\u1923-\u1926\u1929-\u192b\u1930-\u1931\u1933-\u1938\u19b0-\u19c0\u19c8-\u19c9\u1a19-\u1a1a\u1a55\u1a57\u1a61\u1a63-\u1a64\u1a6d-\u1a72\u1b04\u1b35\u1b3b\u1b3d-\u1b41\u1b43-\u1b44\u1b82\u1ba1\u1ba6-\u1ba7\u1baa\u1bac-\u1bad\u1be7\u1bea-\u1bec\u1bee\u1bf2-\u1bf3\u1c24-\u1c2b\u1c34-\u1c35\u1ce1\u1cf2-\u1cf3\u302e-\u302f\ua823-\ua824\ua827\ua880-\ua881\ua8b4-\ua8c3\ua952-\ua953\ua983\ua9b4-\ua9b5\ua9ba-\ua9bb\ua9bd-\ua9c0\uaa2f-\uaa30\uaa33-\uaa34\uaa4d\uaa7b\uaaeb\uaaee-\uaaef\uaaf5\uabe3-\uabe4\uabe6-\uabe7\uabe9-\uabea\uabec' +Mc = u'\u0903\u093b\u093e-\u0940\u0949-\u094c\u094e-\u094f\u0982-\u0983\u09be-\u09c0\u09c7-\u09c8\u09cb-\u09cc\u09d7\u0a03\u0a3e-\u0a40\u0a83\u0abe-\u0ac0\u0ac9\u0acb-\u0acc\u0b02-\u0b03\u0b3e\u0b40\u0b47-\u0b48\u0b4b-\u0b4c\u0b57\u0bbe-\u0bbf\u0bc1-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcc\u0bd7\u0c01-\u0c03\u0c41-\u0c44\u0c82-\u0c83\u0cbe\u0cc0-\u0cc4\u0cc7-\u0cc8\u0cca-\u0ccb\u0cd5-\u0cd6\u0d02-\u0d03\u0d3e-\u0d40\u0d46-\u0d48\u0d4a-\u0d4c\u0d57\u0d82-\u0d83\u0dcf-\u0dd1\u0dd8-\u0ddf\u0df2-\u0df3\u0f3e-\u0f3f\u0f7f\u102b-\u102c\u1031\u1038\u103b-\u103c\u1056-\u1057\u1062-\u1064\u1067-\u106d\u1083-\u1084\u1087-\u108c\u108f\u109a-\u109c\u17b6\u17be-\u17c5\u17c7-\u17c8\u1923-\u1926\u1929-\u192b\u1930-\u1931\u1933-\u1938\u1a19-\u1a1a\u1a55\u1a57\u1a61\u1a63-\u1a64\u1a6d-\u1a72\u1b04\u1b35\u1b3b\u1b3d-\u1b41\u1b43-\u1b44\u1b82\u1ba1\u1ba6-\u1ba7\u1baa\u1be7\u1bea-\u1bec\u1bee\u1bf2-\u1bf3\u1c24-\u1c2b\u1c34-\u1c35\u1ce1\u1cf2-\u1cf3\u1cf7\u302e-\u302f\ua823-\ua824\ua827\ua880-\ua881\ua8b4-\ua8c3\ua952-\ua953\ua983\ua9b4-\ua9b5\ua9ba-\ua9bb\ua9bd-\ua9c0\uaa2f-\uaa30\uaa33-\uaa34\uaa4d\uaa7b\uaa7d\uaaeb\uaaee-\uaaef\uaaf5\uabe3-\uabe4\uabe6-\uabe7\uabe9-\uabea\uabec' -Me = u'\u0488-\u0489\u20dd-\u20e0\u20e2-\u20e4\ua670-\ua672' +Me = u'\u0488-\u0489\u1abe\u20dd-\u20e0\u20e2-\u20e4\ua670-\ua672' -Mn = u'\u0300-\u036f\u0483-\u0487\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u0610-\u061a\u064b-\u065f\u0670\u06d6-\u06dc\u06df-\u06e4\u06e7-\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0859-\u085b\u08e4-\u08fe\u0900-\u0902\u093a\u093c\u0941-\u0948\u094d\u0951-\u0957\u0962-\u0963\u0981\u09bc\u09c1-\u09c4\u09cd\u09e2-\u09e3\u0a01-\u0a02\u0a3c\u0a41-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a70-\u0a71\u0a75\u0a81-\u0a82\u0abc\u0ac1-\u0ac5\u0ac7-\u0ac8\u0acd\u0ae2-\u0ae3\u0b01\u0b3c\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b62-\u0b63\u0b82\u0bc0\u0bcd\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c62-\u0c63\u0cbc\u0cbf\u0cc6\u0ccc-\u0ccd\u0ce2-\u0ce3\u0d41-\u0d44\u0d4d\u0d62-\u0d63\u0dca\u0dd2-\u0dd4\u0dd6\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb-\u0ebc\u0ec8-\u0ecd\u0f18-\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86-\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039-\u103a\u103d-\u103e\u1058-\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085-\u1086\u108d\u109d\u135d-\u135f\u1712-\u1714\u1732-\u1734\u1752-\u1753\u1772-\u1773\u17b4-\u17b5\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u18a9\u1920-\u1922\u1927-\u1928\u1932\u1939-\u193b\u1a17-\u1a18\u1a1b\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80-\u1b81\u1ba2-\u1ba5\u1ba8-\u1ba9\u1bab\u1be6\u1be8-\u1be9\u1bed\u1bef-\u1bf1\u1c2c-\u1c33\u1c36-\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1cf4\u1dc0-\u1de6\u1dfc-\u1dff\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2cef-\u2cf1\u2d7f\u2de0-\u2dff\u302a-\u302d\u3099-\u309a\ua66f\ua674-\ua67d\ua69f\ua6f0-\ua6f1\ua802\ua806\ua80b\ua825-\ua826\ua8c4\ua8e0-\ua8f1\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\uaa29-\uaa2e\uaa31-\uaa32\uaa35-\uaa36\uaa43\uaa4c\uaab0\uaab2-\uaab4\uaab7-\uaab8\uaabe-\uaabf\uaac1\uaaec-\uaaed\uaaf6\uabe5\uabe8\uabed\ufb1e\ufe00-\ufe0f\ufe20-\ufe26' +Mn = u'\u0300-\u036f\u0483-\u0487\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u0610-\u061a\u064b-\u065f\u0670\u06d6-\u06dc\u06df-\u06e4\u06e7-\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u07fd\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0859-\u085b\u08d3-\u08e1\u08e3-\u0902\u093a\u093c\u0941-\u0948\u094d\u0951-\u0957\u0962-\u0963\u0981\u09bc\u09c1-\u09c4\u09cd\u09e2-\u09e3\u09fe\u0a01-\u0a02\u0a3c\u0a41-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a70-\u0a71\u0a75\u0a81-\u0a82\u0abc\u0ac1-\u0ac5\u0ac7-\u0ac8\u0acd\u0ae2-\u0ae3\u0afa-\u0aff\u0b01\u0b3c\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b62-\u0b63\u0b82\u0bc0\u0bcd\u0c00\u0c04\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c62-\u0c63\u0c81\u0cbc\u0cbf\u0cc6\u0ccc-\u0ccd\u0ce2-\u0ce3\u0d00-\u0d01\u0d3b-\u0d3c\u0d41-\u0d44\u0d4d\u0d62-\u0d63\u0dca\u0dd2-\u0dd4\u0dd6\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb-\u0ebc\u0ec8-\u0ecd\u0f18-\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86-\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039-\u103a\u103d-\u103e\u1058-\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085-\u1086\u108d\u109d\u135d-\u135f\u1712-\u1714\u1732-\u1734\u1752-\u1753\u1772-\u1773\u17b4-\u17b5\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u1885-\u1886\u18a9\u1920-\u1922\u1927-\u1928\u1932\u1939-\u193b\u1a17-\u1a18\u1a1b\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1ab0-\u1abd\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80-\u1b81\u1ba2-\u1ba5\u1ba8-\u1ba9\u1bab-\u1bad\u1be6\u1be8-\u1be9\u1bed\u1bef-\u1bf1\u1c2c-\u1c33\u1c36-\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1cf4\u1cf8-\u1cf9\u1dc0-\u1df9\u1dfb-\u1dff\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2cef-\u2cf1\u2d7f\u2de0-\u2dff\u302a-\u302d\u3099-\u309a\ua66f\ua674-\ua67d\ua69e-\ua69f\ua6f0-\ua6f1\ua802\ua806\ua80b\ua825-\ua826\ua8c4-\ua8c5\ua8e0-\ua8f1\ua8ff\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\ua9e5\uaa29-\uaa2e\uaa31-\uaa32\uaa35-\uaa36\uaa43\uaa4c\uaa7c\uaab0\uaab2-\uaab4\uaab7-\uaab8\uaabe-\uaabf\uaac1\uaaec-\uaaed\uaaf6\uabe5\uabe8\uabed\ufb1e\ufe00-\ufe0f\ufe20-\ufe2f' -Nd = u'0-9\u0660-\u0669\u06f0-\u06f9\u07c0-\u07c9\u0966-\u096f\u09e6-\u09ef\u0a66-\u0a6f\u0ae6-\u0aef\u0b66-\u0b6f\u0be6-\u0bef\u0c66-\u0c6f\u0ce6-\u0cef\u0d66-\u0d6f\u0e50-\u0e59\u0ed0-\u0ed9\u0f20-\u0f29\u1040-\u1049\u1090-\u1099\u17e0-\u17e9\u1810-\u1819\u1946-\u194f\u19d0-\u19d9\u1a80-\u1a89\u1a90-\u1a99\u1b50-\u1b59\u1bb0-\u1bb9\u1c40-\u1c49\u1c50-\u1c59\ua620-\ua629\ua8d0-\ua8d9\ua900-\ua909\ua9d0-\ua9d9\uaa50-\uaa59\uabf0-\uabf9\uff10-\uff19' +Nd = u'0-9\u0660-\u0669\u06f0-\u06f9\u07c0-\u07c9\u0966-\u096f\u09e6-\u09ef\u0a66-\u0a6f\u0ae6-\u0aef\u0b66-\u0b6f\u0be6-\u0bef\u0c66-\u0c6f\u0ce6-\u0cef\u0d66-\u0d6f\u0de6-\u0def\u0e50-\u0e59\u0ed0-\u0ed9\u0f20-\u0f29\u1040-\u1049\u1090-\u1099\u17e0-\u17e9\u1810-\u1819\u1946-\u194f\u19d0-\u19d9\u1a80-\u1a89\u1a90-\u1a99\u1b50-\u1b59\u1bb0-\u1bb9\u1c40-\u1c49\u1c50-\u1c59\ua620-\ua629\ua8d0-\ua8d9\ua900-\ua909\ua9d0-\ua9d9\ua9f0-\ua9f9\uaa50-\uaa59\uabf0-\uabf9\uff10-\uff19' Nl = u'\u16ee-\u16f0\u2160-\u2182\u2185-\u2188\u3007\u3021-\u3029\u3038-\u303a\ua6e6-\ua6ef' -No = u'\xb2-\xb3\xb9\xbc-\xbe\u09f4-\u09f9\u0b72-\u0b77\u0bf0-\u0bf2\u0c78-\u0c7e\u0d70-\u0d75\u0f2a-\u0f33\u1369-\u137c\u17f0-\u17f9\u19da\u2070\u2074-\u2079\u2080-\u2089\u2150-\u215f\u2189\u2460-\u249b\u24ea-\u24ff\u2776-\u2793\u2cfd\u3192-\u3195\u3220-\u3229\u3248-\u324f\u3251-\u325f\u3280-\u3289\u32b1-\u32bf\ua830-\ua835' +No = u'\xb2-\xb3\xb9\xbc-\xbe\u09f4-\u09f9\u0b72-\u0b77\u0bf0-\u0bf2\u0c78-\u0c7e\u0d58-\u0d5e\u0d70-\u0d78\u0f2a-\u0f33\u1369-\u137c\u17f0-\u17f9\u19da\u2070\u2074-\u2079\u2080-\u2089\u2150-\u215f\u2189\u2460-\u249b\u24ea-\u24ff\u2776-\u2793\u2cfd\u3192-\u3195\u3220-\u3229\u3248-\u324f\u3251-\u325f\u3280-\u3289\u32b1-\u32bf\ua830-\ua835' Pc = u'_\u203f-\u2040\u2054\ufe33-\ufe34\ufe4d-\ufe4f\uff3f' -Pd = u'\\-\u058a\u05be\u1400\u1806\u2010-\u2015\u2e17\u2e1a\u2e3a-\u2e3b\u301c\u3030\u30a0\ufe31-\ufe32\ufe58\ufe63\uff0d' +Pd = u'\\-\u058a\u05be\u1400\u1806\u2010-\u2015\u2e17\u2e1a\u2e3a-\u2e3b\u2e40\u301c\u3030\u30a0\ufe31-\ufe32\ufe58\ufe63\uff0d' -Pe = u')\\]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u2309\u230b\u232a\u2769\u276b\u276d\u276f\u2771\u2773\u2775\u27c6\u27e7\u27e9\u27eb\u27ed\u27ef\u2984\u2986\u2988\u298a\u298c\u298e\u2990\u2992\u2994\u2996\u2998\u29d9\u29db\u29fd\u2e23\u2e25\u2e27\u2e29\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e-\u301f\ufd3f\ufe18\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42\ufe44\ufe48\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff60\uff63' +Pe = u')\\]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u2309\u230b\u232a\u2769\u276b\u276d\u276f\u2771\u2773\u2775\u27c6\u27e7\u27e9\u27eb\u27ed\u27ef\u2984\u2986\u2988\u298a\u298c\u298e\u2990\u2992\u2994\u2996\u2998\u29d9\u29db\u29fd\u2e23\u2e25\u2e27\u2e29\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e-\u301f\ufd3e\ufe18\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42\ufe44\ufe48\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff60\uff63' Pf = u'\xbb\u2019\u201d\u203a\u2e03\u2e05\u2e0a\u2e0d\u2e1d\u2e21' Pi = u'\xab\u2018\u201b-\u201c\u201f\u2039\u2e02\u2e04\u2e09\u2e0c\u2e1c\u2e20' -Po = u"!-#%-'*,.-/:-;?-@\\\\\xa1\xa7\xb6-\xb7\xbf\u037e\u0387\u055a-\u055f\u0589\u05c0\u05c3\u05c6\u05f3-\u05f4\u0609-\u060a\u060c-\u060d\u061b\u061e-\u061f\u066a-\u066d\u06d4\u0700-\u070d\u07f7-\u07f9\u0830-\u083e\u085e\u0964-\u0965\u0970\u0af0\u0df4\u0e4f\u0e5a-\u0e5b\u0f04-\u0f12\u0f14\u0f85\u0fd0-\u0fd4\u0fd9-\u0fda\u104a-\u104f\u10fb\u1360-\u1368\u166d-\u166e\u16eb-\u16ed\u1735-\u1736\u17d4-\u17d6\u17d8-\u17da\u1800-\u1805\u1807-\u180a\u1944-\u1945\u1a1e-\u1a1f\u1aa0-\u1aa6\u1aa8-\u1aad\u1b5a-\u1b60\u1bfc-\u1bff\u1c3b-\u1c3f\u1c7e-\u1c7f\u1cc0-\u1cc7\u1cd3\u2016-\u2017\u2020-\u2027\u2030-\u2038\u203b-\u203e\u2041-\u2043\u2047-\u2051\u2053\u2055-\u205e\u2cf9-\u2cfc\u2cfe-\u2cff\u2d70\u2e00-\u2e01\u2e06-\u2e08\u2e0b\u2e0e-\u2e16\u2e18-\u2e19\u2e1b\u2e1e-\u2e1f\u2e2a-\u2e2e\u2e30-\u2e39\u3001-\u3003\u303d\u30fb\ua4fe-\ua4ff\ua60d-\ua60f\ua673\ua67e\ua6f2-\ua6f7\ua874-\ua877\ua8ce-\ua8cf\ua8f8-\ua8fa\ua92e-\ua92f\ua95f\ua9c1-\ua9cd\ua9de-\ua9df\uaa5c-\uaa5f\uaade-\uaadf\uaaf0-\uaaf1\uabeb\ufe10-\ufe16\ufe19\ufe30\ufe45-\ufe46\ufe49-\ufe4c\ufe50-\ufe52\ufe54-\ufe57\ufe5f-\ufe61\ufe68\ufe6a-\ufe6b\uff01-\uff03\uff05-\uff07\uff0a\uff0c\uff0e-\uff0f\uff1a-\uff1b\uff1f-\uff20\uff3c\uff61\uff64-\uff65" +Po = u"!-#%-'*,.-/:-;?-@\\\\\xa1\xa7\xb6-\xb7\xbf\u037e\u0387\u055a-\u055f\u0589\u05c0\u05c3\u05c6\u05f3-\u05f4\u0609-\u060a\u060c-\u060d\u061b\u061e-\u061f\u066a-\u066d\u06d4\u0700-\u070d\u07f7-\u07f9\u0830-\u083e\u085e\u0964-\u0965\u0970\u09fd\u0a76\u0af0\u0c84\u0df4\u0e4f\u0e5a-\u0e5b\u0f04-\u0f12\u0f14\u0f85\u0fd0-\u0fd4\u0fd9-\u0fda\u104a-\u104f\u10fb\u1360-\u1368\u166d-\u166e\u16eb-\u16ed\u1735-\u1736\u17d4-\u17d6\u17d8-\u17da\u1800-\u1805\u1807-\u180a\u1944-\u1945\u1a1e-\u1a1f\u1aa0-\u1aa6\u1aa8-\u1aad\u1b5a-\u1b60\u1bfc-\u1bff\u1c3b-\u1c3f\u1c7e-\u1c7f\u1cc0-\u1cc7\u1cd3\u2016-\u2017\u2020-\u2027\u2030-\u2038\u203b-\u203e\u2041-\u2043\u2047-\u2051\u2053\u2055-\u205e\u2cf9-\u2cfc\u2cfe-\u2cff\u2d70\u2e00-\u2e01\u2e06-\u2e08\u2e0b\u2e0e-\u2e16\u2e18-\u2e19\u2e1b\u2e1e-\u2e1f\u2e2a-\u2e2e\u2e30-\u2e39\u2e3c-\u2e3f\u2e41\u2e43-\u2e4e\u3001-\u3003\u303d\u30fb\ua4fe-\ua4ff\ua60d-\ua60f\ua673\ua67e\ua6f2-\ua6f7\ua874-\ua877\ua8ce-\ua8cf\ua8f8-\ua8fa\ua8fc\ua92e-\ua92f\ua95f\ua9c1-\ua9cd\ua9de-\ua9df\uaa5c-\uaa5f\uaade-\uaadf\uaaf0-\uaaf1\uabeb\ufe10-\ufe16\ufe19\ufe30\ufe45-\ufe46\ufe49-\ufe4c\ufe50-\ufe52\ufe54-\ufe57\ufe5f-\ufe61\ufe68\ufe6a-\ufe6b\uff01-\uff03\uff05-\uff07\uff0a\uff0c\uff0e-\uff0f\uff1a-\uff1b\uff1f-\uff20\uff3c\uff61\uff64-\uff65" -Ps = u'(\\[{\u0f3a\u0f3c\u169b\u201a\u201e\u2045\u207d\u208d\u2308\u230a\u2329\u2768\u276a\u276c\u276e\u2770\u2772\u2774\u27c5\u27e6\u27e8\u27ea\u27ec\u27ee\u2983\u2985\u2987\u2989\u298b\u298d\u298f\u2991\u2993\u2995\u2997\u29d8\u29da\u29fc\u2e22\u2e24\u2e26\u2e28\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d\ufd3e\ufe17\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe47\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff5f\uff62' +Ps = u'(\\[{\u0f3a\u0f3c\u169b\u201a\u201e\u2045\u207d\u208d\u2308\u230a\u2329\u2768\u276a\u276c\u276e\u2770\u2772\u2774\u27c5\u27e6\u27e8\u27ea\u27ec\u27ee\u2983\u2985\u2987\u2989\u298b\u298d\u298f\u2991\u2993\u2995\u2997\u29d8\u29da\u29fc\u2e22\u2e24\u2e26\u2e28\u2e42\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d\ufd3f\ufe17\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe47\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff5f\uff62' -Sc = u'$\xa2-\xa5\u058f\u060b\u09f2-\u09f3\u09fb\u0af1\u0bf9\u0e3f\u17db\u20a0-\u20ba\ua838\ufdfc\ufe69\uff04\uffe0-\uffe1\uffe5-\uffe6' +Sc = u'$\xa2-\xa5\u058f\u060b\u07fe-\u07ff\u09f2-\u09f3\u09fb\u0af1\u0bf9\u0e3f\u17db\u20a0-\u20bf\ua838\ufdfc\ufe69\uff04\uffe0-\uffe1\uffe5-\uffe6' -Sk = u'\\^`\xa8\xaf\xb4\xb8\u02c2-\u02c5\u02d2-\u02df\u02e5-\u02eb\u02ed\u02ef-\u02ff\u0375\u0384-\u0385\u1fbd\u1fbf-\u1fc1\u1fcd-\u1fcf\u1fdd-\u1fdf\u1fed-\u1fef\u1ffd-\u1ffe\u309b-\u309c\ua700-\ua716\ua720-\ua721\ua789-\ua78a\ufbb2-\ufbc1\uff3e\uff40\uffe3' +Sk = u'\\^`\xa8\xaf\xb4\xb8\u02c2-\u02c5\u02d2-\u02df\u02e5-\u02eb\u02ed\u02ef-\u02ff\u0375\u0384-\u0385\u1fbd\u1fbf-\u1fc1\u1fcd-\u1fcf\u1fdd-\u1fdf\u1fed-\u1fef\u1ffd-\u1ffe\u309b-\u309c\ua700-\ua716\ua720-\ua721\ua789-\ua78a\uab5b\ufbb2-\ufbc1\uff3e\uff40\uffe3' Sm = u'+<->|~\xac\xb1\xd7\xf7\u03f6\u0606-\u0608\u2044\u2052\u207a-\u207c\u208a-\u208c\u2118\u2140-\u2144\u214b\u2190-\u2194\u219a-\u219b\u21a0\u21a3\u21a6\u21ae\u21ce-\u21cf\u21d2\u21d4\u21f4-\u22ff\u2320-\u2321\u237c\u239b-\u23b3\u23dc-\u23e1\u25b7\u25c1\u25f8-\u25ff\u266f\u27c0-\u27c4\u27c7-\u27e5\u27f0-\u27ff\u2900-\u2982\u2999-\u29d7\u29dc-\u29fb\u29fe-\u2aff\u2b30-\u2b44\u2b47-\u2b4c\ufb29\ufe62\ufe64-\ufe66\uff0b\uff1c-\uff1e\uff5c\uff5e\uffe2\uffe9-\uffec' -So = u'\xa6\xa9\xae\xb0\u0482\u060e-\u060f\u06de\u06e9\u06fd-\u06fe\u07f6\u09fa\u0b70\u0bf3-\u0bf8\u0bfa\u0c7f\u0d79\u0f01-\u0f03\u0f13\u0f15-\u0f17\u0f1a-\u0f1f\u0f34\u0f36\u0f38\u0fbe-\u0fc5\u0fc7-\u0fcc\u0fce-\u0fcf\u0fd5-\u0fd8\u109e-\u109f\u1390-\u1399\u1940\u19de-\u19ff\u1b61-\u1b6a\u1b74-\u1b7c\u2100-\u2101\u2103-\u2106\u2108-\u2109\u2114\u2116-\u2117\u211e-\u2123\u2125\u2127\u2129\u212e\u213a-\u213b\u214a\u214c-\u214d\u214f\u2195-\u2199\u219c-\u219f\u21a1-\u21a2\u21a4-\u21a5\u21a7-\u21ad\u21af-\u21cd\u21d0-\u21d1\u21d3\u21d5-\u21f3\u2300-\u2307\u230c-\u231f\u2322-\u2328\u232b-\u237b\u237d-\u239a\u23b4-\u23db\u23e2-\u23f3\u2400-\u2426\u2440-\u244a\u249c-\u24e9\u2500-\u25b6\u25b8-\u25c0\u25c2-\u25f7\u2600-\u266e\u2670-\u26ff\u2701-\u2767\u2794-\u27bf\u2800-\u28ff\u2b00-\u2b2f\u2b45-\u2b46\u2b50-\u2b59\u2ce5-\u2cea\u2e80-\u2e99\u2e9b-\u2ef3\u2f00-\u2fd5\u2ff0-\u2ffb\u3004\u3012-\u3013\u3020\u3036-\u3037\u303e-\u303f\u3190-\u3191\u3196-\u319f\u31c0-\u31e3\u3200-\u321e\u322a-\u3247\u3250\u3260-\u327f\u328a-\u32b0\u32c0-\u32fe\u3300-\u33ff\u4dc0-\u4dff\ua490-\ua4c6\ua828-\ua82b\ua836-\ua837\ua839\uaa77-\uaa79\ufdfd\uffe4\uffe8\uffed-\uffee\ufffc-\ufffd' +So = u'\xa6\xa9\xae\xb0\u0482\u058d-\u058e\u060e-\u060f\u06de\u06e9\u06fd-\u06fe\u07f6\u09fa\u0b70\u0bf3-\u0bf8\u0bfa\u0c7f\u0d4f\u0d79\u0f01-\u0f03\u0f13\u0f15-\u0f17\u0f1a-\u0f1f\u0f34\u0f36\u0f38\u0fbe-\u0fc5\u0fc7-\u0fcc\u0fce-\u0fcf\u0fd5-\u0fd8\u109e-\u109f\u1390-\u1399\u1940\u19de-\u19ff\u1b61-\u1b6a\u1b74-\u1b7c\u2100-\u2101\u2103-\u2106\u2108-\u2109\u2114\u2116-\u2117\u211e-\u2123\u2125\u2127\u2129\u212e\u213a-\u213b\u214a\u214c-\u214d\u214f\u218a-\u218b\u2195-\u2199\u219c-\u219f\u21a1-\u21a2\u21a4-\u21a5\u21a7-\u21ad\u21af-\u21cd\u21d0-\u21d1\u21d3\u21d5-\u21f3\u2300-\u2307\u230c-\u231f\u2322-\u2328\u232b-\u237b\u237d-\u239a\u23b4-\u23db\u23e2-\u2426\u2440-\u244a\u249c-\u24e9\u2500-\u25b6\u25b8-\u25c0\u25c2-\u25f7\u2600-\u266e\u2670-\u2767\u2794-\u27bf\u2800-\u28ff\u2b00-\u2b2f\u2b45-\u2b46\u2b4d-\u2b73\u2b76-\u2b95\u2b98-\u2bc8\u2bca-\u2bfe\u2ce5-\u2cea\u2e80-\u2e99\u2e9b-\u2ef3\u2f00-\u2fd5\u2ff0-\u2ffb\u3004\u3012-\u3013\u3020\u3036-\u3037\u303e-\u303f\u3190-\u3191\u3196-\u319f\u31c0-\u31e3\u3200-\u321e\u322a-\u3247\u3250\u3260-\u327f\u328a-\u32b0\u32c0-\u32fe\u3300-\u33ff\u4dc0-\u4dff\ua490-\ua4c6\ua828-\ua82b\ua836-\ua837\ua839\uaa77-\uaa79\ufdfd\uffe4\uffe8\uffed-\uffee\ufffc-\ufffd' Zl = u'\u2028' @@ -77,49 +77,53 @@ Zp = u'\u2029' Zs = u' \xa0\u1680\u2000-\u200a\u202f\u205f\u3000' -xid_continue = u'0-9A-Z_a-z\xaa\xb5\xb7\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0300-\u0374\u0376-\u0377\u037b-\u037d\u0386-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u0483-\u0487\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u05d0-\u05ea\u05f0-\u05f2\u0610-\u061a\u0620-\u0669\u066e-\u06d3\u06d5-\u06dc\u06df-\u06e8\u06ea-\u06fc\u06ff\u0710-\u074a\u074d-\u07b1\u07c0-\u07f5\u07fa\u0800-\u082d\u0840-\u085b\u08a0\u08a2-\u08ac\u08e4-\u08fe\u0900-\u0963\u0966-\u096f\u0971-\u0977\u0979-\u097f\u0981-\u0983\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bc-\u09c4\u09c7-\u09c8\u09cb-\u09ce\u09d7\u09dc-\u09dd\u09df-\u09e3\u09e6-\u09f1\u0a01-\u0a03\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a3c\u0a3e-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a59-\u0a5c\u0a5e\u0a66-\u0a75\u0a81-\u0a83\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abc-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ad0\u0ae0-\u0ae3\u0ae6-\u0aef\u0b01-\u0b03\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3c-\u0b44\u0b47-\u0b48\u0b4b-\u0b4d\u0b56-\u0b57\u0b5c-\u0b5d\u0b5f-\u0b63\u0b66-\u0b6f\u0b71\u0b82-\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd0\u0bd7\u0be6-\u0bef\u0c01-\u0c03\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d-\u0c44\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c58-\u0c59\u0c60-\u0c63\u0c66-\u0c6f\u0c82-\u0c83\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbc-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5-\u0cd6\u0cde\u0ce0-\u0ce3\u0ce6-\u0cef\u0cf1-\u0cf2\u0d02-\u0d03\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d-\u0d44\u0d46-\u0d48\u0d4a-\u0d4e\u0d57\u0d60-\u0d63\u0d66-\u0d6f\u0d7a-\u0d7f\u0d82-\u0d83\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0df2-\u0df3\u0e01-\u0e3a\u0e40-\u0e4e\u0e50-\u0e59\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb9\u0ebb-\u0ebd\u0ec0-\u0ec4\u0ec6\u0ec8-\u0ecd\u0ed0-\u0ed9\u0edc-\u0edf\u0f00\u0f18-\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f3e-\u0f47\u0f49-\u0f6c\u0f71-\u0f84\u0f86-\u0f97\u0f99-\u0fbc\u0fc6\u1000-\u1049\u1050-\u109d\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u135d-\u135f\u1369-\u1371\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1714\u1720-\u1734\u1740-\u1753\u1760-\u176c\u176e-\u1770\u1772-\u1773\u1780-\u17d3\u17d7\u17dc-\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u1820-\u1877\u1880-\u18aa\u18b0-\u18f5\u1900-\u191c\u1920-\u192b\u1930-\u193b\u1946-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u19d0-\u19da\u1a00-\u1a1b\u1a20-\u1a5e\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1aa7\u1b00-\u1b4b\u1b50-\u1b59\u1b6b-\u1b73\u1b80-\u1bf3\u1c00-\u1c37\u1c40-\u1c49\u1c4d-\u1c7d\u1cd0-\u1cd2\u1cd4-\u1cf6\u1d00-\u1de6\u1dfc-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u203f-\u2040\u2054\u2071\u207f\u2090-\u209c\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d7f-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2de0-\u2dff\u3005-\u3007\u3021-\u302f\u3031-\u3035\u3038-\u303c\u3041-\u3096\u3099-\u309a\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua62b\ua640-\ua66f\ua674-\ua67d\ua67f-\ua697\ua69f-\ua6f1\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua827\ua840-\ua873\ua880-\ua8c4\ua8d0-\ua8d9\ua8e0-\ua8f7\ua8fb\ua900-\ua92d\ua930-\ua953\ua960-\ua97c\ua980-\ua9c0\ua9cf-\ua9d9\uaa00-\uaa36\uaa40-\uaa4d\uaa50-\uaa59\uaa60-\uaa76\uaa7a-\uaa7b\uaa80-\uaac2\uaadb-\uaadd\uaae0-\uaaef\uaaf2-\uaaf6\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabea\uabec-\uabed\uabf0-\uabf9\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe00-\ufe0f\ufe20-\ufe26\ufe33-\ufe34\ufe4d-\ufe4f\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff10-\uff19\uff21-\uff3a\uff3f\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' +xid_continue = u'0-9A-Z_a-z\xaa\xb5\xb7\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0300-\u0374\u0376-\u0377\u037b-\u037d\u037f\u0386-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u0483-\u0487\u048a-\u052f\u0531-\u0556\u0559\u0560-\u0588\u0591-\u05bd\u05bf\u05c1-\u05c2\u05c4-\u05c5\u05c7\u05d0-\u05ea\u05ef-\u05f2\u0610-\u061a\u0620-\u0669\u066e-\u06d3\u06d5-\u06dc\u06df-\u06e8\u06ea-\u06fc\u06ff\u0710-\u074a\u074d-\u07b1\u07c0-\u07f5\u07fa\u07fd\u0800-\u082d\u0840-\u085b\u0860-\u086a\u08a0-\u08b4\u08b6-\u08bd\u08d3-\u08e1\u08e3-\u0963\u0966-\u096f\u0971-\u0983\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bc-\u09c4\u09c7-\u09c8\u09cb-\u09ce\u09d7\u09dc-\u09dd\u09df-\u09e3\u09e6-\u09f1\u09fc\u09fe\u0a01-\u0a03\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a3c\u0a3e-\u0a42\u0a47-\u0a48\u0a4b-\u0a4d\u0a51\u0a59-\u0a5c\u0a5e\u0a66-\u0a75\u0a81-\u0a83\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abc-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ad0\u0ae0-\u0ae3\u0ae6-\u0aef\u0af9-\u0aff\u0b01-\u0b03\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3c-\u0b44\u0b47-\u0b48\u0b4b-\u0b4d\u0b56-\u0b57\u0b5c-\u0b5d\u0b5f-\u0b63\u0b66-\u0b6f\u0b71\u0b82-\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd0\u0bd7\u0be6-\u0bef\u0c00-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c39\u0c3d-\u0c44\u0c46-\u0c48\u0c4a-\u0c4d\u0c55-\u0c56\u0c58-\u0c5a\u0c60-\u0c63\u0c66-\u0c6f\u0c80-\u0c83\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbc-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5-\u0cd6\u0cde\u0ce0-\u0ce3\u0ce6-\u0cef\u0cf1-\u0cf2\u0d00-\u0d03\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d44\u0d46-\u0d48\u0d4a-\u0d4e\u0d54-\u0d57\u0d5f-\u0d63\u0d66-\u0d6f\u0d7a-\u0d7f\u0d82-\u0d83\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0de6-\u0def\u0df2-\u0df3\u0e01-\u0e3a\u0e40-\u0e4e\u0e50-\u0e59\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb9\u0ebb-\u0ebd\u0ec0-\u0ec4\u0ec6\u0ec8-\u0ecd\u0ed0-\u0ed9\u0edc-\u0edf\u0f00\u0f18-\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f3e-\u0f47\u0f49-\u0f6c\u0f71-\u0f84\u0f86-\u0f97\u0f99-\u0fbc\u0fc6\u1000-\u1049\u1050-\u109d\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u135d-\u135f\u1369-\u1371\u1380-\u138f\u13a0-\u13f5\u13f8-\u13fd\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f8\u1700-\u170c\u170e-\u1714\u1720-\u1734\u1740-\u1753\u1760-\u176c\u176e-\u1770\u1772-\u1773\u1780-\u17d3\u17d7\u17dc-\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u1820-\u1878\u1880-\u18aa\u18b0-\u18f5\u1900-\u191e\u1920-\u192b\u1930-\u193b\u1946-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u19d0-\u19da\u1a00-\u1a1b\u1a20-\u1a5e\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1aa7\u1ab0-\u1abd\u1b00-\u1b4b\u1b50-\u1b59\u1b6b-\u1b73\u1b80-\u1bf3\u1c00-\u1c37\u1c40-\u1c49\u1c4d-\u1c7d\u1c80-\u1c88\u1c90-\u1cba\u1cbd-\u1cbf\u1cd0-\u1cd2\u1cd4-\u1cf9\u1d00-\u1df9\u1dfb-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u203f-\u2040\u2054\u2071\u207f\u2090-\u209c\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d7f-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2de0-\u2dff\u3005-\u3007\u3021-\u302f\u3031-\u3035\u3038-\u303c\u3041-\u3096\u3099-\u309a\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312f\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fef\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua62b\ua640-\ua66f\ua674-\ua67d\ua67f-\ua6f1\ua717-\ua71f\ua722-\ua788\ua78b-\ua7b9\ua7f7-\ua827\ua840-\ua873\ua880-\ua8c5\ua8d0-\ua8d9\ua8e0-\ua8f7\ua8fb\ua8fd-\ua92d\ua930-\ua953\ua960-\ua97c\ua980-\ua9c0\ua9cf-\ua9d9\ua9e0-\ua9fe\uaa00-\uaa36\uaa40-\uaa4d\uaa50-\uaa59\uaa60-\uaa76\uaa7a-\uaac2\uaadb-\uaadd\uaae0-\uaaef\uaaf2-\uaaf6\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uab30-\uab5a\uab5c-\uab65\uab70-\uabea\uabec-\uabed\uabf0-\uabf9\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe00-\ufe0f\ufe20-\ufe2f\ufe33-\ufe34\ufe4d-\ufe4f\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff10-\uff19\uff21-\uff3a\uff3f\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' -xid_start = u'A-Z_a-z\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376-\u0377\u037b-\u037d\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u05d0-\u05ea\u05f0-\u05f2\u0620-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06e5-\u06e6\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4-\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0977\u0979-\u097f\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58-\u0c59\u0c60-\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e40-\u0e46\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a-\ua62b\ua640-\ua66e\ua67f-\ua697\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' +xid_start = u'A-Z_a-z\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376-\u0377\u037b-\u037d\u037f\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u052f\u0531-\u0556\u0559\u0560-\u0588\u05d0-\u05ea\u05ef-\u05f2\u0620-\u064a\u066e-\u066f\u0671-\u06d3\u06d5\u06e5-\u06e6\u06ee-\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4-\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u0860-\u086a\u08a0-\u08b4\u08b6-\u08bd\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0980\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09f0-\u09f1\u09fc\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0af9\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c39\u0c3d\u0c58-\u0c5a\u0c60-\u0c61\u0c80\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0cf1-\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d54-\u0d56\u0d5f-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e40-\u0e46\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f5\u13f8-\u13fd\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f8\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1878\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191e\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae-\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1c80-\u1c88\u1c90-\u1cba\u1cbd-\u1cbf\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5-\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2-\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312f\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fef\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a-\ua62b\ua640-\ua66e\ua67f-\ua69d\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua7b9\ua7f7-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua8fd-\ua8fe\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\ua9e0-\ua9e4\ua9e6-\ua9ef\ua9fa-\ua9fe\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa7e-\uaaaf\uaab1\uaab5-\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uab30-\uab5a\uab5c-\uab65\uab70-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufc5d\ufc64-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdf9\ufe71\ufe73\ufe77\ufe79\ufe7b\ufe7d\ufe7f-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d\uffa0-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc' if sys.maxunicode > 0xFFFF: # non-BMP characters, use only on wide Unicode builds - Cf += u'\U000110bd\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f' + Cf += u'\U000110bd\U000110cd\U0001bca0-\U0001bca3\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f' - Cn += u'\U0001000c\U00010027\U0001003b\U0001003e\U0001004e-\U0001004f\U0001005e-\U0001007f\U000100fb-\U000100ff\U00010103-\U00010106\U00010134-\U00010136\U0001018b-\U0001018f\U0001019c-\U000101cf\U000101fe-\U0001027f\U0001029d-\U0001029f\U000102d1-\U000102ff\U0001031f\U00010324-\U0001032f\U0001034b-\U0001037f\U0001039e\U000103c4-\U000103c7\U000103d6-\U000103ff\U0001049e-\U0001049f\U000104aa-\U000107ff\U00010806-\U00010807\U00010809\U00010836\U00010839-\U0001083b\U0001083d-\U0001083e\U00010856\U00010860-\U000108ff\U0001091c-\U0001091e\U0001093a-\U0001093e\U00010940-\U0001097f\U000109b8-\U000109bd\U000109c0-\U000109ff\U00010a04\U00010a07-\U00010a0b\U00010a14\U00010a18\U00010a34-\U00010a37\U00010a3b-\U00010a3e\U00010a48-\U00010a4f\U00010a59-\U00010a5f\U00010a80-\U00010aff\U00010b36-\U00010b38\U00010b56-\U00010b57\U00010b73-\U00010b77\U00010b80-\U00010bff\U00010c49-\U00010e5f\U00010e7f-\U00010fff\U0001104e-\U00011051\U00011070-\U0001107f\U000110c2-\U000110cf\U000110e9-\U000110ef\U000110fa-\U000110ff\U00011135\U00011144-\U0001117f\U000111c9-\U000111cf\U000111da-\U0001167f\U000116b8-\U000116bf\U000116ca-\U00011fff\U0001236f-\U000123ff\U00012463-\U0001246f\U00012474-\U00012fff\U0001342f-\U000167ff\U00016a39-\U00016eff\U00016f45-\U00016f4f\U00016f7f-\U00016f8e\U00016fa0-\U0001afff\U0001b002-\U0001cfff\U0001d0f6-\U0001d0ff\U0001d127-\U0001d128\U0001d1de-\U0001d1ff\U0001d246-\U0001d2ff\U0001d357-\U0001d35f\U0001d372-\U0001d3ff\U0001d455\U0001d49d\U0001d4a0-\U0001d4a1\U0001d4a3-\U0001d4a4\U0001d4a7-\U0001d4a8\U0001d4ad\U0001d4ba\U0001d4bc\U0001d4c4\U0001d506\U0001d50b-\U0001d50c\U0001d515\U0001d51d\U0001d53a\U0001d53f\U0001d545\U0001d547-\U0001d549\U0001d551\U0001d6a6-\U0001d6a7\U0001d7cc-\U0001d7cd\U0001d800-\U0001edff\U0001ee04\U0001ee20\U0001ee23\U0001ee25-\U0001ee26\U0001ee28\U0001ee33\U0001ee38\U0001ee3a\U0001ee3c-\U0001ee41\U0001ee43-\U0001ee46\U0001ee48\U0001ee4a\U0001ee4c\U0001ee50\U0001ee53\U0001ee55-\U0001ee56\U0001ee58\U0001ee5a\U0001ee5c\U0001ee5e\U0001ee60\U0001ee63\U0001ee65-\U0001ee66\U0001ee6b\U0001ee73\U0001ee78\U0001ee7d\U0001ee7f\U0001ee8a\U0001ee9c-\U0001eea0\U0001eea4\U0001eeaa\U0001eebc-\U0001eeef\U0001eef2-\U0001efff\U0001f02c-\U0001f02f\U0001f094-\U0001f09f\U0001f0af-\U0001f0b0\U0001f0bf-\U0001f0c0\U0001f0d0\U0001f0e0-\U0001f0ff\U0001f10b-\U0001f10f\U0001f12f\U0001f16c-\U0001f16f\U0001f19b-\U0001f1e5\U0001f203-\U0001f20f\U0001f23b-\U0001f23f\U0001f249-\U0001f24f\U0001f252-\U0001f2ff\U0001f321-\U0001f32f\U0001f336\U0001f37d-\U0001f37f\U0001f394-\U0001f39f\U0001f3c5\U0001f3cb-\U0001f3df\U0001f3f1-\U0001f3ff\U0001f43f\U0001f441\U0001f4f8\U0001f4fd-\U0001f4ff\U0001f53e-\U0001f53f\U0001f544-\U0001f54f\U0001f568-\U0001f5fa\U0001f641-\U0001f644\U0001f650-\U0001f67f\U0001f6c6-\U0001f6ff\U0001f774-\U0001ffff\U0002a6d7-\U0002a6ff\U0002b735-\U0002b73f\U0002b81e-\U0002f7ff\U0002fa1e-\U000e0000\U000e0002-\U000e001f\U000e0080-\U000e00ff\U000e01f0-\U000effff\U000ffffe-\U000fffff\U0010fffe-\U0010ffff' + Cn += u'\U0001000c\U00010027\U0001003b\U0001003e\U0001004e-\U0001004f\U0001005e-\U0001007f\U000100fb-\U000100ff\U00010103-\U00010106\U00010134-\U00010136\U0001018f\U0001019c-\U0001019f\U000101a1-\U000101cf\U000101fe-\U0001027f\U0001029d-\U0001029f\U000102d1-\U000102df\U000102fc-\U000102ff\U00010324-\U0001032c\U0001034b-\U0001034f\U0001037b-\U0001037f\U0001039e\U000103c4-\U000103c7\U000103d6-\U000103ff\U0001049e-\U0001049f\U000104aa-\U000104af\U000104d4-\U000104d7\U000104fc-\U000104ff\U00010528-\U0001052f\U00010564-\U0001056e\U00010570-\U000105ff\U00010737-\U0001073f\U00010756-\U0001075f\U00010768-\U000107ff\U00010806-\U00010807\U00010809\U00010836\U00010839-\U0001083b\U0001083d-\U0001083e\U00010856\U0001089f-\U000108a6\U000108b0-\U000108df\U000108f3\U000108f6-\U000108fa\U0001091c-\U0001091e\U0001093a-\U0001093e\U00010940-\U0001097f\U000109b8-\U000109bb\U000109d0-\U000109d1\U00010a04\U00010a07-\U00010a0b\U00010a14\U00010a18\U00010a36-\U00010a37\U00010a3b-\U00010a3e\U00010a49-\U00010a4f\U00010a59-\U00010a5f\U00010aa0-\U00010abf\U00010ae7-\U00010aea\U00010af7-\U00010aff\U00010b36-\U00010b38\U00010b56-\U00010b57\U00010b73-\U00010b77\U00010b92-\U00010b98\U00010b9d-\U00010ba8\U00010bb0-\U00010bff\U00010c49-\U00010c7f\U00010cb3-\U00010cbf\U00010cf3-\U00010cf9\U00010d28-\U00010d2f\U00010d3a-\U00010e5f\U00010e7f-\U00010eff\U00010f28-\U00010f2f\U00010f5a-\U00010fff\U0001104e-\U00011051\U00011070-\U0001107e\U000110c2-\U000110cc\U000110ce-\U000110cf\U000110e9-\U000110ef\U000110fa-\U000110ff\U00011135\U00011147-\U0001114f\U00011177-\U0001117f\U000111ce-\U000111cf\U000111e0\U000111f5-\U000111ff\U00011212\U0001123f-\U0001127f\U00011287\U00011289\U0001128e\U0001129e\U000112aa-\U000112af\U000112eb-\U000112ef\U000112fa-\U000112ff\U00011304\U0001130d-\U0001130e\U00011311-\U00011312\U00011329\U00011331\U00011334\U0001133a\U00011345-\U00011346\U00011349-\U0001134a\U0001134e-\U0001134f\U00011351-\U00011356\U00011358-\U0001135c\U00011364-\U00011365\U0001136d-\U0001136f\U00011375-\U000113ff\U0001145a\U0001145c\U0001145f-\U0001147f\U000114c8-\U000114cf\U000114da-\U0001157f\U000115b6-\U000115b7\U000115de-\U000115ff\U00011645-\U0001164f\U0001165a-\U0001165f\U0001166d-\U0001167f\U000116b8-\U000116bf\U000116ca-\U000116ff\U0001171b-\U0001171c\U0001172c-\U0001172f\U00011740-\U000117ff\U0001183c-\U0001189f\U000118f3-\U000118fe\U00011900-\U000119ff\U00011a48-\U00011a4f\U00011a84-\U00011a85\U00011aa3-\U00011abf\U00011af9-\U00011bff\U00011c09\U00011c37\U00011c46-\U00011c4f\U00011c6d-\U00011c6f\U00011c90-\U00011c91\U00011ca8\U00011cb7-\U00011cff\U00011d07\U00011d0a\U00011d37-\U00011d39\U00011d3b\U00011d3e\U00011d48-\U00011d4f\U00011d5a-\U00011d5f\U00011d66\U00011d69\U00011d8f\U00011d92\U00011d99-\U00011d9f\U00011daa-\U00011edf\U00011ef9-\U00011fff\U0001239a-\U000123ff\U0001246f\U00012475-\U0001247f\U00012544-\U00012fff\U0001342f-\U000143ff\U00014647-\U000167ff\U00016a39-\U00016a3f\U00016a5f\U00016a6a-\U00016a6d\U00016a70-\U00016acf\U00016aee-\U00016aef\U00016af6-\U00016aff\U00016b46-\U00016b4f\U00016b5a\U00016b62\U00016b78-\U00016b7c\U00016b90-\U00016e3f\U00016e9b-\U00016eff\U00016f45-\U00016f4f\U00016f7f-\U00016f8e\U00016fa0-\U00016fdf\U00016fe2-\U00016fff\U000187f2-\U000187ff\U00018af3-\U0001afff\U0001b11f-\U0001b16f\U0001b2fc-\U0001bbff\U0001bc6b-\U0001bc6f\U0001bc7d-\U0001bc7f\U0001bc89-\U0001bc8f\U0001bc9a-\U0001bc9b\U0001bca4-\U0001cfff\U0001d0f6-\U0001d0ff\U0001d127-\U0001d128\U0001d1e9-\U0001d1ff\U0001d246-\U0001d2df\U0001d2f4-\U0001d2ff\U0001d357-\U0001d35f\U0001d379-\U0001d3ff\U0001d455\U0001d49d\U0001d4a0-\U0001d4a1\U0001d4a3-\U0001d4a4\U0001d4a7-\U0001d4a8\U0001d4ad\U0001d4ba\U0001d4bc\U0001d4c4\U0001d506\U0001d50b-\U0001d50c\U0001d515\U0001d51d\U0001d53a\U0001d53f\U0001d545\U0001d547-\U0001d549\U0001d551\U0001d6a6-\U0001d6a7\U0001d7cc-\U0001d7cd\U0001da8c-\U0001da9a\U0001daa0\U0001dab0-\U0001dfff\U0001e007\U0001e019-\U0001e01a\U0001e022\U0001e025\U0001e02b-\U0001e7ff\U0001e8c5-\U0001e8c6\U0001e8d7-\U0001e8ff\U0001e94b-\U0001e94f\U0001e95a-\U0001e95d\U0001e960-\U0001ec70\U0001ecb5-\U0001edff\U0001ee04\U0001ee20\U0001ee23\U0001ee25-\U0001ee26\U0001ee28\U0001ee33\U0001ee38\U0001ee3a\U0001ee3c-\U0001ee41\U0001ee43-\U0001ee46\U0001ee48\U0001ee4a\U0001ee4c\U0001ee50\U0001ee53\U0001ee55-\U0001ee56\U0001ee58\U0001ee5a\U0001ee5c\U0001ee5e\U0001ee60\U0001ee63\U0001ee65-\U0001ee66\U0001ee6b\U0001ee73\U0001ee78\U0001ee7d\U0001ee7f\U0001ee8a\U0001ee9c-\U0001eea0\U0001eea4\U0001eeaa\U0001eebc-\U0001eeef\U0001eef2-\U0001efff\U0001f02c-\U0001f02f\U0001f094-\U0001f09f\U0001f0af-\U0001f0b0\U0001f0c0\U0001f0d0\U0001f0f6-\U0001f0ff\U0001f10d-\U0001f10f\U0001f16c-\U0001f16f\U0001f1ad-\U0001f1e5\U0001f203-\U0001f20f\U0001f23c-\U0001f23f\U0001f249-\U0001f24f\U0001f252-\U0001f25f\U0001f266-\U0001f2ff\U0001f6d5-\U0001f6df\U0001f6ed-\U0001f6ef\U0001f6fa-\U0001f6ff\U0001f774-\U0001f77f\U0001f7d9-\U0001f7ff\U0001f80c-\U0001f80f\U0001f848-\U0001f84f\U0001f85a-\U0001f85f\U0001f888-\U0001f88f\U0001f8ae-\U0001f8ff\U0001f90c-\U0001f90f\U0001f93f\U0001f971-\U0001f972\U0001f977-\U0001f979\U0001f97b\U0001f9a3-\U0001f9af\U0001f9ba-\U0001f9bf\U0001f9c3-\U0001f9cf\U0001fa00-\U0001fa5f\U0001fa6e-\U0001ffff\U0002a6d7-\U0002a6ff\U0002b735-\U0002b73f\U0002b81e-\U0002b81f\U0002cea2-\U0002ceaf\U0002ebe1-\U0002f7ff\U0002fa1e-\U000e0000\U000e0002-\U000e001f\U000e0080-\U000e00ff\U000e01f0-\U000effff\U000ffffe-\U000fffff\U0010fffe-\U0010ffff' Co += u'\U000f0000-\U000ffffd\U00100000-\U0010fffd' - Ll += u'\U00010428-\U0001044f\U0001d41a-\U0001d433\U0001d44e-\U0001d454\U0001d456-\U0001d467\U0001d482-\U0001d49b\U0001d4b6-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d4cf\U0001d4ea-\U0001d503\U0001d51e-\U0001d537\U0001d552-\U0001d56b\U0001d586-\U0001d59f\U0001d5ba-\U0001d5d3\U0001d5ee-\U0001d607\U0001d622-\U0001d63b\U0001d656-\U0001d66f\U0001d68a-\U0001d6a5\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6e1\U0001d6fc-\U0001d714\U0001d716-\U0001d71b\U0001d736-\U0001d74e\U0001d750-\U0001d755\U0001d770-\U0001d788\U0001d78a-\U0001d78f\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7c9\U0001d7cb' + Ll += u'\U00010428-\U0001044f\U000104d8-\U000104fb\U00010cc0-\U00010cf2\U000118c0-\U000118df\U00016e60-\U00016e7f\U0001d41a-\U0001d433\U0001d44e-\U0001d454\U0001d456-\U0001d467\U0001d482-\U0001d49b\U0001d4b6-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d4cf\U0001d4ea-\U0001d503\U0001d51e-\U0001d537\U0001d552-\U0001d56b\U0001d586-\U0001d59f\U0001d5ba-\U0001d5d3\U0001d5ee-\U0001d607\U0001d622-\U0001d63b\U0001d656-\U0001d66f\U0001d68a-\U0001d6a5\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6e1\U0001d6fc-\U0001d714\U0001d716-\U0001d71b\U0001d736-\U0001d74e\U0001d750-\U0001d755\U0001d770-\U0001d788\U0001d78a-\U0001d78f\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7c9\U0001d7cb\U0001e922-\U0001e943' - Lm += u'\U00016f93-\U00016f9f' + Lm += u'\U00016b40-\U00016b43\U00016f93-\U00016f9f\U00016fe0-\U00016fe1' - Lo += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U00010340\U00010342-\U00010349\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U00010450-\U0001049d\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011183-\U000111b2\U000111c1-\U000111c4\U00011680-\U000116aa\U00012000-\U0001236e\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50\U0001b000-\U0001b001\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d' + Lo += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031f\U0001032d-\U00010340\U00010342-\U00010349\U00010350-\U00010375\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U00010450-\U0001049d\U00010500-\U00010527\U00010530-\U00010563\U00010600-\U00010736\U00010740-\U00010755\U00010760-\U00010767\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010860-\U00010876\U00010880-\U0001089e\U000108e0-\U000108f2\U000108f4-\U000108f5\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a35\U00010a60-\U00010a7c\U00010a80-\U00010a9c\U00010ac0-\U00010ac7\U00010ac9-\U00010ae4\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010b80-\U00010b91\U00010c00-\U00010c48\U00010d00-\U00010d23\U00010f00-\U00010f1c\U00010f27\U00010f30-\U00010f45\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011144\U00011150-\U00011172\U00011176\U00011183-\U000111b2\U000111c1-\U000111c4\U000111da\U000111dc\U00011200-\U00011211\U00011213-\U0001122b\U00011280-\U00011286\U00011288\U0001128a-\U0001128d\U0001128f-\U0001129d\U0001129f-\U000112a8\U000112b0-\U000112de\U00011305-\U0001130c\U0001130f-\U00011310\U00011313-\U00011328\U0001132a-\U00011330\U00011332-\U00011333\U00011335-\U00011339\U0001133d\U00011350\U0001135d-\U00011361\U00011400-\U00011434\U00011447-\U0001144a\U00011480-\U000114af\U000114c4-\U000114c5\U000114c7\U00011580-\U000115ae\U000115d8-\U000115db\U00011600-\U0001162f\U00011644\U00011680-\U000116aa\U00011700-\U0001171a\U00011800-\U0001182b\U000118ff\U00011a00\U00011a0b-\U00011a32\U00011a3a\U00011a50\U00011a5c-\U00011a83\U00011a86-\U00011a89\U00011a9d\U00011ac0-\U00011af8\U00011c00-\U00011c08\U00011c0a-\U00011c2e\U00011c40\U00011c72-\U00011c8f\U00011d00-\U00011d06\U00011d08-\U00011d09\U00011d0b-\U00011d30\U00011d46\U00011d60-\U00011d65\U00011d67-\U00011d68\U00011d6a-\U00011d89\U00011d98\U00011ee0-\U00011ef2\U00012000-\U00012399\U00012480-\U00012543\U00013000-\U0001342e\U00014400-\U00014646\U00016800-\U00016a38\U00016a40-\U00016a5e\U00016ad0-\U00016aed\U00016b00-\U00016b2f\U00016b63-\U00016b77\U00016b7d-\U00016b8f\U00016f00-\U00016f44\U00016f50\U00017000-\U000187f1\U00018800-\U00018af2\U0001b000-\U0001b11e\U0001b170-\U0001b2fb\U0001bc00-\U0001bc6a\U0001bc70-\U0001bc7c\U0001bc80-\U0001bc88\U0001bc90-\U0001bc99\U0001e800-\U0001e8c4\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002b820-\U0002cea1\U0002ceb0-\U0002ebe0\U0002f800-\U0002fa1d' - Lu += u'\U00010400-\U00010427\U0001d400-\U0001d419\U0001d434-\U0001d44d\U0001d468-\U0001d481\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b5\U0001d4d0-\U0001d4e9\U0001d504-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d538-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d56c-\U0001d585\U0001d5a0-\U0001d5b9\U0001d5d4-\U0001d5ed\U0001d608-\U0001d621\U0001d63c-\U0001d655\U0001d670-\U0001d689\U0001d6a8-\U0001d6c0\U0001d6e2-\U0001d6fa\U0001d71c-\U0001d734\U0001d756-\U0001d76e\U0001d790-\U0001d7a8\U0001d7ca' + Lu += u'\U00010400-\U00010427\U000104b0-\U000104d3\U00010c80-\U00010cb2\U000118a0-\U000118bf\U00016e40-\U00016e5f\U0001d400-\U0001d419\U0001d434-\U0001d44d\U0001d468-\U0001d481\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b5\U0001d4d0-\U0001d4e9\U0001d504-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d538-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d56c-\U0001d585\U0001d5a0-\U0001d5b9\U0001d5d4-\U0001d5ed\U0001d608-\U0001d621\U0001d63c-\U0001d655\U0001d670-\U0001d689\U0001d6a8-\U0001d6c0\U0001d6e2-\U0001d6fa\U0001d71c-\U0001d734\U0001d756-\U0001d76e\U0001d790-\U0001d7a8\U0001d7ca\U0001e900-\U0001e921' - Mc += u'\U00011000\U00011002\U00011082\U000110b0-\U000110b2\U000110b7-\U000110b8\U0001112c\U00011182\U000111b3-\U000111b5\U000111bf-\U000111c0\U000116ac\U000116ae-\U000116af\U000116b6\U00016f51-\U00016f7e\U0001d165-\U0001d166\U0001d16d-\U0001d172' + Mc += u'\U00011000\U00011002\U00011082\U000110b0-\U000110b2\U000110b7-\U000110b8\U0001112c\U00011145-\U00011146\U00011182\U000111b3-\U000111b5\U000111bf-\U000111c0\U0001122c-\U0001122e\U00011232-\U00011233\U00011235\U000112e0-\U000112e2\U00011302-\U00011303\U0001133e-\U0001133f\U00011341-\U00011344\U00011347-\U00011348\U0001134b-\U0001134d\U00011357\U00011362-\U00011363\U00011435-\U00011437\U00011440-\U00011441\U00011445\U000114b0-\U000114b2\U000114b9\U000114bb-\U000114be\U000114c1\U000115af-\U000115b1\U000115b8-\U000115bb\U000115be\U00011630-\U00011632\U0001163b-\U0001163c\U0001163e\U000116ac\U000116ae-\U000116af\U000116b6\U00011720-\U00011721\U00011726\U0001182c-\U0001182e\U00011838\U00011a39\U00011a57-\U00011a58\U00011a97\U00011c2f\U00011c3e\U00011ca9\U00011cb1\U00011cb4\U00011d8a-\U00011d8e\U00011d93-\U00011d94\U00011d96\U00011ef5-\U00011ef6\U00016f51-\U00016f7e\U0001d165-\U0001d166\U0001d16d-\U0001d172' - Mn += u'\U000101fd\U00010a01-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a0f\U00010a38-\U00010a3a\U00010a3f\U00011001\U00011038-\U00011046\U00011080-\U00011081\U000110b3-\U000110b6\U000110b9-\U000110ba\U00011100-\U00011102\U00011127-\U0001112b\U0001112d-\U00011134\U00011180-\U00011181\U000111b6-\U000111be\U000116ab\U000116ad\U000116b0-\U000116b5\U000116b7\U00016f8f-\U00016f92\U0001d167-\U0001d169\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U000e0100-\U000e01ef' + Mn += u'\U000101fd\U000102e0\U00010376-\U0001037a\U00010a01-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a0f\U00010a38-\U00010a3a\U00010a3f\U00010ae5-\U00010ae6\U00010d24-\U00010d27\U00010f46-\U00010f50\U00011001\U00011038-\U00011046\U0001107f-\U00011081\U000110b3-\U000110b6\U000110b9-\U000110ba\U00011100-\U00011102\U00011127-\U0001112b\U0001112d-\U00011134\U00011173\U00011180-\U00011181\U000111b6-\U000111be\U000111c9-\U000111cc\U0001122f-\U00011231\U00011234\U00011236-\U00011237\U0001123e\U000112df\U000112e3-\U000112ea\U00011300-\U00011301\U0001133b-\U0001133c\U00011340\U00011366-\U0001136c\U00011370-\U00011374\U00011438-\U0001143f\U00011442-\U00011444\U00011446\U0001145e\U000114b3-\U000114b8\U000114ba\U000114bf-\U000114c0\U000114c2-\U000114c3\U000115b2-\U000115b5\U000115bc-\U000115bd\U000115bf-\U000115c0\U000115dc-\U000115dd\U00011633-\U0001163a\U0001163d\U0001163f-\U00011640\U000116ab\U000116ad\U000116b0-\U000116b5\U000116b7\U0001171d-\U0001171f\U00011722-\U00011725\U00011727-\U0001172b\U0001182f-\U00011837\U00011839-\U0001183a\U00011a01-\U00011a0a\U00011a33-\U00011a38\U00011a3b-\U00011a3e\U00011a47\U00011a51-\U00011a56\U00011a59-\U00011a5b\U00011a8a-\U00011a96\U00011a98-\U00011a99\U00011c30-\U00011c36\U00011c38-\U00011c3d\U00011c3f\U00011c92-\U00011ca7\U00011caa-\U00011cb0\U00011cb2-\U00011cb3\U00011cb5-\U00011cb6\U00011d31-\U00011d36\U00011d3a\U00011d3c-\U00011d3d\U00011d3f-\U00011d45\U00011d47\U00011d90-\U00011d91\U00011d95\U00011d97\U00011ef3-\U00011ef4\U00016af0-\U00016af4\U00016b30-\U00016b36\U00016f8f-\U00016f92\U0001bc9d-\U0001bc9e\U0001d167-\U0001d169\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U0001da00-\U0001da36\U0001da3b-\U0001da6c\U0001da75\U0001da84\U0001da9b-\U0001da9f\U0001daa1-\U0001daaf\U0001e000-\U0001e006\U0001e008-\U0001e018\U0001e01b-\U0001e021\U0001e023-\U0001e024\U0001e026-\U0001e02a\U0001e8d0-\U0001e8d6\U0001e944-\U0001e94a\U000e0100-\U000e01ef' - Nd += u'\U000104a0-\U000104a9\U00011066-\U0001106f\U000110f0-\U000110f9\U00011136-\U0001113f\U000111d0-\U000111d9\U000116c0-\U000116c9\U0001d7ce-\U0001d7ff' + Nd += u'\U000104a0-\U000104a9\U00010d30-\U00010d39\U00011066-\U0001106f\U000110f0-\U000110f9\U00011136-\U0001113f\U000111d0-\U000111d9\U000112f0-\U000112f9\U00011450-\U00011459\U000114d0-\U000114d9\U00011650-\U00011659\U000116c0-\U000116c9\U00011730-\U00011739\U000118e0-\U000118e9\U00011c50-\U00011c59\U00011d50-\U00011d59\U00011da0-\U00011da9\U00016a60-\U00016a69\U00016b50-\U00016b59\U0001d7ce-\U0001d7ff\U0001e950-\U0001e959' - Nl += u'\U00010140-\U00010174\U00010341\U0001034a\U000103d1-\U000103d5\U00012400-\U00012462' + Nl += u'\U00010140-\U00010174\U00010341\U0001034a\U000103d1-\U000103d5\U00012400-\U0001246e' - No += u'\U00010107-\U00010133\U00010175-\U00010178\U0001018a\U00010320-\U00010323\U00010858-\U0001085f\U00010916-\U0001091b\U00010a40-\U00010a47\U00010a7d-\U00010a7e\U00010b58-\U00010b5f\U00010b78-\U00010b7f\U00010e60-\U00010e7e\U00011052-\U00011065\U0001d360-\U0001d371\U0001f100-\U0001f10a' + No += u'\U00010107-\U00010133\U00010175-\U00010178\U0001018a-\U0001018b\U000102e1-\U000102fb\U00010320-\U00010323\U00010858-\U0001085f\U00010879-\U0001087f\U000108a7-\U000108af\U000108fb-\U000108ff\U00010916-\U0001091b\U000109bc-\U000109bd\U000109c0-\U000109cf\U000109d2-\U000109ff\U00010a40-\U00010a48\U00010a7d-\U00010a7e\U00010a9d-\U00010a9f\U00010aeb-\U00010aef\U00010b58-\U00010b5f\U00010b78-\U00010b7f\U00010ba9-\U00010baf\U00010cfa-\U00010cff\U00010e60-\U00010e7e\U00010f1d-\U00010f26\U00010f51-\U00010f54\U00011052-\U00011065\U000111e1-\U000111f4\U0001173a-\U0001173b\U000118ea-\U000118f2\U00011c5a-\U00011c6c\U00016b5b-\U00016b61\U00016e80-\U00016e96\U0001d2e0-\U0001d2f3\U0001d360-\U0001d378\U0001e8c7-\U0001e8cf\U0001ec71-\U0001ecab\U0001ecad-\U0001ecaf\U0001ecb1-\U0001ecb4\U0001f100-\U0001f10c' - Po += u'\U00010100-\U00010102\U0001039f\U000103d0\U00010857\U0001091f\U0001093f\U00010a50-\U00010a58\U00010a7f\U00010b39-\U00010b3f\U00011047-\U0001104d\U000110bb-\U000110bc\U000110be-\U000110c1\U00011140-\U00011143\U000111c5-\U000111c8\U00012470-\U00012473' + Po += u'\U00010100-\U00010102\U0001039f\U000103d0\U0001056f\U00010857\U0001091f\U0001093f\U00010a50-\U00010a58\U00010a7f\U00010af0-\U00010af6\U00010b39-\U00010b3f\U00010b99-\U00010b9c\U00010f55-\U00010f59\U00011047-\U0001104d\U000110bb-\U000110bc\U000110be-\U000110c1\U00011140-\U00011143\U00011174-\U00011175\U000111c5-\U000111c8\U000111cd\U000111db\U000111dd-\U000111df\U00011238-\U0001123d\U000112a9\U0001144b-\U0001144f\U0001145b\U0001145d\U000114c6\U000115c1-\U000115d7\U00011641-\U00011643\U00011660-\U0001166c\U0001173c-\U0001173e\U0001183b\U00011a3f-\U00011a46\U00011a9a-\U00011a9c\U00011a9e-\U00011aa2\U00011c41-\U00011c45\U00011c70-\U00011c71\U00011ef7-\U00011ef8\U00012470-\U00012474\U00016a6e-\U00016a6f\U00016af5\U00016b37-\U00016b3b\U00016b44\U00016e97-\U00016e9a\U0001bc9f\U0001da87-\U0001da8b\U0001e95e-\U0001e95f' + + Sc += u'\U0001ecb0' + + Sk += u'\U0001f3fb-\U0001f3ff' Sm += u'\U0001d6c1\U0001d6db\U0001d6fb\U0001d715\U0001d735\U0001d74f\U0001d76f\U0001d789\U0001d7a9\U0001d7c3\U0001eef0-\U0001eef1' - So += u'\U00010137-\U0001013f\U00010179-\U00010189\U00010190-\U0001019b\U000101d0-\U000101fc\U0001d000-\U0001d0f5\U0001d100-\U0001d126\U0001d129-\U0001d164\U0001d16a-\U0001d16c\U0001d183-\U0001d184\U0001d18c-\U0001d1a9\U0001d1ae-\U0001d1dd\U0001d200-\U0001d241\U0001d245\U0001d300-\U0001d356\U0001f000-\U0001f02b\U0001f030-\U0001f093\U0001f0a0-\U0001f0ae\U0001f0b1-\U0001f0be\U0001f0c1-\U0001f0cf\U0001f0d1-\U0001f0df\U0001f110-\U0001f12e\U0001f130-\U0001f16b\U0001f170-\U0001f19a\U0001f1e6-\U0001f202\U0001f210-\U0001f23a\U0001f240-\U0001f248\U0001f250-\U0001f251\U0001f300-\U0001f320\U0001f330-\U0001f335\U0001f337-\U0001f37c\U0001f380-\U0001f393\U0001f3a0-\U0001f3c4\U0001f3c6-\U0001f3ca\U0001f3e0-\U0001f3f0\U0001f400-\U0001f43e\U0001f440\U0001f442-\U0001f4f7\U0001f4f9-\U0001f4fc\U0001f500-\U0001f53d\U0001f540-\U0001f543\U0001f550-\U0001f567\U0001f5fb-\U0001f640\U0001f645-\U0001f64f\U0001f680-\U0001f6c5\U0001f700-\U0001f773' + So += u'\U00010137-\U0001013f\U00010179-\U00010189\U0001018c-\U0001018e\U00010190-\U0001019b\U000101a0\U000101d0-\U000101fc\U00010877-\U00010878\U00010ac8\U0001173f\U00016b3c-\U00016b3f\U00016b45\U0001bc9c\U0001d000-\U0001d0f5\U0001d100-\U0001d126\U0001d129-\U0001d164\U0001d16a-\U0001d16c\U0001d183-\U0001d184\U0001d18c-\U0001d1a9\U0001d1ae-\U0001d1e8\U0001d200-\U0001d241\U0001d245\U0001d300-\U0001d356\U0001d800-\U0001d9ff\U0001da37-\U0001da3a\U0001da6d-\U0001da74\U0001da76-\U0001da83\U0001da85-\U0001da86\U0001ecac\U0001f000-\U0001f02b\U0001f030-\U0001f093\U0001f0a0-\U0001f0ae\U0001f0b1-\U0001f0bf\U0001f0c1-\U0001f0cf\U0001f0d1-\U0001f0f5\U0001f110-\U0001f16b\U0001f170-\U0001f1ac\U0001f1e6-\U0001f202\U0001f210-\U0001f23b\U0001f240-\U0001f248\U0001f250-\U0001f251\U0001f260-\U0001f265\U0001f300-\U0001f3fa\U0001f400-\U0001f6d4\U0001f6e0-\U0001f6ec\U0001f6f0-\U0001f6f9\U0001f700-\U0001f773\U0001f780-\U0001f7d8\U0001f800-\U0001f80b\U0001f810-\U0001f847\U0001f850-\U0001f859\U0001f860-\U0001f887\U0001f890-\U0001f8ad\U0001f900-\U0001f90b\U0001f910-\U0001f93e\U0001f940-\U0001f970\U0001f973-\U0001f976\U0001f97a\U0001f97c-\U0001f9a2\U0001f9b0-\U0001f9b9\U0001f9c0-\U0001f9c2\U0001f9d0-\U0001f9ff\U0001fa60-\U0001fa6d' - xid_continue += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U000101fd\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U0001034a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U000104a0-\U000104a9\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a38-\U00010a3a\U00010a3f\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011000-\U00011046\U00011066-\U0001106f\U00011080-\U000110ba\U000110d0-\U000110e8\U000110f0-\U000110f9\U00011100-\U00011134\U00011136-\U0001113f\U00011180-\U000111c4\U000111d0-\U000111d9\U00011680-\U000116b7\U000116c0-\U000116c9\U00012000-\U0001236e\U00012400-\U00012462\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50-\U00016f7e\U00016f8f-\U00016f9f\U0001b000-\U0001b001\U0001d165-\U0001d169\U0001d16d-\U0001d172\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001d7ce-\U0001d7ff\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d\U000e0100-\U000e01ef' + xid_continue += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U000101fd\U00010280-\U0001029c\U000102a0-\U000102d0\U000102e0\U00010300-\U0001031f\U0001032d-\U0001034a\U00010350-\U0001037a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U000104a0-\U000104a9\U000104b0-\U000104d3\U000104d8-\U000104fb\U00010500-\U00010527\U00010530-\U00010563\U00010600-\U00010736\U00010740-\U00010755\U00010760-\U00010767\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010860-\U00010876\U00010880-\U0001089e\U000108e0-\U000108f2\U000108f4-\U000108f5\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00-\U00010a03\U00010a05-\U00010a06\U00010a0c-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a35\U00010a38-\U00010a3a\U00010a3f\U00010a60-\U00010a7c\U00010a80-\U00010a9c\U00010ac0-\U00010ac7\U00010ac9-\U00010ae6\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010b80-\U00010b91\U00010c00-\U00010c48\U00010c80-\U00010cb2\U00010cc0-\U00010cf2\U00010d00-\U00010d27\U00010d30-\U00010d39\U00010f00-\U00010f1c\U00010f27\U00010f30-\U00010f50\U00011000-\U00011046\U00011066-\U0001106f\U0001107f-\U000110ba\U000110d0-\U000110e8\U000110f0-\U000110f9\U00011100-\U00011134\U00011136-\U0001113f\U00011144-\U00011146\U00011150-\U00011173\U00011176\U00011180-\U000111c4\U000111c9-\U000111cc\U000111d0-\U000111da\U000111dc\U00011200-\U00011211\U00011213-\U00011237\U0001123e\U00011280-\U00011286\U00011288\U0001128a-\U0001128d\U0001128f-\U0001129d\U0001129f-\U000112a8\U000112b0-\U000112ea\U000112f0-\U000112f9\U00011300-\U00011303\U00011305-\U0001130c\U0001130f-\U00011310\U00011313-\U00011328\U0001132a-\U00011330\U00011332-\U00011333\U00011335-\U00011339\U0001133b-\U00011344\U00011347-\U00011348\U0001134b-\U0001134d\U00011350\U00011357\U0001135d-\U00011363\U00011366-\U0001136c\U00011370-\U00011374\U00011400-\U0001144a\U00011450-\U00011459\U0001145e\U00011480-\U000114c5\U000114c7\U000114d0-\U000114d9\U00011580-\U000115b5\U000115b8-\U000115c0\U000115d8-\U000115dd\U00011600-\U00011640\U00011644\U00011650-\U00011659\U00011680-\U000116b7\U000116c0-\U000116c9\U00011700-\U0001171a\U0001171d-\U0001172b\U00011730-\U00011739\U00011800-\U0001183a\U000118a0-\U000118e9\U000118ff\U00011a00-\U00011a3e\U00011a47\U00011a50-\U00011a83\U00011a86-\U00011a99\U00011a9d\U00011ac0-\U00011af8\U00011c00-\U00011c08\U00011c0a-\U00011c36\U00011c38-\U00011c40\U00011c50-\U00011c59\U00011c72-\U00011c8f\U00011c92-\U00011ca7\U00011ca9-\U00011cb6\U00011d00-\U00011d06\U00011d08-\U00011d09\U00011d0b-\U00011d36\U00011d3a\U00011d3c-\U00011d3d\U00011d3f-\U00011d47\U00011d50-\U00011d59\U00011d60-\U00011d65\U00011d67-\U00011d68\U00011d6a-\U00011d8e\U00011d90-\U00011d91\U00011d93-\U00011d98\U00011da0-\U00011da9\U00011ee0-\U00011ef6\U00012000-\U00012399\U00012400-\U0001246e\U00012480-\U00012543\U00013000-\U0001342e\U00014400-\U00014646\U00016800-\U00016a38\U00016a40-\U00016a5e\U00016a60-\U00016a69\U00016ad0-\U00016aed\U00016af0-\U00016af4\U00016b00-\U00016b36\U00016b40-\U00016b43\U00016b50-\U00016b59\U00016b63-\U00016b77\U00016b7d-\U00016b8f\U00016e40-\U00016e7f\U00016f00-\U00016f44\U00016f50-\U00016f7e\U00016f8f-\U00016f9f\U00016fe0-\U00016fe1\U00017000-\U000187f1\U00018800-\U00018af2\U0001b000-\U0001b11e\U0001b170-\U0001b2fb\U0001bc00-\U0001bc6a\U0001bc70-\U0001bc7c\U0001bc80-\U0001bc88\U0001bc90-\U0001bc99\U0001bc9d-\U0001bc9e\U0001d165-\U0001d169\U0001d16d-\U0001d172\U0001d17b-\U0001d182\U0001d185-\U0001d18b\U0001d1aa-\U0001d1ad\U0001d242-\U0001d244\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001d7ce-\U0001d7ff\U0001da00-\U0001da36\U0001da3b-\U0001da6c\U0001da75\U0001da84\U0001da9b-\U0001da9f\U0001daa1-\U0001daaf\U0001e000-\U0001e006\U0001e008-\U0001e018\U0001e01b-\U0001e021\U0001e023-\U0001e024\U0001e026-\U0001e02a\U0001e800-\U0001e8c4\U0001e8d0-\U0001e8d6\U0001e900-\U0001e94a\U0001e950-\U0001e959\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002b820-\U0002cea1\U0002ceb0-\U0002ebe0\U0002f800-\U0002fa1d\U000e0100-\U000e01ef' - xid_start += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031e\U00010330-\U0001034a\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a33\U00010a60-\U00010a7c\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010c00-\U00010c48\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011183-\U000111b2\U000111c1-\U000111c4\U00011680-\U000116aa\U00012000-\U0001236e\U00012400-\U00012462\U00013000-\U0001342e\U00016800-\U00016a38\U00016f00-\U00016f44\U00016f50\U00016f93-\U00016f9f\U0001b000-\U0001b001\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002f800-\U0002fa1d' + xid_start += u'\U00010000-\U0001000b\U0001000d-\U00010026\U00010028-\U0001003a\U0001003c-\U0001003d\U0001003f-\U0001004d\U00010050-\U0001005d\U00010080-\U000100fa\U00010140-\U00010174\U00010280-\U0001029c\U000102a0-\U000102d0\U00010300-\U0001031f\U0001032d-\U0001034a\U00010350-\U00010375\U00010380-\U0001039d\U000103a0-\U000103c3\U000103c8-\U000103cf\U000103d1-\U000103d5\U00010400-\U0001049d\U000104b0-\U000104d3\U000104d8-\U000104fb\U00010500-\U00010527\U00010530-\U00010563\U00010600-\U00010736\U00010740-\U00010755\U00010760-\U00010767\U00010800-\U00010805\U00010808\U0001080a-\U00010835\U00010837-\U00010838\U0001083c\U0001083f-\U00010855\U00010860-\U00010876\U00010880-\U0001089e\U000108e0-\U000108f2\U000108f4-\U000108f5\U00010900-\U00010915\U00010920-\U00010939\U00010980-\U000109b7\U000109be-\U000109bf\U00010a00\U00010a10-\U00010a13\U00010a15-\U00010a17\U00010a19-\U00010a35\U00010a60-\U00010a7c\U00010a80-\U00010a9c\U00010ac0-\U00010ac7\U00010ac9-\U00010ae4\U00010b00-\U00010b35\U00010b40-\U00010b55\U00010b60-\U00010b72\U00010b80-\U00010b91\U00010c00-\U00010c48\U00010c80-\U00010cb2\U00010cc0-\U00010cf2\U00010d00-\U00010d23\U00010f00-\U00010f1c\U00010f27\U00010f30-\U00010f45\U00011003-\U00011037\U00011083-\U000110af\U000110d0-\U000110e8\U00011103-\U00011126\U00011144\U00011150-\U00011172\U00011176\U00011183-\U000111b2\U000111c1-\U000111c4\U000111da\U000111dc\U00011200-\U00011211\U00011213-\U0001122b\U00011280-\U00011286\U00011288\U0001128a-\U0001128d\U0001128f-\U0001129d\U0001129f-\U000112a8\U000112b0-\U000112de\U00011305-\U0001130c\U0001130f-\U00011310\U00011313-\U00011328\U0001132a-\U00011330\U00011332-\U00011333\U00011335-\U00011339\U0001133d\U00011350\U0001135d-\U00011361\U00011400-\U00011434\U00011447-\U0001144a\U00011480-\U000114af\U000114c4-\U000114c5\U000114c7\U00011580-\U000115ae\U000115d8-\U000115db\U00011600-\U0001162f\U00011644\U00011680-\U000116aa\U00011700-\U0001171a\U00011800-\U0001182b\U000118a0-\U000118df\U000118ff\U00011a00\U00011a0b-\U00011a32\U00011a3a\U00011a50\U00011a5c-\U00011a83\U00011a86-\U00011a89\U00011a9d\U00011ac0-\U00011af8\U00011c00-\U00011c08\U00011c0a-\U00011c2e\U00011c40\U00011c72-\U00011c8f\U00011d00-\U00011d06\U00011d08-\U00011d09\U00011d0b-\U00011d30\U00011d46\U00011d60-\U00011d65\U00011d67-\U00011d68\U00011d6a-\U00011d89\U00011d98\U00011ee0-\U00011ef2\U00012000-\U00012399\U00012400-\U0001246e\U00012480-\U00012543\U00013000-\U0001342e\U00014400-\U00014646\U00016800-\U00016a38\U00016a40-\U00016a5e\U00016ad0-\U00016aed\U00016b00-\U00016b2f\U00016b40-\U00016b43\U00016b63-\U00016b77\U00016b7d-\U00016b8f\U00016e40-\U00016e7f\U00016f00-\U00016f44\U00016f50\U00016f93-\U00016f9f\U00016fe0-\U00016fe1\U00017000-\U000187f1\U00018800-\U00018af2\U0001b000-\U0001b11e\U0001b170-\U0001b2fb\U0001bc00-\U0001bc6a\U0001bc70-\U0001bc7c\U0001bc80-\U0001bc88\U0001bc90-\U0001bc99\U0001d400-\U0001d454\U0001d456-\U0001d49c\U0001d49e-\U0001d49f\U0001d4a2\U0001d4a5-\U0001d4a6\U0001d4a9-\U0001d4ac\U0001d4ae-\U0001d4b9\U0001d4bb\U0001d4bd-\U0001d4c3\U0001d4c5-\U0001d505\U0001d507-\U0001d50a\U0001d50d-\U0001d514\U0001d516-\U0001d51c\U0001d51e-\U0001d539\U0001d53b-\U0001d53e\U0001d540-\U0001d544\U0001d546\U0001d54a-\U0001d550\U0001d552-\U0001d6a5\U0001d6a8-\U0001d6c0\U0001d6c2-\U0001d6da\U0001d6dc-\U0001d6fa\U0001d6fc-\U0001d714\U0001d716-\U0001d734\U0001d736-\U0001d74e\U0001d750-\U0001d76e\U0001d770-\U0001d788\U0001d78a-\U0001d7a8\U0001d7aa-\U0001d7c2\U0001d7c4-\U0001d7cb\U0001e800-\U0001e8c4\U0001e900-\U0001e943\U0001ee00-\U0001ee03\U0001ee05-\U0001ee1f\U0001ee21-\U0001ee22\U0001ee24\U0001ee27\U0001ee29-\U0001ee32\U0001ee34-\U0001ee37\U0001ee39\U0001ee3b\U0001ee42\U0001ee47\U0001ee49\U0001ee4b\U0001ee4d-\U0001ee4f\U0001ee51-\U0001ee52\U0001ee54\U0001ee57\U0001ee59\U0001ee5b\U0001ee5d\U0001ee5f\U0001ee61-\U0001ee62\U0001ee64\U0001ee67-\U0001ee6a\U0001ee6c-\U0001ee72\U0001ee74-\U0001ee77\U0001ee79-\U0001ee7c\U0001ee7e\U0001ee80-\U0001ee89\U0001ee8b-\U0001ee9b\U0001eea1-\U0001eea3\U0001eea5-\U0001eea9\U0001eeab-\U0001eebb\U00020000-\U0002a6d6\U0002a700-\U0002b734\U0002b740-\U0002b81d\U0002b820-\U0002cea1\U0002ceb0-\U0002ebe0\U0002f800-\U0002fa1d' cats = ['Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu', 'Mc', 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf', 'Pi', 'Po', 'Ps', 'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs'] -# Generated from unidata 6.3.0 +# Generated from unidata 11.0.0 def combine(*args): return u''.join(globals()[cat] for cat in args) diff --git a/pygments/util.py b/pygments/util.py index 45070063..054b705e 100644 --- a/pygments/util.py +++ b/pygments/util.py @@ -5,7 +5,7 @@ Utility functions. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/scripts/check_sources.py b/scripts/check_sources.py index db09de42..f047168c 100755 --- a/scripts/check_sources.py +++ b/scripts/check_sources.py @@ -7,7 +7,7 @@ Make sure each Python file has a correct file header including copyright and license information. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -36,7 +36,7 @@ def checker(*suffixes, **kwds): name_mail_re = r'[\w ]+(<.*?>)?' -copyright_re = re.compile(r'^ :copyright: Copyright 2006-2017 by ' +copyright_re = re.compile(r'^ :copyright: Copyright 2006-2019 by ' r'the Pygments team, see AUTHORS\.$', re.UNICODE) copyright_2_re = re.compile(r'^ %s(, %s)*[,.]$' % (name_mail_re, name_mail_re), re.UNICODE) @@ -185,7 +185,8 @@ def main(argv): print("Checking %s..." % fn) try: - lines = open(fn, 'rb').read().decode('utf-8').splitlines() + with open(fn, 'rb') as f: + lines = f.read().decode('utf-8').splitlines() except (IOError, OSError) as err: print("%s: cannot open: %s" % (fn, err)) num += 1 diff --git a/scripts/debug_lexer.py b/scripts/debug_lexer.py index 02bb9fef..ef01a23f 100755 --- a/scripts/debug_lexer.py +++ b/scripts/debug_lexer.py @@ -8,7 +8,7 @@ the text where Error tokens are being generated, along with some context. - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/scripts/get_vimkw.py b/scripts/get_vimkw.py index 688a0c64..a1b6873e 100644 --- a/scripts/get_vimkw.py +++ b/scripts/get_vimkw.py @@ -16,7 +16,7 @@ HEADER = '''\ This file is autogenerated by scripts/get_vimkw.py - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/scripts/release-checklist b/scripts/release-checklist new file mode 100644 index 00000000..efc1e1e8 --- /dev/null +++ b/scripts/release-checklist @@ -0,0 +1,24 @@ +Release checklist +================= + +* Check ``hg status`` +* ``make check`` +* LATER when configured properly: ``make pylint`` +* ``tox`` +* Update version info in ``setup.py/__init__.py`` +* Check setup.py metadata: long description, trove classifiers +* Update release date/code name in ``CHANGES`` +* ``hg commit`` +* ``make clean`` +* ``python2 setup.py release bdist_wheel`` +* ``python3 setup.py release bdist_wheel sdist`` +* ``twine upload dist/Pygments-$NEWVER*`` +* Check PyPI release page for obvious errors +* ``hg tag`` +* Merge default into stable if this was a ``x.y.0`` +* Update homepage (release info), regenerate docs (+printable!) +* Add new version/milestone to tracker categories +* Write announcement and send to mailing list/python-announce +* Update version info, add new ``CHANGES`` entry for next version +* ``hg commit`` +* ``hg push`` @@ -1,23 +1,24 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -"""Pygments - ~~~~~~~~ +""" +Pygments +~~~~~~~~ - Pygments is a syntax highlighting package written in Python. +Pygments is a syntax highlighting package written in Python. - It is a generic syntax highlighter suitable for use in code hosting, forums, - wikis or other applications that need to prettify source code. Highlights - are: +It is a generic syntax highlighter suitable for use in code hosting, forums, +wikis or other applications that need to prettify source code. Highlights +are: - * a wide range of over 300 languages and other text formats is supported - * special attention is paid to details, increasing quality by a fair amount - * support for new languages and formats are added easily - * a number of output formats, presently HTML, LaTeX, RTF, SVG, all image \ - formats that PIL supports and ANSI sequences - * it is usable as a command-line tool and as a library +* a wide range of over 300 languages and other text formats is supported +* special attention is paid to details, increasing quality by a fair amount +* support for new languages and formats are added easily +* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image \ + formats that PIL supports and ANSI sequences +* it is usable as a command-line tool and as a library - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. +:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. +:license: BSD, see LICENSE for details. """ try: @@ -48,7 +49,7 @@ else: setup( name = 'Pygments', - version = '2.2.0', + version = '2.4.2', url = 'http://pygments.org/', license = 'BSD License', author = 'Georg Brandl', @@ -60,6 +61,7 @@ setup( platforms = 'any', zip_safe = False, include_package_data = True, + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', classifiers = [ 'License :: OSI Approved :: BSD License', 'Intended Audience :: Developers', @@ -68,7 +70,11 @@ setup( 'Development Status :: 6 - Mature', 'Programming Language :: Python', 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Operating System :: OS Independent', 'Topic :: Text Processing :: Filters', 'Topic :: Utilities', diff --git a/tests/examplefiles/99_bottles_of_beer.chpl b/tests/examplefiles/99_bottles_of_beer.chpl index cdc1e650..ff50b294 100644 --- a/tests/examplefiles/99_bottles_of_beer.chpl +++ b/tests/examplefiles/99_bottles_of_beer.chpl @@ -177,3 +177,30 @@ private module M3 { private var x: int; } +prototype module X { + + proc f() throws { + throw new Error(); + } + + proc g() { + try { + f(); + try! f(); + } catch e { + writeln("Caught ", e); + } + } + + proc int.add() { } + + g(); + + override proc test() throws { + var a = new borrowed IntPair(); + var b = new owned IntPair(); + var c = new shared IntPair(); + throw new unmanaged Error(); + } +} + diff --git a/tests/examplefiles/Charmci.ci b/tests/examplefiles/Charmci.ci new file mode 100644 index 00000000..2e5cd5c6 --- /dev/null +++ b/tests/examplefiles/Charmci.ci @@ -0,0 +1,20 @@ +module CkCallback { + readonly CProxy_ckcallback_group _ckcallbackgroup; + message CkCcsRequestMsg { + char data[]; + }; + message CkDataMsg { + char data[]; + }; + + mainchare ckcallback_main { + entry ckcallback_main(CkArgMsg *m); + }; + group [migratable] ckcallback_group : IrrGroup { + entry ckcallback_group(); + entry void registerCcsCallback(char name[strlen(name)+1], + CkCallback cb); + entry void call(CkCallback c,CkMarshalledMessage msg); + entry void call(CkCallback c, int length, char data[length]); + }; +}; diff --git a/tests/examplefiles/StdGeneric.icl b/tests/examplefiles/StdGeneric.icl index 2e6c3931..891b510a 100644 --- a/tests/examplefiles/StdGeneric.icl +++ b/tests/examplefiles/StdGeneric.icl @@ -1,5 +1,13 @@ implementation module StdGeneric +/** + * NOTE: this is a collection of different tricky parts of Clean modules (even + * though the file is simply called StdGeneric.icl). The code is taken from: + * + * - StdGeneric (StdEnv) + * - Graphics.Scalable.Image (Platform) + */ + import StdInt, StdMisc, StdClass, StdFunc generic bimap a b :: Bimap .a .b @@ -89,4 +97,38 @@ where = [ ConsLeft : doit i (n/2) ] | otherwise = [ ConsRight : doit (i - (n/2)) (n - (n/2)) ] -
\ No newline at end of file + +:: NoAttr m = NoAttr +:: DashAttr m = { dash :: ![Int] } +:: FillAttr m = { fill :: !SVGColor } +:: LineEndMarker m = { endmarker :: !Image m } +:: LineMidMarker m = { midmarker :: !Image m } +:: LineStartMarker m = { startmarker :: !Image m } +:: MaskAttr m = { mask :: !Image m } +:: OpacityAttr m = { opacity :: !Real } +:: StrokeAttr m = { stroke :: !SVGColor } +:: StrokeWidthAttr m = { strokewidth :: !Span } +:: XRadiusAttr m = { xradius :: !Span } +:: YRadiusAttr m = { yradius :: !Span } + + +instance tuneImage NoAttr where tuneImage image _ = image +instance tuneImage DashAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgDashAttr attr.DashAttr.dash)) image +instance tuneImage FillAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgFillAttr attr.FillAttr.fill)) image +instance tuneImage LineEndMarker where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineEndMarker.endmarker, markerPos = LineMarkerEnd}) image +instance tuneImage LineMidMarker where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineMidMarker.midmarker, markerPos = LineMarkerMid}) image +instance tuneImage LineStartMarker where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineStartMarker.startmarker, markerPos = LineMarkerStart}) image +instance tuneImage MaskAttr where tuneImage image attr = Attr` (MaskAttr` attr.MaskAttr.mask) image +instance tuneImage OpacityAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgFillOpacityAttr attr.OpacityAttr.opacity)) image +instance tuneImage StrokeAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgStrokeAttr attr.StrokeAttr.stroke)) image +instance tuneImage StrokeWidthAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgStrokeWidthAttr attr.StrokeWidthAttr.strokewidth)) image +instance tuneImage XRadiusAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgXRadiusAttr attr.XRadiusAttr.xradius)) image +instance tuneImage YRadiusAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgYRadiusAttr attr.YRadiusAttr.yradius)) image + +instance tuneImage DraggableAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerDraggableAttr attr)) image +instance tuneImage OnClickAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnClickAttr attr)) image +instance tuneImage OnMouseDownAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseDownAttr attr)) image +instance tuneImage OnMouseMoveAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseMoveAttr attr)) image +instance tuneImage OnMouseOutAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseOutAttr attr)) image +instance tuneImage OnMouseOverAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseOverAttr attr)) image +instance tuneImage OnMouseUpAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseUpAttr attr)) image diff --git a/tests/examplefiles/apache2.conf b/tests/examplefiles/apache2.conf index d0e838e0..5db66b10 100644 --- a/tests/examplefiles/apache2.conf +++ b/tests/examplefiles/apache2.conf @@ -391,3 +391,8 @@ BrowserMatch "^WebDAVFS/1.[012]" redirect-carefully # Include the virtual host configurations: Include /etc/apache2/sites-enabled/[^.#]* + +# From PR#766 +<IfVersion >= 2.4> +ErrorLogFormat "%{cu}t %M" +</IfVersion>
\ No newline at end of file diff --git a/tests/examplefiles/docker.docker b/tests/examplefiles/docker.docker index d65385b6..1ae3c3a1 100644 --- a/tests/examplefiles/docker.docker +++ b/tests/examplefiles/docker.docker @@ -1,5 +1,34 @@ -maintainer First O'Last +FROM alpine:3.5 +MAINTAINER First O'Last +# comment run echo \ 123 $bar -# comment +RUN apk --update add rsync dumb-init + +# Test env with both syntax +ENV FOO = "BAR" +ENV FOO \ + "BAR" + +COPY foo "bar" +COPY foo \ + "bar" + +HEALTHCHECK \ + --interval=5m --timeout=3s \ + CMD curl -f http://localhost/ || exit 1 + +# ONBUILD keyword, then with linebreak +ONBUILD ADD . /app/src +ONBUILD \ + RUN echo 123 $bar + +# Potential JSON array parsing, mixed with linebreaks +VOLUME \ + /foo +VOLUME \ + ["/bar"] +VOLUME ["/bar"] +VOLUME /foo +CMD ["foo", "bar"] diff --git a/tests/examplefiles/example.bbc b/tests/examplefiles/example.bbc new file mode 100644 index 00000000..ebdb8537 --- /dev/null +++ b/tests/examplefiles/example.bbc @@ -0,0 +1,156 @@ +10REM >EIRC +20REM The simplest IRC client you can write. Maybe. +30REM (C) Justin Fletcher, 1998 +40: +50END=PAGE+1024*16 +60REM Change these if you wish +70host$="irc.stealth.net" +80port=6667 +90nick$="eirc" +100ourchan$="#acorn" +110: +120REM Start connecting to a host +130SYS "ESocket_ConnectToHost",host$,port TO handle +140REPEAT +150 SYS "ESocket_CheckState",handle TO state +160 IF state<-1 THENSYS "ESocket_Forget",handle:SYS "ESocket_DecodeState",state TO a$:ERROR 1,"Failed ("+a$+")" +170UNTIL state=4 +180: +190REM We are now connected +200PRINT"Connected" +210: +220REM Log on to the server +230SYS "ESocket_SendLine",handle,"USER "+nick$+" x x :"+nick$ +240SYS "ESocket_SendLine",handle,"NICK "+nick$ +250SYS "ESocket_SendLine",handle,"JOIN "+ourchan$ +260REM Install a monitor so that we don't waste time +270SYS "ESocket_Monitor",0,handle TO monitor +280SYS "ESocket_ResetMonitor",monitor,0 TO polladdr% +290: +300REM If we crash, we should tidy up after ourselves +310ON ERROR SYS "XESocket_Forget",handle:SYS "XESocket_Forget",monitor:ERROR EXT ERR,REPORT$+" at line "+STR$ERL +320: +330REM Memory buffer for our data +340bufsize%=1024 +350DIM buf% bufsize% +360: +370input$="":REM The input line +380REPEAT +390 REM In a taskwindow we should yield until there is data +400 SYS "OS_UpCall",6,polladdr% +410 IF !polladdr%<>0 THEN +420 REM Reset the monitor for the time being +430 SYS "ESocket_ResetMonitor",monitor,0 TO polladdr% +440 REPEAT +450 REM Read lines from the connection until this buffer is empty +460 SYS "ESocket_ReadLine",handle,buf%,bufsize%,%100 TO ,str,len +470 IF str<>0 AND $str<>"" THEN +480 line$=$str +490 IF LEFT$(line$,4)="PING" THEN +500 REM Ping's must be replied to immediately +510 SYS "ESocket_SendLine",handle,"PONG "+MID$(line$,6) +520 ELSE +530 REM Extract source info +540 from$=MID$(LEFT$(line$,INSTR(line$+" "," ")-1),2) +550 line$=MID$(line$,INSTR(line$+" "," ")+1) +560 uid$=LEFT$(from$,INSTR(from$+"!","!")-1) +570 com$=LEFT$(line$,INSTR(line$+" "," ")-1) +580 line$=MID$(line$,INSTR(line$+" "," ")+1) +590 REM remove the input line +600 IF input$<>"" THENFORI=1TOLEN(input$):VDU127:NEXT +610 CASE FNupper(com$) OF +620 WHEN "PRIVMSG" +630 REM Extract the destination +640 chan$=LEFT$(line$,INSTR(line$+" "," ")-1) +650 line$=MID$(line$,INSTR(line$+" "," ")+2):REM Skip : +660 IF LEFT$(line$,1)=CHR$1 THEN +670 REM CTCP, so respond to it +680 line$=MID$(line$,2,LEN(line$)-2) +690 com$=LEFT$(line$,INSTR(line$+" "," ")-1) +700 line$=MID$(line$,INSTR(line$+" "," ")+1) +710 CASE FNupper(com$) OF +720 WHEN "PING" +730 REM Ping lag timing +740 line$="PONG "+line$ +750 PRINTuid$;" pinged us" +760 WHEN "VERSION" +770 REM Version checking +780 line$="VERSION EIRC 1.00 (c) Justin Fletcher" +790 PRINTuid$;" wanted our version" +800 WHEN "ACTION" +810 PRINT"* ";uid$;" ";line$ +820 line$="" +830 OTHERWISE +840 REM everything else is an error +850 line$="ERRMSG "+com$+" not understood" +860 PRINT"CTCP '";com$;"' from ";uid$;" (";line$;")" +870 ENDCASE +880 IF line$<>"" THEN +890 SYS "ESocket_SendLine",handle,"NOTICE "+uid$+" :"+CHR$1+line$+CHR$1 +900 ENDIF +910 ELSE +920 REM Somebody said something... +930 PRINT"<";uid$;"> ";FNsafe(line$) +940 ENDIF +950 WHEN "JOIN" +960 REM We (or someone else) has joined the channel +970 chan$=LEFT$(line$,INSTR(line$+" "," ")):REM Skip : +980 IF LEFT$(chan$,1)=":" THENchan$=MID$(chan$,2) +990 PRINTuid$;" has joined ";chan$ +1000 WHEN "PART" +1010 REM Someone else has left the channel +1020 chan$=LEFT$(line$,INSTR(line$+" "," ")-1) +1030 IF LEFT$(chan$,1)=":" THENchan$=MID$(chan$,2) +1040 PRINTuid$;" has left ";chan$ +1050 WHEN "QUIT" +1060 REM Someone else has quit IRC +1070 PRINTuid$;" quit IRC" +1080 OTHERWISE +1090 REM Some unknown command +1100 PRINTuid$;":";com$;":";FNsafe(line$) +1110 ENDCASE +1120 REM Re-display our input line +1130 PRINTinput$; +1140 ENDIF +1150 ENDIF +1160 UNTIL str=0 +1170 ENDIF +1180 b$=INKEY$(0) +1190 IF b$<>"" THEN +1200 CASE b$ OF +1210 WHEN CHR$13 +1220 SYS "ESocket_SendLine",handle,"PRIVMSG "+ourchan$+" :"+input$ +1230 REM Remove the line +1240 IF input$<>"" THENFORI=1TOLEN(input$):VDU127:NEXT +1250 REM We said it... +1260 PRINT"<"+nick$+"> ";input$ +1270 input$="" +1280 WHEN CHR$127,CHR$8 +1290 REM Backspace +1300 IF input$<>"" THENVDU127 +1310 input$=LEFT$(input$) +1320 OTHERWISE +1330 REM Ad to current input +1340 input$+=b$ +1350 PRINTb$; +1360 ENDCASE +1370 ENDIF +1380 REM Has the socket closed +1390 SYS "ESocket_Closed",handle,%0 TO closed +1400UNTIL closed +1410SYS "ESocket_Forget",handle +1420SYS "ESocket_Forget",monitor +1430END +1440: +1450DEFFNupper(a$):LOCAL c$,b$,I +1460FORI=1TOLEN(a$) +1470c$=MID$(a$,I,1):IF c$>="a"ANDc$<="z"THENc$=CHR$(ASC(c$)-32) +1480b$+=c$:NEXT:=b$ +1490 +1500REM Remove control codes +1510DEFFNsafe(line$) +1520LOCAL I +1530FORI=1TOLEN(line$) +1540 IF MID$(line$,I,1)<" " THENMID$(line$,I,1)="*" +1550NEXT +1560=line$ diff --git a/tests/examplefiles/example.boa b/tests/examplefiles/example.boa new file mode 100644 index 00000000..a18f1626 --- /dev/null +++ b/tests/examplefiles/example.boa @@ -0,0 +1,18 @@ +# Computes Number of Public Methods (NPM) for each project, per-type
+# Output is: NPM[ProjectID][TypeName] = NPM value
+p: Project = input;
+NPM: output sum[string][string] of int;
+
+visit(p, visitor {
+ # only look at the latest snapshot
+ before n: CodeRepository -> {
+ snapshot := getsnapshot(n);
+ foreach (i: int; def(snapshot[i]))
+ visit(snapshot[i]);
+ stop;
+ }
+ before node: Declaration ->
+ if (node.kind == TypeKind.CLASS)
+ foreach (i: int; has_modifier_public(node.methods[i]))
+ NPM[p.id][node.name] << 1;
+});
diff --git a/tests/examplefiles/example.flo b/tests/examplefiles/example.flo new file mode 100644 index 00000000..2d4ab5e7 --- /dev/null +++ b/tests/examplefiles/example.flo @@ -0,0 +1,40 @@ +#example mission box1.flo +#from: https://github.com/ioflo/ioflo + +house box1 + + framer vehiclesim be active first vehicle_run + frame vehicle_run + do simulator motion uuv + + framer mission be active first northleg + frame northleg + set elapsed with 20.0 + set heading with 0.0 + set depth with 5.0 + set speed with 2.5 + go next if elapsed >= goal + + frame eastleg + set heading with 90.0 + go next if elapsed >= goal + + frame southleg + set heading with 180.0 + go next if elapsed >= goal + + frame westleg + set heading with 270.0 + go next if elapsed >= goal + + frame mission_stop + bid stop vehiclesim + bid stop autopilot + bid stop me + + framer autopilot be active first autopilot_run + frame autopilot_run + do controller pid speed + do controller pid heading + do controller pid depth + do controller pid pitch
\ No newline at end of file diff --git a/tests/examplefiles/example.hlsl b/tests/examplefiles/example.hlsl new file mode 100644 index 00000000..21d0a672 --- /dev/null +++ b/tests/examplefiles/example.hlsl @@ -0,0 +1,168 @@ +// A few random snippets of HLSL shader code I gathered...
+
+[numthreads(256, 1, 1)]
+void cs_main(uint3 threadId : SV_DispatchThreadID)
+{
+ // Seed the PRNG using the thread ID
+ rng_state = threadId.x;
+
+ // Generate a few numbers...
+ uint r0 = rand_xorshift();
+ uint r1 = rand_xorshift();
+ // Do some stuff with them...
+
+ // Generate a random float in [0, 1)...
+ float f0 = float(rand_xorshift()) * (1.0 / 4294967296.0);
+
+ // ...etc.
+}
+
+// Constant buffer of parameters
+cbuffer IntegratorParams : register(b0)
+{
+ float2 specPow; // Spec powers in XY directions (equal for isotropic BRDFs)
+ float3 L; // Unit vector toward light
+ int2 cThread; // Total threads launched in XY dimensions
+ int2 xyOutput; // Where in the output buffer to store the result
+}
+
+static const float pi = 3.141592654;
+
+float AshikhminShirleyNDF(float3 H)
+{
+ float normFactor = sqrt((specPow.x + 2.0f) * (specPow.y + 2.0)) * (0.5f / pi);
+ float NdotH = H.z;
+ float2 Hxy = normalize(H.xy);
+ return normFactor * pow(NdotH, dot(specPow, Hxy * Hxy));
+}
+
+float BeckmannNDF(float3 H)
+{
+ float glossFactor = specPow.x * 0.5f + 1.0f; // This is 1/m^2 in the usual Beckmann formula
+ float normFactor = glossFactor * (1.0f / pi);
+ float NdotHSq = H.z * H.z;
+ return normFactor / (NdotHSq * NdotHSq) * exp(glossFactor * (1.0f - 1.0f / NdotHSq));
+}
+
+// Output buffer for compute shader (actually float, but must be declared as uint
+// for atomic operations to work)
+globallycoherent RWTexture2D<uint> o_data : register(u0);
+
+// Sum up the outputs of all threads and store to the output location
+static const uint threadGroupSize2D = 16;
+static const uint threadGroupSize1D = threadGroupSize2D * threadGroupSize2D;
+groupshared float g_partialSums[threadGroupSize1D];
+void SumAcrossThreadsAndStore(float value, uint iThreadInGroup)
+{
+ // First reduce within the threadgroup: partial sums of 2, 4, 8... elements
+ // are calculated by 1/2, 1/4, 1/8... of the threads, always keeping the
+ // active threads at the front of the group to minimize divergence.
+
+ // NOTE: there are faster ways of doing this...but this is simple to code
+ // and good enough.
+
+ g_partialSums[iThreadInGroup] = value;
+ GroupMemoryBarrierWithGroupSync();
+
+ [unroll] for (uint i = threadGroupSize1D / 2; i > 0; i /= 2)
+ {
+ if (iThreadInGroup < i)
+ {
+ g_partialSums[iThreadInGroup] += g_partialSums[iThreadInGroup + i];
+ }
+ GroupMemoryBarrierWithGroupSync();
+ }
+
+ // Then reduce across threadgroups: one thread from each group adds the group
+ // total to the final output location, using a software transactional memory
+ // style since D3D11 doesn't support atomic add on floats.
+ // (Assumes the output value has been cleared to zero beforehand.)
+
+ if (iThreadInGroup == 0)
+ {
+ float threadGroupSum = g_partialSums[0];
+ uint outputValueRead = o_data[xyOutput];
+ while (true)
+ {
+ uint newOutputValue = asuint(asfloat(outputValueRead) + threadGroupSum);
+ uint previousOutputValue;
+ InterlockedCompareExchange(
+ o_data[xyOutput], outputValueRead, newOutputValue, previousOutputValue);
+ if (previousOutputValue == outputValueRead)
+ break;
+ outputValueRead = previousOutputValue;
+ }
+ }
+}
+
+void main(
+ in Vertex i_vtx,
+ out Vertex o_vtx,
+ out float3 o_vecCamera : CAMERA,
+ out float4 o_uvzwShadow : UVZW_SHADOW,
+ out float4 o_posClip : SV_Position)
+{
+ o_vtx = i_vtx;
+ o_vecCamera = g_posCamera - i_vtx.m_pos;
+ o_uvzwShadow = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToUvzwShadow);
+ o_posClip = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToClip);
+}
+
+#pragma pack_matrix(row_major)
+
+struct Vertex
+{
+ float3 m_pos : POSITION;
+ float3 m_normal : NORMAL;
+ float2 m_uv : UV;
+};
+
+cbuffer CBFrame : CB_FRAME // matches struct CBFrame in test.cpp
+{
+ float4x4 g_matWorldToClip;
+ float4x4 g_matWorldToUvzwShadow;
+ float3x3 g_matWorldToUvzShadowNormal;
+ float3 g_posCamera;
+
+ float3 g_vecDirectionalLight;
+ float3 g_rgbDirectionalLight;
+
+ float2 g_dimsShadowMap;
+ float g_normalOffsetShadow;
+ float g_shadowSharpening;
+
+ float g_exposure; // Exposure multiplier
+}
+
+Texture2D<float3> g_texDiffuse : register(t0);
+SamplerState g_ss : register(s0);
+
+void main(
+ in Vertex i_vtx,
+ in float3 i_vecCamera : CAMERA,
+ in float4 i_uvzwShadow : UVZW_SHADOW,
+ out float3 o_rgb : SV_Target)
+{
+ float3 normal = normalize(i_vtx.m_normal);
+
+ // Sample shadow map
+ float shadow = EvaluateShadow(i_uvzwShadow, normal);
+
+ // Evaluate diffuse lighting
+ float3 diffuseColor = g_texDiffuse.Sample(g_ss, i_vtx.m_uv);
+ float3 diffuseLight = g_rgbDirectionalLight * (shadow * saturate(dot(normal, g_vecDirectionalLight)));
+ diffuseLight += SimpleAmbient(normal);
+
+ o_rgb = diffuseColor * diffuseLight;
+}
+
+[domain("quad")]
+void ds(
+ in float edgeFactors[4] : SV_TessFactor,
+ in float insideFactors[2] : SV_InsideTessFactor,
+ in OutputPatch<VData, 4> inp,
+ in float2 uv : SV_DomainLocation,
+ out float4 o_pos : SV_Position)
+{
+ o_pos = lerp(lerp(inp[0].pos, inp[1].pos, uv.x), lerp(inp[2].pos, inp[3].pos, uv.x), uv.y);
+}
diff --git a/tests/examplefiles/example.icn b/tests/examplefiles/example.icn new file mode 100644 index 00000000..c8fcf335 --- /dev/null +++ b/tests/examplefiles/example.icn @@ -0,0 +1,283 @@ +# +# $Id: button.icn,v 1.7 2006-07-09 23:43:07 rparlett Exp $ +# +# This file is in the public domain. +# +# Author: Robert Parlett (parlett@dial.pipex.com) +# + +package gui +link graphics + +$include "guih.icn" + + +# +# This is the parent class of the button classes, including +# checkboxes. +# +# A {Button} produces a BUTTON_PRESS_EVENT when the button is +# depressed, and code BUTTON_RELEASE_EVENT when it is released, +# as well as an ACTION_EVENT. +# +# By default, when a button holds the keyboard focus a dashed +# line appears just within the button. Then, when return is +# pressed an ACTION_EVENT is generated. The method +# {Dialog.set_initial_focus()} can be used to have the button +# have the focus when the dialog is first displayed. +# +# Buttons also repeatedly produce a BUTTON_HELD_EVENT whilst they +# are held down, rather like a repeating keyboard press. The +# delay between the initial repeat event and subsequent repeat +# events is set in the parent dialog (see above). +# +class Button : Toggle : Component( + is_down, # + is_held, # + is_checked_flag, # + label, + img_up, # + img_down, # + img_w, # + img_h, # + parent_check_box_group, # + parent_button_group, # + repeat_delay, + no_keyboard_flag, # + toggles_flag + ) + + method set_parent_button_group(x) + return self.parent_button_group := x + end + + # + # Invoking this method disables the keyboard control over the + # button described above. No dashed line will ever appear in + # the button display and return will have no effect on the + # button even if it has the focus. + # + method set_no_keyboard() + self.no_keyboard_flag := 1 + self.accepts_focus_flag := &null + end + + # + # Clear the no keyboard behaviour (the default) + # + method clear_no_keyboard() + self.no_keyboard_flag := &null + self.accepts_focus_flag := 1 + end + + method tick() + if dispatcher.curr_time_of_day() > self.repeat_delay then + fire(BUTTON_HELD_EVENT) + end + + method go_down() + self.is_down := 1 + set_ticker(self.parent_dialog.repeat_rate) + end + + method go_up() + self.is_down := &null + stop_ticker() + end + + method handle_press(e) + local b + if self.in_region() then { + go_down() + self.repeat_delay := dispatcher.curr_time_of_day() + self.parent_dialog.repeat_delay + self.is_held := 1 + every b := !(\self.parent_button_group).buttons do { + if b.is_unhidden() then { + b.is_held := 1 + b.repeat_delay := self.repeat_delay + } + } + self.invalidate() + fire(BUTTON_PRESS_EVENT, e) + } + end + + method handle_drag(e) + if \self.is_held then { + # + # Button held down; toggle on/off as it goes over the button + # + if self.in_region() then { + if /self.is_down then { + go_down() + invalidate() + } + } else { + if \self.is_down then { + go_up() + invalidate() + } + } + } + end + + method handle_release(e) + if \self.is_held then { + self.is_held := &null + if \self.is_down then { + go_up() + fire(BUTTON_RELEASE_EVENT, e) + on_action(e) + } + } + end + + method on_action(e) + if \self.toggles_flag then { + if \self.parent_check_box_group then + self.parent_check_box_group.set_which_one(self) + else + self.toggle_is_checked() + } + self.invalidate() + fire(ACTION_EVENT, e) + end + + method handle_accel(e) + self.Component.handle_accel(e) + on_action(e) + end + + method handle_default(e) + if \self.has_focus then { + if /self.no_keyboard_flag & e == ("\r" | "\l" | " ") then { + on_action(e) + } + } + end + + method handle_event(e) + if e === (&lpress | &rpress | &mpress) then { + handle_press(e) + } else if e === (&ldrag | &rdrag | &mdrag) then { + handle_drag(e) + } else if e === (&lrelease | &rrelease | &mrelease) then { + handle_release(e) + } else + handle_default(e) + end + + # + # Set the up/down images (if any) to the strings provided, + # which should be in Icon image format. + # The two images must have the same dimensions. + # @param x The up image + # @param y The down image + # + method set_imgs(x, y) + self.img_up := x + self.img_w := img_width(x) = img_width(y) | fatal("Image widths differ") + self.img_h := img_height(x) = img_height(y) | fatal("Image heights differ") + + self.img_down := y + + return + end + + # + # Set the image (if any) to the given string, which should be in Icon image + # format. + # @param x The image + # + method set_img(x) + self.img_up := self.img_down := x + self.img_w := img_width(x) + self.img_h := img_height(x) + return x + end + + # + # Toggle the checked status of the button. This method, and + # the following two methods, may be + # inappropriate for non-toggle styles of button. + # + method toggle_is_checked() + self.Toggle.toggle_is_checked() + self.invalidate() + end + + # + # Set the status to checked. + # + method set_is_checked() + self.Toggle.set_is_checked() + self.invalidate() + end + + # + # Set the status to unchecked. + # + method clear_is_checked() + self.Toggle.clear_is_checked() + self.invalidate() + end + + # + # Set the button so that when it is pressed, it toggles + # between two states, as indicated by the is_checked + # flag. + # + # Instances of Checkbox have this flag on by default, but + # TextButton and IconButton do not. When the flag is on, + # the latter classes indicate their checked status by + # showing the button as being "down". + # + method set_toggles() + self.toggles_flag := 1 + self.invalidate() + end + + # + # Clear the toggles flag. + # + method clear_toggles() + self.toggles_flag := &null + self.invalidate() + end + + # + # Set the label of the button, if any. + # @param x The label + # + method set_label(x) + self.label := x + self.invalidate() + return x + end + + method set_one(attr, val) + case attr of { + "label" : set_label(string_val(attr, val)) + "is_checked" : + if test_flag(attr, val) then + set_is_checked() + else + clear_is_checked() + "toggles" : + if test_flag(attr, val) then + set_toggles() + else + clear_toggles() + "no_keyboard" : + if test_flag(attr, val) then + set_no_keyboard() + else + clear_no_keyboard() + default: self.Component.set_one(attr, val) + } + end + + initially() + self.Component.initially() + self.accepts_focus_flag := 1 +end
\ No newline at end of file diff --git a/tests/examplefiles/example.icon b/tests/examplefiles/example.icon new file mode 100644 index 00000000..29bc548b --- /dev/null +++ b/tests/examplefiles/example.icon @@ -0,0 +1,381 @@ +############################################################################ +# +# File: kaleid.icn +# +# Subject: Program to produce kaleidoscope +# +# Author: Stephen B. Wampler +# +# Date: May 2, 2001 +# +############################################################################ +# +# This file is in the public domain. +# +############################################################################ +# +# Lots of options, most easily set by with the interface after +# startup. The only one that isn't set that way is -wn where 'n' is +# the size of the kaleidoscope window (default is 600 square). +# +# Terminology (and options): +# +# Window_size (-wN): How big of a display window to use. +# At the current time, this can only be set via a +# command line argument. +# +# Density (-dN): How many circles per octant to keep on display +# at any one time. There is NO LIMIT to the density. +# +# Duration (-lN): How long to keep drawing circles (measured in +# in circles) once the density is reached. There is NO LIMIT +# to the duration. +# +# MaxRadius (-MN): Maximum radius of any circle. +# +# MinRadius (-mN): Preferred minimum radius. Circles with centers +# near the edge have their radii forced down to fit entirely +# on the display +# +# MaxOffset (-XN): Maximum offset from center of display (may wrap). +# +# MinOffset (-xN): Minimum offset +# +# Skew (-sN): Shift probability of placing a circle at a 'typical' +# offset. +# +# Fill (-F): Turns off filling the circles. +# +# Clear (-C): After the duration, reduces density back to 0 before +# quitting. +# +# Random Seed: (-rN): Sets the random number seed. +# +# Thanks to Jon Lipp for help on using vidgets, and to Mary Camaron +# for her Interface Builder. +# +############################################################################ +# +# Requires: Version 9 graphics +# +############################################################################ +# +# Links: vidgets, vslider, vtext, vbuttons, vradio, wopen, xcompat +# +############################################################################ + +link vidgets +link vslider +link vtext +link vbuttons +link vradio +link wopen +link xcompat + +global Clear, fill, duration, density, maxoff, minoff +global maxradius, minradius, r_seed, skew, win_size, mid_win +global root, check1, mainwin, use_dialog +global draw_circle + +global du_v, de_v, rs_v, sk_v + +procedure main (args) + + draw_circle := DrawCircle + + init_globs() + process_args(args) + + if \use_dialog then { # have vidgets, so use them for args. + mainwin := WOpen("label=Kaleidoscope", "width=404", "height=313", + "font=6x12") | + stop ("bad mainwin") + root := ui (mainwin) + GetEvents (root, quit) + } + else { # just rely on command line arguments + kaleidoscope(r_seed) + } + +end + +procedure init_globs() + + duration := 500 # set default characteristics + density := 30 + win_size := 600 + minoff := 1 + maxradius := 150 + minradius := 1 + skew := 1 + fill := "On" + draw_circle := FillCircle + Clear := "Off" + r_seed := map("HhMmYy", "Hh:Mm:Yy", &clock) + # See if the Vidget library is available or not + if \VSet then use_dialog := "yes" + else use_dialog := &null + +end + +procedure process_args(args) + local arg + + # really only needed if you don't use the dialog box + every arg := !args do case arg[1+:2] of { + "-w" : win_size := integer(arg[3:0]) # window size + "-d" : density := integer(arg[3:0]) # density of circles + "-l" : duration := integer(arg[3:0]) # duration + "-M" : maxradius := integer(arg[3:0]) # maximum radius + "-m" : minradius := integer(arg[3:0]) # minimum radius + "-X" : maxoff := integer(arg[3:0]) # maximum offset + "-x" : minoff := integer(arg[3:0]) # minimum offset + "-s" : skew := numeric(arg[3:0]) # set skewedness + "-F" : fill := &null # turn off fill + "-C" : Clear := "yes" # turn on clear mode + "-r" : r_seed := integer(arg[3:0]) # random seed + "-h" : stop("usage: kal [-wn] [-dn] [-ln] [-Mn] [-mn] [-Xn] [-xn] _ + [-sn] [-F] [-C] [-rn]") + } + # adjust parameters that depend on the window size... + mid_win := win_size/2 + maxoff := win_size-1 +end + +# Lorraine Callahan's kaleidoscope program, translated into icon. +# (some of the things she did were too sophisticated for me +# to spend time to figure out, so the output is square instead of +# round), and I use 'xor' to draw instead of writing to separate +# bit planes. + +global putcircle, clrcircle + +procedure kaleidoscope(r) + local colors + + # What colors to use? This can be changed to whatever! + colors := ["red","green","blue","cyan","magenta","yellow"] + + &window := WOpen("label=Kaleidoscope: 'q' quits", "width="||win_size, + "height="||win_size, "bg=black") + WAttrib("drawop=xor") + + # Create two *indentical* sequences of circles, one to use when + # when drawing, one for erasing. (Since 'xor' is used to + # place them, these both just draw the circles!) + + putcircle := create { # draws sequence of circles + &random :=: r + |{ + Fg(?colors) + outcircle() + &random <-> r + } + } + + clrcircle := create { # erases sequence of circles + &random :=: r + |{ + Fg(?colors) + outcircle() + &random <-> r + } + } + + every 1 to density do @putcircle # fill screen to density + + every 1 to duration do { # maintain steady state + @putcircle + @clrcircle + if *Pending(&window) > 0 then break + } + + every (Clear == "On") & 1 to density do @clrcircle + + close(&window) +end + + +procedure outcircle() # select a circle at random, +local radius, xoff, yoff # draw it in kaleidoscopic form + + # get a random center point and radius + xoff := (?(maxoff - minoff) + minoff) % mid_win + yoff := (?(maxoff - minoff) + minoff) % mid_win + radius := ?0 ^ skew + # force radius to 'fit' + radius := ((maxradius-minradius) * radius + minradius) % + (mid_win - ((xoff < yoff)|xoff)) + + # put into all 8 octants + draw_circle(mid_win+xoff, mid_win+yoff, radius) + draw_circle(mid_win+xoff, mid_win-yoff, radius) + draw_circle(mid_win-xoff, mid_win+yoff, radius) + draw_circle(mid_win-xoff, mid_win-yoff, radius) + + draw_circle(mid_win+yoff, mid_win+xoff, radius) + draw_circle(mid_win+yoff, mid_win-xoff, radius) + draw_circle(mid_win-yoff, mid_win+xoff, radius) + draw_circle(mid_win-yoff, mid_win-xoff, radius) + + return +end + + +############################################################################ +# +# Vidget-based user interface -- developed originally using Mary +# Camaron's XIB program. Don't expect this to be very readable - +# you should have to play with it! +# +############################################################################ +procedure ui (win) + local cv1, cv2, cv3, cv4 + local + radio_button2, + radio_button1, + text_input6, + text_input5, + slider4, + slider3, + text_input4, + text_input3, + slider2, + slider1 + + /win := WOpen("label=ui", "width=404", "height=313", "font=6x12") | + stop ("bad win") + root := Vroot_frame (win) + + VInsert (root, Vmessage(win, win_size/2), 168, 98) + VInsert (root, Vmessage(win, "1"), 108, 97) + + VInsert (root, sk_v := Vtext(win,"Skew:\\=1",get_skew,,6), 280, 39) + + VInsert (root, du_v := Vtext(win, "Duration:\\="||duration, get_duration,,9), + 237, 15) + + VInsert (root, Vmessage(win, "Clear at end?"), 232, 145) + VInsert (root, Vmessage(win, "Fill?"), 105, 142) + VInsert (root, Vmessage(win,"Quit?"), 267, 259) + VInsert (root, Vmessage(win,"Display it?"), 26, 260) + + VInsert (root, Vcheckbox(win, do_quit, "check2",20), 305, 255, 20, 20) + + VInsert (root, check1:=Vcheckbox(win, do_display, "check1",20), + 106, 258, 20, 20) + + radio_button2 := Vradio_buttons (win, ["On", "Off"], get_clear, , V_CIRCLE) + VSet(radio_button2,Clear) + VInsert (root, radio_button2, 253, 165) + + radio_button1 := Vradio_buttons (win, ["On", "Off"], get_fill, , V_CIRCLE) + VSet(radio_button1,fill) + VInsert (root, radio_button1, 99, 165) + + cv1 := Vcoupler() + VAddClient(cv1, get_max_offset) + text_input6 := Vtext (win, "Max Offset:\\="||(win_size-1), cv1, , 3) + VAddClient(cv1, text_input6) + slider4 := Vhoriz_slider (win, cv1, "slider4", 70, 12, 0, + win_size-1, win_size-1, ) + VAddClient(cv1, slider4) + VInsert (root, text_input6, 196, 103) + VInsert (root, slider4, 306, 106) + + cv2 := Vcoupler() + VAddClient(cv2, get_min_offset) + text_input5 := Vtext (win, "Min Offset\\=1", cv2, , 3) + VAddClient(cv2, text_input5) + slider3 := Vhoriz_slider (win, cv2, "slider3", 70, 12, 1, win_size-1, 1, ) + VAddClient(cv2, slider3) + VInsert (root, text_input5, 201, 80) + VInsert (root, slider3, 307, 82) + + cv3 := Vcoupler() + VAddClient(cv3, get_max_radius) + text_input4 := Vtext (win, "Max Radius\\="||(win_size/4), cv3, , 3) + VAddClient(cv3, text_input4) + slider2 := Vhoriz_slider (win, cv3, "slider2", 70, 12, 1, win_size/2, + win_size/4, ) + VAddClient(cv3, slider2) + VInsert (root, text_input4, 10, 104) + VInsert (root, slider2, 110, 108) + + cv4 := Vcoupler() + VAddClient(cv4, get_min_radius) + text_input3 := Vtext (win, "Min Radius\\=1", cv4, , 3) + VAddClient(cv4, text_input3) + slider1 := Vhoriz_slider (win, cv4, "slider1", 70, 12, 1, win_size/2, 1, ) + VAddClient(cv4, slider1) + VInsert (root, text_input3, 10, 81) + VInsert (root, slider1, 110, 84) + + VInsert (root, rs_v := Vtext(win,"Random Seed:\\="||r_seed, get_random,, 11), + 30, 41) + VInsert (root, de_v := Vtext(win,"Density:\\="||density, get_density,,8), + 71, 16) + + VResize (root) + return root +end + +procedure get_skew (wit, value) + skew := value +end + +procedure get_duration (wit, value) + duration := value +end + +procedure do_quit (wit, value) + stop() +end + +procedure do_display (wit, value) + r_seed := numeric(rs_v.data) + duration := integer(du_v.data) + density := integer(de_v.data) + skew := integer(sk_v.data) + kaleidoscope(r_seed) + wit.callback.value := &null + VDraw(check1) +end + +procedure get_clear (wit, value) + Clear := value +end + +procedure get_fill (wit, value) + fill := value + if fill == "Off" then draw_circle := DrawCircle + else draw_circle := FillCircle +end + +procedure get_max_offset (wit, value) + maxoff := value +end + +procedure get_min_offset (wit, value) + minoff := value +end + +procedure get_max_radius (wit, value) + maxradius := value +end + +procedure get_min_radius (wit, value) + minradius := value +end + +procedure get_random (wit, value) + r_seed := integer(value) +end + +procedure get_density (wit, value) + density := integer(value) +end + +procedure quit(e) + if e === "q" then stop ("Exiting Kaleidoscope") +end diff --git a/tests/examplefiles/example.md b/tests/examplefiles/example.md index 2befb107..e2bbacf1 100644 --- a/tests/examplefiles/example.md +++ b/tests/examplefiles/example.md @@ -46,6 +46,9 @@ this sentence @tweets a person about a #topic. [google](https://google.com/some/path.html)  +[reference link][id] +[id]: http://example.com/ + ``` * this is just unformated __text__ diff --git a/tests/examplefiles/example.pony b/tests/examplefiles/example.pony new file mode 100644 index 00000000..654f2376 --- /dev/null +++ b/tests/examplefiles/example.pony @@ -0,0 +1,18 @@ +use "somepkg" + +/* + /* Nested */ +*/ + +class trn Foo[A: Stringable ref] is Stringable + let _x = "\"" + + fun val dofoo() => + """ + DocString + """ + (U64(2), "foo")._2 + +actor Main + new create(env: Env) => + env.out.print("Hello world") diff --git a/tests/examplefiles/example.sgf b/tests/examplefiles/example.sgf new file mode 100644 index 00000000..024a461e --- /dev/null +++ b/tests/examplefiles/example.sgf @@ -0,0 +1,35 @@ +(;FF[4]GM[1]SZ[19]FG[257:Figure 1]PM[1] +PB[Takemiya Masaki]BR[9 dan]PW[Cho Chikun] +WR[9 dan]RE[W+Resign]KM[5.5]TM[28800]DT[1996-10-18,19] +EV[21st Meijin]RO[2 (final)]SO[Go World #78]US[Arno Hollosi] +;B[pd];W[dp];B[pp];W[dd];B[pj];W[nc];B[oe];W[qc];B[pc];W[qd] +(;B[qf];W[rf];B[rg];W[re];B[qg];W[pb];B[ob];W[qb] +(;B[mp];W[fq];B[ci];W[cg];B[dl];W[cn];B[qo];W[ec];B[jp];W[jd] +;B[ei];W[eg];B[kk]LB[qq:a][dj:b][ck:c][qp:d]N[Figure 1] + +;W[me]FG[257:Figure 2];B[kf];W[ke];B[lf];W[jf];B[jg] +(;W[mf];B[if];W[je];B[ig];W[mg];B[mj];W[mq];B[lq];W[nq] +(;B[lr];W[qq];B[pq];W[pr];B[rq];W[rr];B[rp];W[oq];B[mr];W[oo];B[mn] +(;W[nr];B[qp]LB[kd:a][kh:b]N[Figure 2] + +;W[pk]FG[257:Figure 3];B[pm];W[oj];B[ok];W[qr];B[os];W[ol];B[nk];W[qj] +;B[pi];W[pl];B[qm];W[ns];B[sr];W[om];B[op];W[qi];B[oi] +(;W[rl];B[qh];W[rm];B[rn];W[ri];B[ql];W[qk];B[sm];W[sk];B[sh];W[og] +;B[oh];W[np];B[no];W[mm];B[nn];W[lp];B[kp];W[lo];B[ln];W[ko];B[mo] +;W[jo];B[km]N[Figure 3]) + +(;W[ql]VW[ja:ss]FG[257:Dia. 6]MN[1];B[rm];W[ph];B[oh];W[pg];B[og];W[pf] +;B[qh];W[qe];B[sh];W[of];B[sj]TR[oe][pd][pc][ob]LB[pe:a][sg:b][si:c] +N[Diagram 6])) + +(;W[no]VW[jj:ss]FG[257:Dia. 5]MN[1];B[pn]N[Diagram 5])) + +(;B[pr]FG[257:Dia. 4]MN[1];W[kq];B[lp];W[lr];B[jq];W[jr];B[kp];W[kr];B[ir] +;W[hr]LB[is:a][js:b][or:c]N[Diagram 4])) + +(;W[if]FG[257:Dia. 3]MN[1];B[mf];W[ig];B[jh]LB[ki:a]N[Diagram 3])) + +(;W[oc]VW[aa:sk]FG[257:Dia. 2]MN[1];B[md];W[mc];B[ld]N[Diagram 2])) + +(;B[qe]VW[aa:sj]FG[257:Dia. 1]MN[1];W[re];B[qf];W[rf];B[qg];W[pb];B[ob] +;W[qb]LB[rg:a]N[Diagram 1])) diff --git a/tests/examplefiles/example.sl b/tests/examplefiles/example.sl new file mode 100644 index 00000000..5fb430de --- /dev/null +++ b/tests/examplefiles/example.sl @@ -0,0 +1,6 @@ +#!/bin/bash +#SBATCH --partition=part +#SBATCH --job-name=job +#SBATCH --mem=1G +#SBATCH --cpus-per-task=8 +srun /usr/bin/sleep
\ No newline at end of file diff --git a/tests/examplefiles/example.stan b/tests/examplefiles/example.stan index 69c9ac70..03b7b1b5 100644 --- a/tests/examplefiles/example.stan +++ b/tests/examplefiles/example.stan @@ -16,7 +16,7 @@ functions { data { // valid name int abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_abc; - // all types should be highlighed + // all types should be highlighted int a3; real foo[2]; vector[3] bar; @@ -48,7 +48,7 @@ transformed data { thud <- -12309865; // ./ and .* should be recognized as operators grault2 <- grault .* garply ./ garply; - // ' and \ should be regognized as operators + // ' and \ should be recognized as operators qux2 <- qux' \ bar; } diff --git a/tests/examplefiles/example.tf b/tests/examplefiles/example.tf index 5a85dbee..4cbef52c 100644 --- a/tests/examplefiles/example.tf +++ b/tests/examplefiles/example.tf @@ -23,6 +23,14 @@ variable "aws_amis" { } +resource "aws_internet_gateway" "base_igw" { + vpc_id = "${aws_vpc.something.id}" + tags { + Name = "igw-${var.something}-${var.something}" + } +} + + @@ -170,3 +178,31 @@ resource "aws_instance" "web" { } } + + +resource "aws_autoscaling_group" "bar" { + name = "terraform-asg-example" + launch_configuration = "${aws_launch_configuration.as_conf.name}" + min_size = 1 + max_size = 2 + + lifecycle { + create_before_destroy = true + } +} + + +resource "aws_db_instance" "timeout_example" { + allocated_storage = 10 + engine = "mysql" + engine_version = "5.6.17" + instance_class = "db.t1.micro" + name = "mydb" + + timeouts { + create = "60m" + delete = "2h" + } +} + + diff --git a/tests/examplefiles/example.toml b/tests/examplefiles/example.toml new file mode 100644 index 00000000..9c60c79f --- /dev/null +++ b/tests/examplefiles/example.toml @@ -0,0 +1,181 @@ +# This is a TOML document comment + +title = "TOML example file" # This is an inline comment + +[examples] +# Examples taken from https://github.com/toml-lang/toml/blob/master/README.md +key = "value" +bare_key = "value" +bare-key = "value" +1234 = "value" +"127.0.0.1" = "value" +"character encoding" = "value" +"ʎǝʞ" = "value" +'key2' = "value" +'quoted "value"' = "value" +name = "Orange" +physical.color = "orange" +physical.shape = "round" +site."google.com" = true +a.b.c = 1 +a.d = 2 + +[strings] +str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." +str1 = """ +Roses are red +Violets are blue""" +str2 = "Roses are red\nViolets are blue" +str3 = "Roses are red\r\nViolets are blue" + + [strings.equivalents] + str1 = "The quick brown fox jumps over the lazy dog." + str2 = """ +The quick brown \ + + + fox jumps over \ + the lazy dog.""" + str3 = """\ + The quick brown \ + fox jumps over \ + the lazy dog.\ + """ + + [strings.literal] + winpath = 'C:\Users\nodejs\templates' + winpath2 = '\\ServerX\admin$\system32\' + quoted = 'Tom "Dubs" Preston-Werner' + regex = '<\i\c*\s*>' + + [strings.multiline] + regex2 = '''I [dw]on't need \d{2} apples''' + lines = ''' +The first newline is +trimmed in raw strings. + All other whitespace + is preserved. +''' + +[integers] +int1 = +99 +int2 = 42 +int3 = 0 +int4 = -17 +int5 = 1_000 +int6 = 5_349_221 +int7 = 1_2_3_4_5 # discouraged format +# hexadecimal with prefix `0x` +hex1 = 0xDEADBEEF +hex2 = 0xdeadbeef +hex3 = 0xdead_beef +# octal with prefix `0o` +oct1 = 0o01234567 +oct2 = 0o755 # useful for Unix file permissions +# binary with prefix `0b` +bin1 = 0b11010110 + +[floats] +# fractional +flt1 = +1.0 +flt2 = 3.1415 +flt3 = -0.01 +# exponent +flt4 = 5e+22 +flt5 = 1e6 +flt6 = -2E-2 +# both +flt7 = 6.626e-34 +# with underscores, for readability +flt8 = 224_617.445_991_228 +# infinity +sf1 = inf # positive infinity +sf2 = +inf # positive infinity +sf3 = -inf # negative infinity +# not a number +sf4 = nan # actual sNaN/qNaN encoding is implementation specific +sf5 = +nan # same as `nan` +sf6 = -nan # valid, actual encoding is implementation specific +# plus/minus zero +sf0_1 = +0.0 +sf0_2 = -0.0 + +[booleans] +bool1 = true +bool2 = false + +[datetime.offset] +odt1 = 1979-05-27T07:32:00Z +odt2 = 1979-05-27T00:32:00-07:00 +odt3 = 1979-05-27T00:32:00.999999-07:00 +odt4 = 1979-05-27 07:32:00Z + +[datetime.local] +ldt1 = 1979-05-27T07:32:00 +ldt2 = 1979-05-27T00:32:00.999999 + +[date.local] +ld1 = 1979-05-27 + +[time.local] +lt1 = 07:32:00 +lt2 = 00:32:00.999999 + +[arrays] +arr1 = [ 1, 2, 3 ] +arr2 = [ "red", "yellow", "green" ] +arr3 = [ [ 1, 2 ], [3, 4, 5] ] +arr4 = [ "all", 'strings', """are the same""", '''type'''] +arr5 = [ [ 1, 2 ], ["a", "b", "c"] ] +arr6 = [ 1, 2.0 ] # INVALID +arr7 = [ + 1, 2, 3 +] +arr8 = [ + 1, + 2, # this is ok +] + +["inline tables"] +name = { first = "Tom", last = "Preston-Werner" } +point = { x = 1, y = 2 } +animal = { type.name = "pug" } + +["arrays of tables"] +points = [ { x = 1, y = 2, z = 3 }, + { x = 7, y = 8, z = 9 }, + { x = 2, y = 4, z = 8 } ] + + [products] + + [[products]] + name = "Hammer" + sku = 738594937 + + [[products]] + + [[products]] + name = "Nail" + sku = 284758393 + color = "gray" + + [fruits] + + [[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + + [[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" diff --git a/tests/examplefiles/example.u b/tests/examplefiles/example.u index 42c85902..8c6686eb 100644 --- a/tests/examplefiles/example.u +++ b/tests/examplefiles/example.u @@ -545,4 +545,3 @@ var y = 0; - diff --git a/tests/examplefiles/example.u1 b/tests/examplefiles/example.u1 new file mode 100644 index 00000000..92c45365 --- /dev/null +++ b/tests/examplefiles/example.u1 @@ -0,0 +1,111 @@ +version U12.1.00 +uid version.u1-1494453463-0 +impl local +global 1 + 0,000005,version,0 + + +proc version + local 0,000000,tab + local 1,000000,find + local 2,000000,many + con 0,010000,8,126,145,162,163,151,157,156,040 + con 1,002000,1,8 + con 2,020000,11,060,061,062,063,064,065,066,067,070,071,056 + con 3,002000,1,1 + declend + filen version.icn + line 23 + colm 11 + synt any + mark L1 + line 25 + colm 4 + synt any + keywd version + line 25 + colm 13 + synt any + bscan + mark L2 + mark L3 + var 0 + pnull + var 1 + str 0 + line 26 + colm 15 + synt any + invoke 1 + int 1 + line 26 + colm 28 + synt any + plus + line 26 + colm 10 + synt any + invoke 1 + line 26 + colm 33 + synt any + esusp + goto L4 +lab L3 + line 26 + colm 35 + synt any + pfail +lab L4 + unmark +lab L2 + var 0 + var 2 + cset 2 + line 27 + colm 15 + synt any + invoke 1 + line 27 + colm 10 + synt any + invoke 1 + line 27 + colm 32 + synt any + bscan + mark L5 + var 0 + pnull + int 3 + line 27 + colm 45 + synt any + neg + line 27 + colm 44 + synt any + invoke 1 + line 27 + colm 34 + synt any + pret +lab L5 + synt any + pfail + line 27 + colm 32 + synt any + escan + line 25 + colm 13 + synt any + escan + unmark +lab L1 + pnull + line 30 + colm 1 + synt any + pfail + end
\ No newline at end of file diff --git a/tests/examplefiles/example.vbs b/tests/examplefiles/example.vbs new file mode 100644 index 00000000..d962b73d --- /dev/null +++ b/tests/examplefiles/example.vbs @@ -0,0 +1,55 @@ +rem VBScript examples + +' Various constants of different types +const someText = "some " & """text""" +const someInt = 123 +const someHex = &h3110c0d3 +const someFloat = 123.45e-67 +const someDate = #1/2/2016# +const someTime = #12:34:56 AM# +const someBool = vbTrue ' -1 + +' Do some math. +radius = 1.e2 +area = radius ^ 2 * 3.1315 +a = 17 : b = 23 +c = sqr(a ^2 + b ^ 2) + +' Write 10 files. +For i = 1 to 10 + createFile( i ) +Next + +Public Sub createFile(a) + Dim fso, TargetFile + TargetPath = "C:\some_" & a & ".tmp" + Set fso = CreateObject("Scripting.FileSystemObject") + Set TargetFile = fso.CreateTextFile(TargetPath) + TargetFile.WriteLine("Hello " & vbCrLf & "world!") + TargetFile.Close +End Sub + +' Define a class with a property. +Class Customer + Private m_CustomerName + + Private Sub Class_Initialize + m_CustomerName = "" + End Sub + + ' CustomerName property. + Public Property Get CustomerName + CustomerName = m_CustomerName + End Property + + Public Property Let CustomerName(custname) + m_CustomerName = custname + End Property +End Class + +' Special constructs +Option Explicit +On Error Resume Next +On Error Goto 0 + +' Comment without terminating CR/LF.
\ No newline at end of file diff --git a/tests/examplefiles/example.zig b/tests/examplefiles/example.zig new file mode 100644 index 00000000..32e72849 --- /dev/null +++ b/tests/examplefiles/example.zig @@ -0,0 +1,263 @@ +const std = @import("std"); +const Allocator = mem.Allocator; +const mem = std.mem; +const ast = std.zig.ast; +const Visib = @import("visib.zig").Visib; +const event = std.event; +const Value = @import("value.zig").Value; +const Token = std.zig.Token; +const errmsg = @import("errmsg.zig"); +const Scope = @import("scope.zig").Scope; +const Compilation = @import("compilation.zig").Compilation; + +pub const Decl = struct { + id: Id, + name: []const u8, + visib: Visib, + resolution: event.Future(Compilation.BuildError!void), + parent_scope: *Scope, + + // TODO when we destroy the decl, deref the tree scope + tree_scope: *Scope.AstTree, + + pub const Table = std.HashMap([]const u8, *Decl, mem.hash_slice_u8, mem.eql_slice_u8); + + pub fn cast(base: *Decl, comptime T: type) ?*T { + if (base.id != @field(Id, @typeName(T))) return null; + return @fieldParentPtr(T, "base", base); + } + + pub fn isExported(base: *const Decl, tree: *ast.Tree) bool { + switch (base.id) { + Id.Fn => { + const fn_decl = @fieldParentPtr(Fn, "base", base); + return fn_decl.isExported(tree); + }, + else => return false, + } + } + + pub fn getSpan(base: *const Decl) errmsg.Span { + switch (base.id) { + Id.Fn => { + const fn_decl = @fieldParentPtr(Fn, "base", base); + const fn_proto = fn_decl.fn_proto; + const start = fn_proto.fn_token; + const end = fn_proto.name_token orelse start; + return errmsg.Span{ + .first = start, + .last = end + 1, + }; + }, + else => @panic("TODO"), + } + } + + pub fn findRootScope(base: *const Decl) *Scope.Root { + return base.parent_scope.findRoot(); + } + + pub const Id = enum { + Var, + Fn, + CompTime, + }; + + pub const Var = struct { + base: Decl, + }; + + pub const Fn = struct { + base: Decl, + value: Val, + fn_proto: *ast.Node.FnProto, + + // TODO https://github.com/ziglang/zig/issues/683 and then make this anonymous + pub const Val = union(enum) { + Unresolved: void, + Fn: *Value.Fn, + FnProto: *Value.FnProto, + }; + + pub fn externLibName(self: Fn, tree: *ast.Tree) ?[]const u8 { + return if (self.fn_proto.extern_export_inline_token) |tok_index| x: { + const token = tree.tokens.at(tok_index); + break :x switch (token.id) { + Token.Id.Extern => tree.tokenSlicePtr(token), + else => null, + }; + } else null; + } + + pub fn isExported(self: Fn, tree: *ast.Tree) bool { + if (self.fn_proto.extern_export_inline_token) |tok_index| { + const token = tree.tokens.at(tok_index); + return token.id == Token.Id.Keyword_export; + } else { + return false; + } + } + }; + + pub const CompTime = struct { + base: Decl, + }; +}; + +pub const info_zen = + \\ + \\ * Communicate intent precisely. + \\ * Edge cases matter. + \\ * Favor reading code over writing code. + \\ * Only one obvious way to do things. + \\ * Runtime crashes are better than bugs. + \\ * Compile errors are better than runtime crashes. + \\ * Incremental improvements. + \\ * Avoid local maximums. + \\ * Reduce the amount one must remember. + \\ * Minimize energy spent on coding style. + \\ * Together we serve end users. + \\ + \\ +; + +fn cmdZen(allocator: *Allocator, args: []const []const u8) !void { + try stdout.write(info_zen); +} + +const usage_internal = + \\usage: zig internal [subcommand] + \\ + \\Sub-Commands: + \\ build-info Print static compiler build-info + \\ + \\ +; + +fn cmdInternal(allocator: *Allocator, args: []const []const u8) !void { + if (args.len == 0) { + try stderr.write(usage_internal); + os.exit(1); + } + + const sub_commands = []Command{Command{ + .name = "build-info", + .exec = cmdInternalBuildInfo, + }}; + + for (sub_commands) |sub_command| { + if (mem.eql(u8, sub_command.name, args[0])) { + try sub_command.exec(allocator, args[1..]); + return; + } + } + + try stderr.print("unknown sub command: {}\n\n", args[0]); + try stderr.write(usage_internal); +} + +fn cmdInternalBuildInfo(allocator: *Allocator, args: []const []const u8) !void { + try stdout.print( + \\ZIG_CMAKE_BINARY_DIR {} + \\ZIG_CXX_COMPILER {} + \\ZIG_LLVM_CONFIG_EXE {} + \\ZIG_LLD_INCLUDE_PATH {} + \\ZIG_LLD_LIBRARIES {} + \\ZIG_STD_FILES {} + \\ZIG_C_HEADER_FILES {} + \\ZIG_DIA_GUIDS_LIB {} + \\ + , + std.cstr.toSliceConst(c.ZIG_CMAKE_BINARY_DIR), + std.cstr.toSliceConst(c.ZIG_CXX_COMPILER), + std.cstr.toSliceConst(c.ZIG_LLVM_CONFIG_EXE), + std.cstr.toSliceConst(c.ZIG_LLD_INCLUDE_PATH), + std.cstr.toSliceConst(c.ZIG_LLD_LIBRARIES), + std.cstr.toSliceConst(c.ZIG_STD_FILES), + std.cstr.toSliceConst(c.ZIG_C_HEADER_FILES), + std.cstr.toSliceConst(c.ZIG_DIA_GUIDS_LIB), + ); +} + +fn test__floatuntisf(a: u128, expected: f32) void { + const x = __floatuntisf(a); + testing.expect(x == expected); +} + +test "floatuntisf" { + test__floatuntisf(0, 0.0); + + test__floatuntisf(1, 1.0); + test__floatuntisf(2, 2.0); + test__floatuntisf(20, 20.0); + + test__floatuntisf(0x7FFFFF8000000000, 0x1.FFFFFEp+62); + test__floatuntisf(0x7FFFFF0000000000, 0x1.FFFFFCp+62); + + test__floatuntisf(make_ti(0x8000008000000000, 0), 0x1.000001p+127); + test__floatuntisf(make_ti(0x8000000000000800, 0), 0x1.0p+127); + test__floatuntisf(make_ti(0x8000010000000000, 0), 0x1.000002p+127); + + test__floatuntisf(make_ti(0x8000000000000000, 0), 0x1.000000p+127); + + test__floatuntisf(0x0007FB72E8000000, 0x1.FEDCBAp+50); + + test__floatuntisf(0x0007FB72EA000000, 0x1.FEDCBA8p+50); + test__floatuntisf(0x0007FB72EB000000, 0x1.FEDCBACp+50); + + test__floatuntisf(0x0007FB72EC000000, 0x1.FEDCBBp+50); + + test__floatuntisf(0x0007FB72E6000000, 0x1.FEDCB98p+50); + test__floatuntisf(0x0007FB72E7000000, 0x1.FEDCB9Cp+50); + test__floatuntisf(0x0007FB72E4000000, 0x1.FEDCB9p+50); + + test__floatuntisf(0xFFFFFFFFFFFFFFFE, 0x1p+64); + test__floatuntisf(0xFFFFFFFFFFFFFFFF, 0x1p+64); + + test__floatuntisf(0x0007FB72E8000000, 0x1.FEDCBAp+50); + + test__floatuntisf(0x0007FB72EA000000, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72EB000000, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72EBFFFFFF, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72EC000000, 0x1.FEDCBCp+50); + test__floatuntisf(0x0007FB72E8000001, 0x1.FEDCBAp+50); + + test__floatuntisf(0x0007FB72E6000000, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72E7000000, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72E7FFFFFF, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72E4000001, 0x1.FEDCBAp+50); + test__floatuntisf(0x0007FB72E4000000, 0x1.FEDCB8p+50); + + test__floatuntisf(make_ti(0x0000000000001FED, 0xCB90000000000001), 0x1.FEDCBAp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBA0000000000000), 0x1.FEDCBAp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBAFFFFFFFFFFFFF), 0x1.FEDCBAp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBB0000000000000), 0x1.FEDCBCp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBB0000000000001), 0x1.FEDCBCp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBBFFFFFFFFFFFFF), 0x1.FEDCBCp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBC0000000000000), 0x1.FEDCBCp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBC0000000000001), 0x1.FEDCBCp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBD0000000000000), 0x1.FEDCBCp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBD0000000000001), 0x1.FEDCBEp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBDFFFFFFFFFFFFF), 0x1.FEDCBEp+76); + test__floatuntisf(make_ti(0x0000000000001FED, 0xCBE0000000000000), 0x1.FEDCBEp+76); +} + +fn trimStart(slice: []const u8, ch: u8) []const u8 { + var i: usize = 0; + const test_string = "test\"string"; + for (slice) |b| { + if (b == '\xa3') break; + if (b == '\ua3d3') break; + if (b == '\Ua3d3d3') break; + if (b == '\t') break; + if (b == '\n') break; + if (b == '\\') break; + if (b == '\'') break; + if (b == '"') break; + if (b != 'n') break; + if (b != '-') break; + i += 1; + } + + return slice[i..]; +} diff --git a/tests/examplefiles/fennelview.fnl b/tests/examplefiles/fennelview.fnl new file mode 100644 index 00000000..fd0fc648 --- /dev/null +++ b/tests/examplefiles/fennelview.fnl @@ -0,0 +1,156 @@ +;; A pretty-printer that outputs tables in Fennel syntax. +;; Loosely based on inspect.lua: http://github.com/kikito/inspect.lua + +(local quote (fn [str] (.. '"' (: str :gsub '"' '\\"') '"'))) + +(local short-control-char-escapes + {"\a" "\\a" "\b" "\\b" "\f" "\\f" "\n" "\\n" + "\r" "\\r" "\t" "\\t" "\v" "\\v"}) + +(local long-control-char-esapes + (let [long {}] + (for [i 0 31] + (let [ch (string.char i)] + (when (not (. short-control-char-escapes ch)) + (tset short-control-char-escapes ch (.. "\\" i)) + (tset long ch (: "\\%03d" :format i))))) + long)) + +(fn escape [str] + (let [str (: str :gsub "\\" "\\\\") + str (: str :gsub "(%c)%f[0-9]" long-control-char-esapes)] + (: str :gsub "%c" short-control-char-escapes))) + +(fn sequence-key? [k len] + (and (= (type k) "number") + (<= 1 k) + (<= k len) + (= (math.floor k) k))) + +(local type-order {:number 1 :boolean 2 :string 3 :table 4 + :function 5 :userdata 6 :thread 7}) + +(fn sort-keys [a b] + (let [ta (type a) tb (type b)] + (if (and (= ta tb) (~= ta "boolean") + (or (= ta "string") (= ta "number"))) + (< a b) + (let [dta (. type-order a) + dtb (. type-order b)] + (if (and dta dtb) + (< dta dtb) + dta true + dtb false + :else (< ta tb)))))) + +(fn get-sequence-length [t] + (var len 1) + (each [i (ipairs t)] (set len i)) + len) + +(fn get-nonsequential-keys [t] + (let [keys {} + sequence-length (get-sequence-length t)] + (each [k (pairs t)] + (when (not (sequence-key? k sequence-length)) + (table.insert keys k))) + (table.sort keys sort-keys) + (values keys sequence-length))) + +(fn count-table-appearances [t appearances] + (if (= (type t) "table") + (when (not (. appearances t)) + (tset appearances t 1) + (each [k v (pairs t)] + (count-table-appearances k appearances) + (count-table-appearances v appearances))) + (when (and t (= t t)) ; no nans please + (tset appearances t (+ (or (. appearances t) 0) 1)))) + appearances) + + + +(var put-value nil) ; mutual recursion going on; defined below + +(fn puts [self ...] + (each [_ v (ipairs [...])] + (table.insert self.buffer v))) + +(fn tabify [self] (puts self "\n" (: self.indent :rep self.level))) + +(fn already-visited? [self v] (~= (. self.ids v) nil)) + +(fn get-id [self v] + (var id (. self.ids v)) + (when (not id) + (let [tv (type v)] + (set id (+ (or (. self.max-ids tv) 0) 1)) + (tset self.max-ids tv id) + (tset self.ids v id))) + (tostring id)) + +(fn put-sequential-table [self t length] + (puts self "[") + (set self.level (+ self.level 1)) + (for [i 1 length] + (puts self " ") + (put-value self (. t i))) + (set self.level (- self.level 1)) + (puts self " ]")) + +(fn put-key [self k] + (if (and (= (type k) "string") + (: k :find "^[-%w?\\^_`!#$%&*+./@~:|<=>]+$")) + (puts self ":" k) + (put-value self k))) + +(fn put-kv-table [self t] + (puts self "{") + (set self.level (+ self.level 1)) + (each [k v (pairs t)] + (tabify self) + (put-key self k) + (puts self " ") + (put-value self v)) + (set self.level (- self.level 1)) + (tabify self) + (puts self "}")) + +(fn put-table [self t] + (if (already-visited? self t) + (puts self "#<table " (get-id self t) ">") + (>= self.level self.depth) + (puts self "{...}") + :else + (let [(non-seq-keys length) (get-nonsequential-keys t) + id (get-id self t)] + (if (> (. self.appearances t) 1) + (puts self "#<" id ">") + (and (= (# non-seq-keys) 0) (= (# t) 0)) + (puts self "{}") + (= (# non-seq-keys) 0) + (put-sequential-table self t length) + :else + (put-kv-table self t))))) + +(set put-value (fn [self v] + (let [tv (type v)] + (if (= tv "string") + (puts self (quote (escape v))) + (or (= tv "number") (= tv "boolean") (= tv "nil")) + (puts self (tostring v)) + (= tv "table") + (put-table self v) + :else + (puts self "#<" (tostring v) ">"))))) + + + +(fn fennelview [root options] + (let [options (or options {}) + inspector {:appearances (count-table-appearances root {}) + :depth (or options.depth 128) + :level 0 :buffer {} :ids {} :max-ids {} + :indent (or options.indent " ")}] + (put-value inspector root) + (table.concat inspector.buffer))) diff --git a/tests/examplefiles/freefem.edp b/tests/examplefiles/freefem.edp new file mode 100644 index 00000000..d4313338 --- /dev/null +++ b/tests/examplefiles/freefem.edp @@ -0,0 +1,94 @@ +// Example of problem solving in parallel + +// Usage: +// ff-mpirun -np 12 LaplacianParallel.edp (here 12 is the number of threads (command nproc to know that) +// Need FreeFem++ with PETSc + +// Parallel stuff +load "PETSc" +macro partitioner()metis// +macro dimension()2// +include "./macro_ddm.idp" + +macro def(i)[i]// +macro init(i)[i]// +//macro meshN()mesh// //these macro are defined in macro_ddm.idp +//macro intN()int2d// + +// Parameters +int nn = 500; +real L = 1.; +real H = 1.; + +func f = 1.; + +func Pk = P1; + +// Mesh +border b1(t=0, L){x=t; y=0; label=1;} +border b2(t=0, H){x=L; y=t; label=2;} +border b3(t=L, 0){x=t; y=H; label=3;} +border b4(t=H, 0){x=0; y=t; label=4;} + +meshN Th = buildmesh(b1(1) + b2(1) + b3(1) + b4(1)); //build a really coarse mesh (just to build the fespace later) +//meshN Th = square(1, 1, [L*x, H*y]); + +int[int] Wall = [1, 2, 3, 4]; + +// Fespace +fespace Uh(Th, Pk); + +// Mesh partition +int[int] ArrayIntersection; +int[int][int] RestrictionIntersection(0); +real[int] D; + +meshN ThBorder; +meshN ThGlobal = buildmesh(b1(nn*L) + b2(nn*H) + b3(nn*L) + b4(nn*H)); //build the mesh to partition +//meshN ThGlobal = square(nn*L, nn*H, [L*x, H*y]); +int InterfaceLabel = 10; +int Split = 1; +int Overlap = 1; +build(Th, ThBorder, ThGlobal, InterfaceLabel, Split, Overlap, D, ArrayIntersection, RestrictionIntersection, Uh, Pk, mpiCommWorld, false); //see macro_ddm.idp for detailed parameters + +// Macro +macro grad(u) [dx(u), dy(u)] // + +// Problem +varf vLaplacian (u, uh) //Problem in varf formulation mandatory + = intN(Th)( + grad(u)' * grad(uh) + ) + - intN(Th)( + f * uh + ) + + on(Wall, u=0) + ; + +matrix<real> Laplacian = vLaplacian(Uh, Uh); //build the sequential matrix +real[int] LaplacianBoundary = vLaplacian(0, Uh);// and right hand side + +//// In sequential, you normally do that: +//// Solve +//Uh def(u)=init(0); +//u[] = Laplacian^-1 * LaplacianBoundary; + +//// Plot +//plot(u); + +// In parallel: +// Matrix construction +dmatrix PLaplacian(Laplacian, ArrayIntersection, RestrictionIntersection, D, bs=1); //build the parallel matrix +set(PLaplacian, sparams="-pc_type lu -pc_factor_mat_solver_package mumps"); //preconditioner LU and MUMPS solver (see PETSc doc for detailed parameters) + +// Solve +Uh def(u)=init(0); //define the unknown (must be defined after mesh partitioning) +u[] = PLaplacian^-1 * LaplacianBoundary; + +// Export results to vtk (there is not plot in parallel) +{ + fespace PV(Th, P1); + PV uu=u; + int[int] Order = [1]; + export("Result", Th, uu, Order, mpiCommWorld); +} diff --git a/tests/examplefiles/teraterm.ttl b/tests/examplefiles/teraterm.ttl new file mode 100644 index 00000000..f6a3648a --- /dev/null +++ b/tests/examplefiles/teraterm.ttl @@ -0,0 +1,34 @@ +messagebox "text \\not escaped \nescaped n" "other\n\rthing"
+messagebox "goto label /* a string */ ; same string"
+a=10
+b= 'abc'#$41'def'
+c =#65 /* multiline comment * / * / *//*
+comment */ d = 10 ; inline comment /* still inline */
+e = d + 20 - (($a * 2) / 4) << 3 % (2 >> 1) + result
+
+
+:thing
+
+strcompare c "thing"
+if result = 1 then
+ goto label_
+elseif result > -1 then
+ goto 10
+elseif d > (1+2*3)/7 then
+ messagebox "thing"
+else
+ messagebox "done"
+endif
+
+if abc messagebox "thing1" "title"
+
+
+; Invalid syntax
+bad = "no closing double quote
+bad = 'no closing single quote
+garbage
+...
+...
+...
+
+endgarbage
diff --git a/tests/examplefiles/test.csd b/tests/examplefiles/test.csd index 9122309b..6512d99e 100644 --- a/tests/examplefiles/test.csd +++ b/tests/examplefiles/test.csd @@ -1,260 +1,18 @@ +/* + * comment + */ +; comment +// comment +/ <CsoundSynthesizer> <CsInstruments> -// This is a Csound orchestra file for testing a Pygments <http://pygments.org> -// lexer. Csound single-line comments can be preceded by a pair of forward -// slashes... -; ...or a semicolon. - -/* Block comments begin with /* and end with */ - -// Orchestras begin with a header of audio parameters. -nchnls = 1 -nchnls_i = 1 -sr = 44100 0dbfs = 1 -ksmps = 10 - -// The control rate kr = sr / ksmps can be omitted when the number of audio -// samples in a control period (ksmps) is set, but kr may appear in older -// orchestras. -kr = 4410 - -// Orchestras contain instruments. These begin with the keyword instr followed -// by a comma-separated list of numbers or names of the instrument. Instruments -// end at the endin keyword and cannot be nested. -instr 1, N_a_M_e_, +Name - // Instruments contain statements. Here is a typical statement: - aSignal oscil 0dbfs, 440, 1 - // Statements are terminated with a newline (possibly preceded by a comment). - // To write a statement on several lines, precede the newline with a - // backslash. - prints \ - "hello, world\n";comment - - // Csound 6 introduced function syntax for opcodes with one or zero outputs. - // The oscil statement above is the same as - aSignal = oscil(0dbfs, 440, 1) - - // Instruments can contain control structures. - kNote = p3 - if (kNote == 0) then - kFrequency = 220 - elseif kNote == 1 then // Parentheses around binary expressions are optional. - kFrequency = 440 - endif - - // Csound 6 introduced looping structures. - iIndex = 0 - while iIndex < 5 do - print iIndex - iIndex += 1 - od - iIndex = 0 - until iIndex >= 5 do - print iIndex - iIndex += 1 - enduntil - // Both kinds of loops can be terminated by either od or enduntil. - - // Single-line strings are enclosed in double-quotes. - prints "string\\\r\n\t\"" - // Multi-line strings are enclosed in pairs of curly braces. - prints {{ - hello, - - world - }} - - // Instruments often end with a statement containing an output opcode. - outc aSignal -endin - -// Orchestras can also contain user-defined opcodes (UDOs). Here is an -// oscillator with one audio-rate output and two control-rate inputs: -opcode anOscillator, a, kk - kAmplitude, kFrequency xin - aSignal vco2 kAmplitude, kFrequency - xout aSignal -endop -instr TestOscillator - outc(anOscillator(0dbfs, 110)) -endin - -// Python can be executed in Csound -// <http://www.csounds.com/manual/html/pyrun.html>. So can Lua -// <http://www.csounds.com/manual/html/lua.html>. -pyruni {{ -import random - -pool = [(1 + i / 10.0) ** 1.2 for i in range(100)] - -def get_number_from_pool(n, p): - if random.random() < p: - i = int(random.random() * len(pool)) - pool[i] = n - return random.choice(pool) -}} - -// The Csound preprocessor supports conditional compilation and including files. -#ifdef DEBUG -#undef DEBUG -#include "filename.orc" -#endif - -// The preprocessor also supports object- and function-like macros. This is an -// object-like macro that defines a number: -#define A_HZ #440# - -// This is a function-like macro: -#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# - -// Bodies of macros are enclosed in # and can contain newlines. The arguments of -// function-like macros are separated by single-quotes. Uses of macros are -// prefixed with a dollar sign. -instr TestMacro - aSignal $OSCIL_MACRO(1'$A_HZ'1) - // Not unlike PHP, macros expand in double-quoted strings. - prints "The frequency of the oscillator is $A_HZ Hz.\n" - out aSignal -endin - -// Here are other things to note about Csound. - -// There are two bitwise NOT operators, ~ and ¬ (U+00AC). The latter is common -// on keyboards in the United Kingdom -// <https://en.wikipedia.org/wiki/British_and_American_keyboards>. -instr TestBitwiseNOT - print ~42 - print ¬42 -endin - -// Csound uses # for bitwise XOR, which the Csound manual calls bitwise -// non-equivalence <http://www.csounds.com/manual/html/opnonequiv.html>. -instr TestBitwiseXOR - print 0 # 0 - print 0 # 1 - print 1 # 0 - print 1 # 1 -endin - -// Loops and if-then statements are relatively recent additions to Csound. There -// are many flow-control opcodes that involve goto and labels. -instr TestGoto - // This... - if p3 > 0 goto if_label - goto else_label -if_label: - prints "if branch\n" - goto endif_label -else_label: - prints "else branch\n" -endif_label: - - // ...is the same as this. - if p3 > 0 then - prints "if branch\n" - else - prints "else branch\n" - endif - - // This... - iIndex = 0 -loop_label: - print iIndex - iIndex += 1 - if iIndex < 10 goto loop_label - - // ...is the same as this... - iIndex = 0 -loop_lt_label: - print iIndex - loop_lt iIndex, 1, 10, loop_lt_label - - // ...and this. - iIndex = 0 - while iIndex < 10 do - print iIndex - iIndex += 1 - od -endin - -// The prints and printks opcodes -// <https://github.com/csound/csound/blob/develop/OOps/ugrw1.c#L831>, arguably -// the primary methods of logging output, treat certain sequences of characters -// different from printf in C. -instr TestPrints - // ^ prints an ESCAPE character (U+001B), not a CIRCUMFLEX ACCENT character - // (U+005E). ^^ prints a CIRCUMFLEX ACCENT. - prints "^^\n" - // ~ prints an ESCAPE character (U+001B) followed by a [, not a TILDE - // character (U+007E). ~~ prints a TILDE. - prints "~~\n" - // \A, \B, \N, \R, and \T correspond to the escaped lowercase characters (that - // is, BELL (U+0007), BACKSPACE (U+0008), new line (U+000A), CARRIAGE RETURN - // (U+000D), and tab (U+0009)). - prints "\T\R\N" - // %n, %r, and %t are the same as \n, \r, and \t, as are %N, %R, and %T. - prints "%t%r%n" - // %! prints a semicolon. This is a hold-over from old versions of Csound that - // allowed comments to begin in strings. - prints "; %!\n" -endin - -// The arguments of function-like macros can be separated by # instead of '. -// These two lines define the same macro. -#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# -#define OSCIL_MACRO(VOLUME#FREQUENCY#TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# - -// Uses of macros can optionally be suffixed with a period. -instr TestMacroPeriodSuffix - aSignal $OSCIL_MACRO.(1'$A_HZ'1) - prints "The frequency of the oscillator is $A_HZ.Hz.\n" - out aSignal -endin - -// Csound has @ and @@ operator-like macros that, when followed by a literal -// non-negative integer, expand to the next power of 2 and the next power of 2 -// plus 1: -// @x = 2^(ceil(log2(x + 1))), x >= 0 -// @@0 = 2 -// @@x = 2^(ceil(log2(x))) + 1, x > 0 -// These macros are in -// <https://github.com/csound/csound/blob/develop/Engine/csound_orc.l#L542> (and -// <https://github.com/csound/csound/blob/develop/Engine/csound_sco.lex#L202>) -// and are described at <http://www.csounds.com/manual/html/ScoreEval.html>. -instr TestAt - prints "%d %2d %2d\n", 0, @0, @@0 - prints "%d %2d %2d\n", 1, @1, @@1 - prints "%d %2d %2d\n", 2, @2, @@2 - prints "%d %2d %2d\n", 3, @3, @@3 - prints "%d %2d %2d\n", 4, @4, @@4 - prints "%d %2d %2d\n", 5, @5, @@5 - prints "%d %2d %2d\n", 6, @6, @@6 - prints "%d %2d %2d\n", 7, @7, @@7 - prints "%d %2d %2d\n", 8, @8, @@8 - prints "%d %2d %2d\n", 9, @9, @@9 -endin - -// Including newlines in macros can lead to confusing code, but it tests the -// lexer. -instr MacroAbuse - if 1 == 1 then - prints "on\n" -#define FOO# -BAR -#endif // This ends the if statement. It is not a preprocessor directive. -endin +prints "hello, world\n" </CsInstruments> <CsScore> -f 1 0 16384 10 1 -i "N_a_M_e_" 0 2 -i "TestOscillator" 2 2 -i "TestBitwiseNOT" 0 1 -i "TestBitwiseXOR" 0 1 -i "TestGoto" 0 1 -i "TestMacroPeriodSuffix" 4 1 -i "TestAt" 0 1 -i "MacroAbuse" 0 1 -e +i 1 0 0 </CsScore> +<html> +<!DOCTYPE html> +</html> </CsoundSynthesizer> diff --git a/tests/examplefiles/test.orc b/tests/examplefiles/test.orc index 36725342..d113303e 100644 --- a/tests/examplefiles/test.orc +++ b/tests/examplefiles/test.orc @@ -1,257 +1,81 @@ -// This is a Csound orchestra file for testing a Pygments <http://pygments.org> -// lexer. Csound single-line comments can be preceded by a pair of forward -// slashes... -; ...or a semicolon. - -/* Block comments begin with /* and end with */ - -// Orchestras begin with a header of audio parameters. -nchnls = 1 -nchnls_i = 1 -sr = 44100 -0dbfs = 1 -ksmps = 10 - -// The control rate kr = sr / ksmps can be omitted when the number of audio -// samples in a control period (ksmps) is set, but kr may appear in older -// orchestras. -kr = 4410 - -// Orchestras contain instruments. These begin with the keyword instr followed -// by a comma-separated list of numbers or names of the instrument. Instruments -// end at the endin keyword and cannot be nested. -instr 1, N_a_M_e_, +Name - // Instruments contain statements. Here is a typical statement: - aSignal oscil 0dbfs, 440, 1 - // Statements are terminated with a newline (possibly preceded by a comment). - // To write a statement on several lines, precede the newline with a - // backslash. - prints \ - "hello, world\n";comment - - // Csound 6 introduced function syntax for opcodes with one or zero outputs. - // The oscil statement above is the same as - aSignal = oscil(0dbfs, 440, 1) - - // Instruments can contain control structures. - kNote = p3 - if (kNote == 0) then - kFrequency = 220 - elseif kNote == 1 then // Parentheses around binary expressions are optional. - kFrequency = 440 - endif - - // Csound 6 introduced looping structures. - iIndex = 0 - while iIndex < 5 do - print iIndex - iIndex += 1 - od - iIndex = 0 - until iIndex >= 5 do - print iIndex - iIndex += 1 - enduntil - // Both kinds of loops can be terminated by either od or enduntil. - - // Single-line strings are enclosed in double-quotes. - prints "string\\\r\n\t\"" - // Multi-line strings are enclosed in pairs of curly braces. - prints {{ - hello, - - world - }} - - // Instruments often end with a statement containing an output opcode. - outc aSignal +/* + * comment + */ +; comment +// comment + +instr/**/1,/**/N_a_M_e_,/**/+Name/**/// + iDuration = p3 + outc:a(aSignal) endin -// Orchestras can also contain user-defined opcodes (UDOs). Here is an -// oscillator with one audio-rate output and two control-rate inputs: -opcode anOscillator, a, kk - kAmplitude, kFrequency xin - aSignal vco2 kAmplitude, kFrequency - xout aSignal +opcode/**/aUDO,/**/i[],/**/aik// + aUDO endop -instr TestOscillator - outc(anOscillator(0dbfs, 110)) -endin -// Python can be executed in Csound -// <http://www.csounds.com/manual/html/pyrun.html>. So can Lua -// <http://www.csounds.com/manual/html/lua.html>. -pyruni {{ -import random +123 0123456789 +0xabcdef0123456789 0XABCDEF +1e2 3e+4 5e-6 7E8 9E+0 1E-2 3. 4.56 .789 -pool = [(1 + i / 10.0) ** 1.2 for i in range(100)] +"characters$MACRO." +"\\\a\b\n\r\t\012\345\67\"" -def get_number_from_pool(n, p): - if random.random() < p: - i = int(random.random() * len(pool)) - pool[i] = n - return random.choice(pool) +{{ +characters$MACRO. }} +{{\\\a\b\n\r\t\"\012\345\67}} -// The Csound preprocessor supports conditional compilation and including files. -#ifdef DEBUG -#undef DEBUG -#include "filename.orc" -#endif - -// The preprocessor also supports object- and function-like macros. This is an -// object-like macro that defines a number: -#define A_HZ #440# - -// This is a function-like macro: -#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# - -// Bodies of macros are enclosed in # and can contain newlines. The arguments of -// function-like macros are separated by single-quotes. Uses of macros are -// prefixed with a dollar sign. -instr TestMacro - aSignal $OSCIL_MACRO(1'$A_HZ'1) - // Not unlike PHP, macros expand in double-quoted strings. - prints "The frequency of the oscillator is $A_HZ Hz.\n" - out aSignal -endin - -// Here are other things to note about Csound. ++ - ~ ¬ ! * / ^ % << >> < > <= >= == != & # | && || ? : += -= *= /= -// There are two bitwise NOT operators, ~ and ¬ (U+00AC). The latter is common -// on keyboards in the United Kingdom -// <https://en.wikipedia.org/wiki/British_and_American_keyboards>. -instr TestBitwiseNOT - print ~42 - print ¬42 -endin - -// Csound uses # for bitwise XOR, which the Csound manual calls bitwise -// non-equivalence <http://www.csounds.com/manual/html/opnonequiv.html>. -instr TestBitwiseXOR - print 0 # 0 - print 0 # 1 - print 1 # 0 - print 1 # 1 -endin +0dbfs A4 kr ksmps nchnls nchnls_i sr -// Loops and if-then statements are relatively recent additions to Csound. There -// are many flow-control opcodes that involve goto and labels. -instr TestGoto - // This... - if p3 > 0 goto if_label - goto else_label -if_label: - prints "if branch\n" - goto endif_label -else_label: - prints "else branch\n" -endif_label: +do else elseif endif enduntil fi if ithen kthen od then until while +return rireturn - // ...is the same as this. - if p3 > 0 then - prints "if branch\n" - else - prints "else branch\n" - endif +aLabel: + label2: - // This... - iIndex = 0 -loop_label: - print iIndex - iIndex += 1 - if iIndex < 10 goto loop_label +goto aLabel +reinit aLabel +cggoto 1==0, aLabel +timout 0, 0, aLabel +loop_ge 0, 0, 0, aLabel - // ...is the same as this... - iIndex = 0 -loop_lt_label: - print iIndex - loop_lt iIndex, 1, 10, loop_lt_label - - // ...and this. - iIndex = 0 - while iIndex < 10 do - print iIndex - iIndex += 1 - od -endin +prints "%! %% %n%N %r%R %t%T \\a\\A \\b\\B \\n\\N \\r\\R \\t\\T" +prints Soutput -// The prints and printks opcodes -// <https://github.com/csound/csound/blob/develop/OOps/ugrw1.c#L831>, arguably -// the primary methods of logging output, treat certain sequences of characters -// different from printf in C. -instr TestPrints - // ^ prints an ESCAPE character (U+001B), not a CIRCUMFLEX ACCENT character - // (U+005E). ^^ prints a CIRCUMFLEX ACCENT. - prints "^^\n" - // ~ prints an ESCAPE character (U+001B) followed by a [, not a TILDE - // character (U+007E). ~~ prints a TILDE. - prints "~~\n" - // \A, \B, \N, \R, and \T correspond to the escaped lowercase characters (that - // is, BELL (U+0007), BACKSPACE (U+0008), new line (U+000A), CARRIAGE RETURN - // (U+000D), and tab (U+0009)). - prints "\T\R\N" - // %n, %r, and %t are the same as \n, \r, and \t, as are %N, %R, and %T. - prints "%t%r%n" - // %! prints a semicolon. This is a hold-over from old versions of Csound that - // allowed comments to begin in strings. - prints "; %!\n" -endin - -// The arguments of function-like macros can be separated by # instead of '. -// These two lines define the same macro. -#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# -#define OSCIL_MACRO(VOLUME#FREQUENCY#TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE# - -// Uses of macros can optionally be suffixed with a period. -instr TestMacroPeriodSuffix - aSignal $OSCIL_MACRO.(1'$A_HZ'1) - prints "The frequency of the oscillator is $A_HZ.Hz.\n" - out aSignal -endin - -// Csound has @ and @@ operator-like macros that, when followed by a literal -// non-negative integer, expand to the next power of 2 and the next power of 2 -// plus 1: -// @x = 2^(ceil(log2(x + 1))), x >= 0 -// @@0 = 2 -// @@x = 2^(ceil(log2(x))) + 1, x > 0 -// These macros are in -// <https://github.com/csound/csound/blob/develop/Engine/csound_orc.l#L542> (and -// <https://github.com/csound/csound/blob/develop/Engine/csound_sco.lex#L202>) -// and are described at <http://www.csounds.com/manual/html/ScoreEval.html>. -instr TestAt - prints "%d %2d %2d\n", 0, @0, @@0 - prints "%d %2d %2d\n", 1, @1, @@1 - prints "%d %2d %2d\n", 2, @2, @@2 - prints "%d %2d %2d\n", 3, @3, @@3 - prints "%d %2d %2d\n", 4, @4, @@4 - prints "%d %2d %2d\n", 5, @5, @@5 - prints "%d %2d %2d\n", 6, @6, @@6 - prints "%d %2d %2d\n", 7, @7, @@7 - prints "%d %2d %2d\n", 8, @8, @@8 - prints "%d %2d %2d\n", 9, @9, @@9 -endin +readscore {{ +i 1 0 0 +}} +pyrun {{ +# Python +}} +lua_exec {{ +-- Lua +}} -// Including newlines in macros can lead to confusing code, but it tests the -// lexer. -instr MacroAbuse - if 1 == 1 then - prints "on\n" -#define FOO# -BAR -#endif // This ends the if statement. It is not a preprocessor directive. -endin +#include/**/"file.udo" +#include/**/|file.udo| -scoreline_i {{ -f 1 0 16384 10 1 -i "N_a_M_e_" 0 2 -i "TestOscillator" 2 2 -i "TestBitwiseNOT" 0 1 -i "TestBitwiseXOR" 0 1 -i "TestGoto" 0 1 -i "TestMacroPeriodSuffix" 4 1 -i "TestAt" 0 1 -i "MacroAbuse" 0 1 -e -}} +#ifdef MACRO +#else +#ifndef MACRO +#endif +#undef MACRO + +# define MACRO#macro_body# +#define/**/ +MACRO/**/ +#\#macro +body\## + +#define MACRO(ARG1#ARG2) #macro_body# +#define/**/ +MACRO(ARG1'ARG2'ARG3)/**/ +#\#macro +body\## + +$MACRO $MACRO. +$MACRO(x) +@0 +@@ 1 diff --git a/tests/examplefiles/test.sco b/tests/examplefiles/test.sco index a0b39251..d997c1b3 100644 --- a/tests/examplefiles/test.sco +++ b/tests/examplefiles/test.sco @@ -1,10 +1,22 @@ -f 1 0 16384 10 1 -i "N_a_M_e_" 0 2 -i "TestOscillator" 2 2 -i "TestBitwiseNOT" 0 1 -i "TestBitwiseXOR" 0 1 -i "TestGoto" 0 1 -i "TestMacroPeriodSuffix" 4 1 -i "TestAt" 0 1 -i "MacroAbuse" 0 1 -e +/* + * comment + */ +; comment +// comment +a b C d e f i q s t v x y +z +np0 nP1 Np2 NP3 +m/**/label; +n label +123 0123456789 +0xabcdef0123456789 0XABCDEF +1e2 3e+4 5e-6 7E8 9E+0 1E-2 3. 4.56 .789 +"characters$MACRO." +{ 1 I + { 2 J + { 3 K + $I $J $K + } + } +} +#include "score.sco" diff --git a/tests/run.py b/tests/run.py index 07665b2a..edebc7a1 100644 --- a/tests/run.py +++ b/tests/run.py @@ -8,7 +8,7 @@ python run.py [testfile ...] - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -16,11 +16,19 @@ from __future__ import print_function import os import sys +import warnings # only find tests in this directory if os.path.dirname(__file__): os.chdir(os.path.dirname(__file__)) +# make FutureWarnings (coming from Regex syntax most likely) and +# DeprecationWarnings due to non-raw strings an error +warnings.filterwarnings("error", module=r"pygments\..*", + category=FutureWarning) +warnings.filterwarnings("error", module=r".*pygments.*", + category=DeprecationWarning) + try: import nose diff --git a/tests/string_asserts.py b/tests/string_asserts.py index 05e95e6a..a02c52bb 100644 --- a/tests/string_asserts.py +++ b/tests/string_asserts.py @@ -3,7 +3,7 @@ Pygments string assert utility ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_asm.py b/tests/test_asm.py new file mode 100644 index 00000000..30a008a1 --- /dev/null +++ b/tests/test_asm.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" + Basic ColdfusionHtmlLexer Test + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest +import os + +from pygments.token import Token +from pygments.lexers import NasmLexer + + +class NasmLexerTest(unittest.TestCase): + + def setUp(self): + self.lexer = NasmLexer() + + def testCPUID(self): + # CPU is a valid directive, and we don't want to parse this as + # cpu id, but as a single token. See bug #1517 + fragment = 'cpuid' + expected = [ + (Token.Name.Function, u'cpuid'), + (Token.Text, u'\n'), + ] + self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_basic.py b/tests/test_basic.py new file mode 100644 index 00000000..03d10cd2 --- /dev/null +++ b/tests/test_basic.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +""" + Pygments Basic lexers tests + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import unittest + +from pygments.lexers.basic import VBScriptLexer +from pygments.token import Error, Name, Number, Punctuation, String, Whitespace + + +class VBScriptLexerTest(unittest.TestCase): + + def setUp(self): + self.lexer = VBScriptLexer() + + def _assert_are_tokens_of_type(self, examples, expected_token_type): + for test_number, example in enumerate(examples.split(), 1): + token_count = 0 + for token_type, token_value in self.lexer.get_tokens(example): + if token_type != Whitespace: + token_count += 1 + self.assertEqual( + token_type, expected_token_type, + 'token_type #%d for %s is be %s but must be %s' % + (test_number, token_value, token_type, expected_token_type)) + self.assertEqual( + token_count, 1, + '%s must yield exactly 1 token instead of %d' % + (example, token_count)) + + def _assert_tokens_match(self, text, expected_tokens_without_trailing_newline): + actual_tokens = tuple(self.lexer.get_tokens(text)) + if (len(actual_tokens) >= 1) and (actual_tokens[-1] == (Whitespace, '\n')): + actual_tokens = tuple(actual_tokens[:-1]) + self.assertEqual( + expected_tokens_without_trailing_newline, actual_tokens, + 'text must yield expected tokens: %s' % text) + + def test_can_lex_float(self): + self._assert_are_tokens_of_type( + '1. 1.e1 .1 1.2 1.2e3 1.2e+3 1.2e-3 1e2', Number.Float) + self._assert_tokens_match( + '1e2.1e2', + ((Number.Float, '1e2'), (Number.Float, '.1e2')) + ) + + def test_can_reject_almost_float(self): + self._assert_tokens_match( + '.e1', + ((Punctuation, '.'), (Name, 'e1'))) + + def test_can_lex_integer(self): + self._assert_are_tokens_of_type( + '1 23 456', Number.Integer) + + def test_can_lex_names(self): + self._assert_are_tokens_of_type( + u'thingy thingy123 _thingy _123', Name) + + def test_can_recover_after_unterminated_string(self): + self._assert_tokens_match( + '"x\nx', + ((String.Double, '"'), (String.Double, 'x'), (Error, '\n'), (Name, 'x')) + ) + + def test_can_recover_from_invalid_character(self): + self._assert_tokens_match( + 'a;bc\nd', + ((Name, 'a'), (Error, ';bc\n'), (Name, 'd')) + ) diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index ac3b4a51..b1b69267 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -3,7 +3,7 @@ Pygments basic API tests ~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_bibtex.py b/tests/test_bibtex.py index 5ad92db4..2f1c395a 100644 --- a/tests/test_bibtex.py +++ b/tests/test_bibtex.py @@ -3,7 +3,7 @@ BibTeX Test ~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_cfm.py b/tests/test_cfm.py index 0ff1b167..e7147a6c 100644 --- a/tests/test_cfm.py +++ b/tests/test_cfm.py @@ -3,7 +3,7 @@ Basic ColdfusionHtmlLexer Test ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_clexer.py b/tests/test_clexer.py index 5095b797..64b765ef 100644 --- a/tests/test_clexer.py +++ b/tests/test_clexer.py @@ -3,7 +3,7 @@ Basic CLexer Test ~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py index 1500c875..169d690d 100644 --- a/tests/test_cmdline.py +++ b/tests/test_cmdline.py @@ -3,7 +3,7 @@ Command line test ~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -28,6 +28,13 @@ def func(args): ''' +def _decode_output(text): + try: + return text.decode('utf-8') + except UnicodeEncodeError: # implicit encode on Python 2 with data loss + return text + + def run_cmdline(*args, **kwds): saved_stdin = sys.stdin saved_stdout = sys.stdout @@ -53,9 +60,9 @@ def run_cmdline(*args, **kwds): sys.stderr = saved_stderr new_stdout.flush() new_stderr.flush() - out, err = stdout_buffer.getvalue().decode('utf-8'), \ - stderr_buffer.getvalue().decode('utf-8') - return (ret, out, err) + out, err = stdout_buffer.getvalue(), \ + stderr_buffer.getvalue() + return (ret, _decode_output(out), _decode_output(err)) class CmdLineTest(unittest.TestCase): diff --git a/tests/test_csound.py b/tests/test_csound.py new file mode 100644 index 00000000..8a253916 --- /dev/null +++ b/tests/test_csound.py @@ -0,0 +1,491 @@ +# -*- coding: utf-8 -*- +""" + Csound lexer tests + ~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest +from textwrap import dedent + +from pygments.token import Comment, Error, Keyword, Name, Number, Operator, Punctuation, \ + String, Text +from pygments.lexers import CsoundOrchestraLexer + + +class CsoundOrchestraTest(unittest.TestCase): + + def setUp(self): + self.lexer = CsoundOrchestraLexer() + self.maxDiff = None + + def testComments(self): + fragment = dedent('''\ + /* + * comment + */ + ; comment + // comment + ''') + tokens = [ + (Comment.Multiline, u'/*\n * comment\n */'), + (Text, u'\n'), + (Comment.Single, u'; comment'), + (Text, u'\n'), + (Comment.Single, u'// comment'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testInstrumentBlocks(self): + fragment = dedent('''\ + instr/**/1,/**/N_a_M_e_,/**/+Name/**/// + iDuration = p3 + outc:a(aSignal) + endin + ''') + tokens = [ + (Keyword.Declaration, u'instr'), + (Comment.Multiline, u'/**/'), + (Name.Function, u'1'), + (Punctuation, u','), + (Comment.Multiline, u'/**/'), + (Name.Function, u'N_a_M_e_'), + (Punctuation, u','), + (Comment.Multiline, u'/**/'), + (Punctuation, u'+'), + (Name.Function, u'Name'), + (Comment.Multiline, u'/**/'), + (Comment.Single, u'//'), + (Text, u'\n'), + (Text, u' '), + (Keyword.Type, u'i'), + (Name, u'Duration'), + (Text, u' '), + (Operator, u'='), + (Text, u' '), + (Name.Variable.Instance, u'p3'), + (Text, u'\n'), + (Text, u' '), + (Name.Builtin, u'outc'), + (Punctuation, u':'), + (Keyword.Type, u'a'), + (Punctuation, u'('), + (Keyword.Type, u'a'), + (Name, u'Signal'), + (Punctuation, u')'), + (Text, u'\n'), + (Keyword.Declaration, u'endin'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testUserDefinedOpcodes(self): + fragment = dedent('''\ + opcode/**/aUDO,/**/i[],/**/aik// + aUDO + endop + ''') + tokens = [ + (Keyword.Declaration, u'opcode'), + (Comment.Multiline, u'/**/'), + (Name.Function, u'aUDO'), + (Punctuation, u','), + (Comment.Multiline, u'/**/'), + (Keyword.Type, u'i[]'), + (Punctuation, u','), + (Comment.Multiline, u'/**/'), + (Keyword.Type, u'aik'), + (Comment.Single, u'//'), + (Text, u'\n'), + (Text, u' '), + (Name.Function, u'aUDO'), + (Text, u'\n'), + (Keyword.Declaration, u'endop'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testNumbers(self): + fragment = '123 0123456789' + tokens = [ + (Number.Integer, u'123'), + (Text, u' '), + (Number.Integer, u'0123456789'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + fragment = '0xabcdef0123456789 0XABCDEF' + tokens = [ + (Keyword.Type, u'0x'), + (Number.Hex, u'abcdef0123456789'), + (Text, u' '), + (Keyword.Type, u'0X'), + (Number.Hex, u'ABCDEF'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + fragments = ['1e2', '3e+4', '5e-6', '7E8', '9E+0', '1E-2', '3.', '4.56', '.789'] + for fragment in fragments: + tokens = [ + (Number.Float, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testQuotedStrings(self): + fragment = '"characters$MACRO."' + tokens = [ + (String, u'"'), + (String, u'characters'), + (Comment.Preproc, u'$MACRO.'), + (String, u'"'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testBracedStrings(self): + fragment = dedent('''\ + {{ + characters$MACRO. + }} + ''') + tokens = [ + (String, u'{{'), + (String, u'\ncharacters$MACRO.\n'), + (String, u'}}'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testEscapeSequences(self): + for character in ['\\', 'a', 'b', 'n', 'r', 't', '"', '012', '345', '67']: + escapedCharacter = '\\' + character + fragment = '"' + escapedCharacter + '"' + tokens = [ + (String, u'"'), + (String.Escape, escapedCharacter), + (String, u'"'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + fragment = '{{' + escapedCharacter + '}}' + tokens = [ + (String, u'{{'), + (String.Escape, escapedCharacter), + (String, u'}}'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testOperators(self): + fragments = ['+', '-', '~', u'¬', '!', '*', '/', '^', '%', '<<', '>>', '<', '>', + '<=', '>=', '==', '!=', '&', '#', '|', '&&', '||', '?', ':', '+=', + '-=', '*=', '/='] + for fragment in fragments: + tokens = [ + (Operator, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testGlobalValueIdentifiers(self): + for fragment in ['0dbfs', 'A4', 'kr', 'ksmps', 'nchnls', 'nchnls_i', 'sr']: + tokens = [ + (Name.Variable.Global, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testKeywords(self): + fragments = ['do', 'else', 'elseif', 'endif', 'enduntil', 'fi', 'if', 'ithen', + 'kthen', 'od', 'then', 'until', 'while'] + for fragment in fragments: + tokens = [ + (Keyword, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + for fragment in ['return', 'rireturn']: + tokens = [ + (Keyword.Pseudo, fragment), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testLabels(self): + fragment = dedent('''\ + aLabel: + label2: + ''') + tokens = [ + (Name.Label, u'aLabel'), + (Punctuation, u':'), + (Text, u'\n'), + (Text, u' '), + (Name.Label, u'label2'), + (Punctuation, u':'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testPrintksAndPrintsEscapeSequences(self): + escapedCharacters = ['%!', '%%', '%n', '%N', '%r', '%R', '%t', '%T', '\\\\a', + '\\\\A', '\\\\b', '\\\\B', '\\\\n', '\\\\N', '\\\\r', + '\\\\R', '\\\\t', '\\\\T'] + for opcode in ['printks', 'prints']: + for escapedCharacter in escapedCharacters: + fragment = opcode + ' "' + escapedCharacter + '"' + tokens = [ + (Name.Builtin, opcode), + (Text, u' '), + (String, u'"'), + (String.Escape, escapedCharacter), + (String, u'"'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testGotoStatements(self): + for keyword in ['goto', 'igoto', 'kgoto']: + fragment = keyword + ' aLabel' + tokens = [ + (Keyword, keyword), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + for opcode in ['reinit', 'rigoto', 'tigoto']: + fragment = opcode + ' aLabel' + tokens = [ + (Keyword.Pseudo, opcode), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + for opcode in ['cggoto', 'cigoto', 'cingoto', 'ckgoto', 'cngoto', 'cnkgoto']: + fragment = opcode + ' 1==0, aLabel' + tokens = [ + (Keyword.Pseudo, opcode), + (Text, u' '), + (Number.Integer, u'1'), + (Operator, u'=='), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + fragment = 'timout 0, 0, aLabel' + tokens = [ + (Keyword.Pseudo, 'timout'), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + for opcode in ['loop_ge', 'loop_gt', 'loop_le', 'loop_lt']: + fragment = opcode + ' 0, 0, 0, aLabel' + tokens = [ + (Keyword.Pseudo, opcode), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Number.Integer, u'0'), + (Punctuation, u','), + (Text, u' '), + (Name.Label, u'aLabel'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testIncludeDirectives(self): + for character in ['"', '|']: + fragment = '#include/**/' + character + 'file.udo' + character + tokens = [ + (Comment.Preproc, u'#include'), + (Comment.Multiline, u'/**/'), + (String, character + u'file.udo' + character), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testObjectLikeMacroDefinitions(self): + fragment = dedent('''\ + # \tdefine MACRO#macro_body# + #define/**/ + MACRO/**/ + #\\#macro + body\\## + ''') + tokens = [ + (Comment.Preproc, u'# \tdefine'), + (Text, u' '), + (Comment.Preproc, u'MACRO'), + (Punctuation, u'#'), + (Comment.Preproc, u'macro_body'), + (Punctuation, u'#'), + (Text, u'\n'), + (Comment.Preproc, u'#define'), + (Comment.Multiline, u'/**/'), + (Text, u'\n'), + (Comment.Preproc, u'MACRO'), + (Comment.Multiline, u'/**/'), + (Text, u'\n'), + (Punctuation, u'#'), + (Comment.Preproc, u'\\#'), + (Comment.Preproc, u'macro\nbody'), + (Comment.Preproc, u'\\#'), + (Punctuation, u'#'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testFunctionLikeMacroDefinitions(self): + fragment = dedent('''\ + #define MACRO(ARG1#ARG2) #macro_body# + #define/**/ + MACRO(ARG1'ARG2' ARG3)/**/ + #\\#macro + body\\## + ''') + tokens = [ + (Comment.Preproc, u'#define'), + (Text, u' '), + (Comment.Preproc, u'MACRO'), + (Punctuation, u'('), + (Comment.Preproc, u'ARG1'), + (Punctuation, u'#'), + (Comment.Preproc, u'ARG2'), + (Punctuation, u')'), + (Text, u' '), + (Punctuation, u'#'), + (Comment.Preproc, u'macro_body'), + (Punctuation, u'#'), + (Text, u'\n'), + (Comment.Preproc, u'#define'), + (Comment.Multiline, u'/**/'), + (Text, u'\n'), + (Comment.Preproc, u'MACRO'), + (Punctuation, u'('), + (Comment.Preproc, u'ARG1'), + (Punctuation, u"'"), + (Comment.Preproc, u'ARG2'), + (Punctuation, u"'"), + (Text, u' '), + (Comment.Preproc, u'ARG3'), + (Punctuation, u')'), + (Comment.Multiline, u'/**/'), + (Text, u'\n'), + (Punctuation, u'#'), + (Comment.Preproc, u'\\#'), + (Comment.Preproc, u'macro\nbody'), + (Comment.Preproc, u'\\#'), + (Punctuation, u'#'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testMacroPreprocessorDirectives(self): + for directive in ['#ifdef', '#ifndef', '#undef']: + fragment = directive + ' MACRO' + tokens = [ + (Comment.Preproc, directive), + (Text, u' '), + (Comment.Preproc, u'MACRO'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testOtherPreprocessorDirectives(self): + fragment = dedent('''\ + #else + #end + #endif + ### + @ \t12345 + @@ \t67890 + ''') + tokens = [ + (Comment.Preproc, u'#else'), + (Text, u'\n'), + (Comment.Preproc, u'#end'), + (Text, u'\n'), + (Comment.Preproc, u'#endif'), + (Text, u'\n'), + (Comment.Preproc, u'###'), + (Text, u'\n'), + (Comment.Preproc, u'@ \t12345'), + (Text, u'\n'), + (Comment.Preproc, u'@@ \t67890'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testFunctionLikeMacros(self): + fragment = "$MACRO.(((x#y\\)))' \"(#'x)\\)x\\))\"# {{x\\))x)\\)(#'}});" + tokens = [ + (Comment.Preproc, u'$MACRO.'), + (Punctuation, u'('), + (Comment.Preproc, u'('), + (Comment.Preproc, u'('), + (Comment.Preproc, u'x#y\\)'), + (Comment.Preproc, u')'), + (Comment.Preproc, u')'), + (Punctuation, u"'"), + (Comment.Preproc, u' '), + (String, u'"'), + (Error, u'('), + (Error, u'#'), + (Error, u"'"), + (String, u'x'), + (Error, u')'), + (Comment.Preproc, u'\\)'), + (String, u'x'), + (Comment.Preproc, u'\\)'), + (Error, u')'), + (String, u'"'), + (Punctuation, u'#'), + (Comment.Preproc, u' '), + (String, u'{{'), + (String, u'x'), + (Comment.Preproc, u'\\)'), + (Error, u')'), + (String, u'x'), + (Error, u')'), + (Comment.Preproc, u'\\)'), + (Error, u'('), + (Error, u'#'), + (Error, u"'"), + (String, u'}}'), + (Punctuation, u')'), + (Comment.Single, u';'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testName(self): + fragment = 'kG:V' + tokens = [ + (Keyword.Type, 'k'), + (Name, 'G'), + (Punctuation, ':'), + (Name, 'V'), + (Text, '\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_data.py b/tests/test_data.py index be371419..20e74be0 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -9,7 +9,7 @@ import unittest -from pygments.lexers import JsonLexer, JsonBareObjectLexer +from pygments.lexers import JsonLexer, JsonBareObjectLexer, YamlLexer from pygments.token import Token @@ -98,3 +98,20 @@ class JsonBareObjectTest(unittest.TestCase): ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) +class YamlTest(unittest.TestCase): + def setUp(self): + self.lexer = YamlLexer() + + def testColonInComment(self): + # Bug #1528: This previously parsed 'token # innocent' as a tag + fragment = u'here: token # innocent: comment\n' + tokens = [ + (Token.Name.Tag, u'here'), + (Token.Punctuation, u':'), + (Token.Text, u' '), + (Token.Literal.Scalar.Plain, u'token'), + (Token.Text, u' '), + (Token.Comment.Single, u'# innocent: comment'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py index 2fae1125..e208403b 100644 --- a/tests/test_examplefiles.py +++ b/tests/test_examplefiles.py @@ -3,7 +3,7 @@ Pygments tests with example files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_html_formatter.py b/tests/test_html_formatter.py index 79990edd..37efd6f0 100644 --- a/tests/test_html_formatter.py +++ b/tests/test_html_formatter.py @@ -3,7 +3,7 @@ Pygments HTML formatter tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -100,7 +100,7 @@ class HtmlFormatterTest(unittest.TestCase): fmt = HtmlFormatter(**optdict) fmt.format(tokensource, outfile) html = outfile.getvalue() - self.assertTrue(re.search("<pre>\s+1\s+2\s+3", html)) + self.assertTrue(re.search(r"<pre>\s+1\s+2\s+3", html)) def test_linenos_with_startnum(self): optdict = dict(linenos=True, linenostart=5) @@ -108,7 +108,7 @@ class HtmlFormatterTest(unittest.TestCase): fmt = HtmlFormatter(**optdict) fmt.format(tokensource, outfile) html = outfile.getvalue() - self.assertTrue(re.search("<pre>\s+5\s+6\s+7", html)) + self.assertTrue(re.search(r"<pre>\s+5\s+6\s+7", html)) def test_lineanchors(self): optdict = dict(lineanchors="foo") @@ -132,9 +132,8 @@ class HtmlFormatterTest(unittest.TestCase): outencoding='utf-8') handle, pathname = tempfile.mkstemp('.html') - tfile = os.fdopen(handle, 'w+b') - fmt.format(tokensource, tfile) - tfile.close() + with os.fdopen(handle, 'w+b') as tfile: + fmt.format(tokensource, tfile) catname = os.path.join(TESTDIR, 'dtds', 'HTML4.soc') try: import subprocess @@ -173,9 +172,8 @@ class HtmlFormatterTest(unittest.TestCase): cssstyles=u'div:before { content: \'bäz\' }', encoding='utf-8') handle, pathname = tempfile.mkstemp('.html') - tfile = os.fdopen(handle, 'w+b') - fmt.format(tokensource, tfile) - tfile.close() + with os.fdopen(handle, 'w+b') as tfile: + fmt.format(tokensource, tfile) def test_ctags(self): try: diff --git a/tests/test_inherit.py b/tests/test_inherit.py index 5da57dd9..38acf328 100644 --- a/tests/test_inherit.py +++ b/tests/test_inherit.py @@ -3,7 +3,7 @@ Tests for inheritance in RegexLexer ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_irc_formatter.py b/tests/test_irc_formatter.py index 3b34f0bc..18bcd58b 100644 --- a/tests/test_irc_formatter.py +++ b/tests/test_irc_formatter.py @@ -3,7 +3,7 @@ Pygments IRC formatter tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_java.py b/tests/test_java.py index 6e5e8992..5f520853 100644 --- a/tests/test_java.py +++ b/tests/test_java.py @@ -3,7 +3,7 @@ Basic JavaLexer Test ~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_javascript.py b/tests/test_javascript.py index 21dff7c4..a2dfb7e1 100644 --- a/tests/test_javascript.py +++ b/tests/test_javascript.py @@ -3,7 +3,7 @@ Javascript tests ~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_julia.py b/tests/test_julia.py index ed46f27e..eda04b1c 100644 --- a/tests/test_julia.py +++ b/tests/test_julia.py @@ -3,7 +3,7 @@ Julia Tests ~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_kotlin.py b/tests/test_kotlin.py new file mode 100644 index 00000000..417d0d9f --- /dev/null +++ b/tests/test_kotlin.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +""" + Basic JavaLexer Test + ~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest + +from pygments.token import Text, Name, Operator, Keyword, Number, Punctuation, String +from pygments.lexers import KotlinLexer + +class KotlinTest(unittest.TestCase): + + def setUp(self): + self.lexer = KotlinLexer() + self.maxDiff = None + + def testCanCopeWithBackTickNamesInFunctions(self): + fragment = u'fun `wo bble`' + tokens = [ + (Keyword, u'fun'), + (Text, u' '), + (Name.Function, u'`wo bble`'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testCanCopeWithCommasAndDashesInBackTickNames(self): + fragment = u'fun `wo,-bble`' + tokens = [ + (Keyword, u'fun'), + (Text, u' '), + (Name.Function, u'`wo,-bble`'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testCanCopeWithDestructuring(self): + fragment = u'val (a, b) = ' + tokens = [ + (Keyword, u'val'), + (Text, u' '), + (Punctuation, u'('), + (Name.Property, u'a'), + (Punctuation, u','), + (Text, u' '), + (Name.Property, u'b'), + (Punctuation, u')'), + (Text, u' '), + (Punctuation, u'='), + (Text, u' '), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testCanCopeGenericsInDestructuring(self): + fragment = u'val (a: List<Something>, b: Set<Wobble>) =' + tokens = [ + (Keyword, u'val'), + (Text, u' '), + (Punctuation, u'('), + (Name.Property, u'a'), + (Punctuation, u':'), + (Text, u' '), + (Name.Property, u'List'), + (Punctuation, u'<'), + (Name, u'Something'), + (Punctuation, u'>'), + (Punctuation, u','), + (Text, u' '), + (Name.Property, u'b'), + (Punctuation, u':'), + (Text, u' '), + (Name.Property, u'Set'), + (Punctuation, u'<'), + (Name, u'Wobble'), + (Punctuation, u'>'), + (Punctuation, u')'), + (Text, u' '), + (Punctuation, u'='), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testCanCopeWithGenerics(self): + fragment = u'inline fun <reified T : ContractState> VaultService.queryBy(): Vault.Page<T> {' + tokens = [ + (Keyword, u'inline fun'), + (Text, u' '), + (Punctuation, u'<'), + (Keyword, u'reified'), + (Text, u' '), + (Name, u'T'), + (Text, u' '), + (Punctuation, u':'), + (Text, u' '), + (Name, u'ContractState'), + (Punctuation, u'>'), + (Text, u' '), + (Name.Class, u'VaultService'), + (Punctuation, u'.'), + (Name.Function, u'queryBy'), + (Punctuation, u'('), + (Punctuation, u')'), + (Punctuation, u':'), + (Text, u' '), + (Name, u'Vault'), + (Punctuation, u'.'), + (Name, u'Page'), + (Punctuation, u'<'), + (Name, u'T'), + (Punctuation, u'>'), + (Text, u' '), + (Punctuation, u'{'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testShouldCopeWithMultilineComments(self): + fragment = u'"""\nthis\nis\na\ncomment"""' + tokens = [ + (String, u'"""\nthis\nis\na\ncomment"""'), + (Text, u'\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_latex_formatter.py b/tests/test_latex_formatter.py index ebed7964..aa4ac3bb 100644 --- a/tests/test_latex_formatter.py +++ b/tests/test_latex_formatter.py @@ -3,7 +3,7 @@ Pygments LaTeX formatter tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_lexers_other.py b/tests/test_lexers_other.py index 3716fb72..8d53c542 100644 --- a/tests/test_lexers_other.py +++ b/tests/test_lexers_other.py @@ -3,7 +3,7 @@ Tests for other lexers ~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import glob diff --git a/tests/test_markdown_lexer.py b/tests/test_markdown_lexer.py new file mode 100644 index 00000000..c143586a --- /dev/null +++ b/tests/test_markdown_lexer.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" + Pygments regex lexer tests + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import unittest + +from pygments.lexers.markup import MarkdownLexer + + +class SameTextTests(unittest.TestCase): + + lexer = MarkdownLexer() + + def assert_same_text(self, text): + """Show that lexed markdown does not remove any content. """ + tokens = list(self.lexer.get_tokens_unprocessed(text)) + output = ''.join(t[2] for t in tokens) + self.assertEqual(text, output) + + def test_code_fence(self): + self.assert_same_text(r'```\nfoo\n```\n') + + def test_code_fence_gsm(self): + self.assert_same_text(r'```markdown\nfoo\n```\n') + + def test_code_fence_gsm_with_no_lexer(self): + self.assert_same_text(r'```invalid-lexer\nfoo\n```\n') diff --git a/tests/test_modeline.py b/tests/test_modeline.py index efe038df..6e1f16a4 100644 --- a/tests/test_modeline.py +++ b/tests/test_modeline.py @@ -3,7 +3,7 @@ Tests for the vim modeline feature ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_objectiveclexer.py b/tests/test_objectiveclexer.py index aee7db66..3db6a9e0 100644 --- a/tests/test_objectiveclexer.py +++ b/tests/test_objectiveclexer.py @@ -3,7 +3,7 @@ Basic CLexer Test ~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -69,7 +69,7 @@ class ObjectiveCLexerTest(unittest.TestCase): ] self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) - def testLieralNumberBoolExpression(self): + def testLiteralNumberBoolExpression(self): fragment = u'@(YES);\n' expected = [ (Token.Literal, u'@('), @@ -79,3 +79,14 @@ class ObjectiveCLexerTest(unittest.TestCase): (Token.Text, u'\n'), ] self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) + + def testModuleImport(self): + fragment = u'@import ModuleA;\n' + expected = [ + (Token.Keyword, u'@import'), + (Token.Text, u' '), + (Token.Name, u'ModuleA'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + ] + self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_perllexer.py b/tests/test_perllexer.py index 102f0a9f..30f9eca8 100644 --- a/tests/test_perllexer.py +++ b/tests/test_perllexer.py @@ -3,7 +3,7 @@ Pygments regex lexer tests ~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_php.py b/tests/test_php.py index b4117381..bb047b91 100644 --- a/tests/test_php.py +++ b/tests/test_php.py @@ -3,7 +3,7 @@ PHP Tests ~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_praat.py b/tests/test_praat.py index 1ca97d1e..9bf3ce76 100644 --- a/tests/test_praat.py +++ b/tests/test_praat.py @@ -3,7 +3,7 @@ Praat lexer tests ~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_properties.py b/tests/test_properties.py index 562778ba..aaa8ce29 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -3,7 +3,7 @@ Properties Tests ~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_python.py b/tests/test_python.py index e99687a6..b9c6c49b 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -3,7 +3,7 @@ Python Tests ~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -111,3 +111,23 @@ class Python3Test(unittest.TestCase): (Token.Text, u'\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def test_pep_515(self): + """ + Tests that the lexer can parse numeric literals with underscores + """ + fragments = ( + (Token.Literal.Number.Integer, u'1_000_000'), + (Token.Literal.Number.Float, u'1_000.000_001'), + (Token.Literal.Number.Float, u'1_000e1_000j'), + (Token.Literal.Number.Hex, u'0xCAFE_F00D'), + (Token.Literal.Number.Bin, u'0b_0011_1111_0100_1110'), + (Token.Literal.Number.Oct, u'0o_777_123'), + ) + + for token, fragment in fragments: + tokens = [ + (token, fragment), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_qbasiclexer.py b/tests/test_qbasiclexer.py index 0ea221a1..f40b8b63 100644 --- a/tests/test_qbasiclexer.py +++ b/tests/test_qbasiclexer.py @@ -3,7 +3,7 @@ Tests for QBasic ~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_r.py b/tests/test_r.py new file mode 100644 index 00000000..70148e53 --- /dev/null +++ b/tests/test_r.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +""" + R Tests + ~~~~~~~~~ + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest + +from pygments.lexers import SLexer +from pygments.token import Token, Name, Punctuation + + +class RTest(unittest.TestCase): + def setUp(self): + self.lexer = SLexer() + + def testCall(self): + fragment = u'f(1, a)\n' + tokens = [ + (Name.Function, u'f'), + (Punctuation, u'('), + (Token.Literal.Number, u'1'), + (Punctuation, u','), + (Token.Text, u' '), + (Token.Name, u'a'), + (Punctuation, u')'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testName1(self): + fragment = u'._a_2.c' + tokens = [ + (Name, u'._a_2.c'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testName2(self): + # Invalid names are valid if backticks are used + fragment = u'`.1 blah`' + tokens = [ + (Name, u'`.1 blah`'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testName3(self): + # Internal backticks can be escaped + fragment = u'`.1 \\` blah`' + tokens = [ + (Name, u'`.1 \\` blah`'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testCustomOperator(self): + fragment = u'7 % and % 8' + tokens = [ + (Token.Literal.Number, u'7'), + (Token.Text, u' '), + (Token.Operator, u'% and %'), + (Token.Text, u' '), + (Token.Literal.Number, u'8'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_regexlexer.py b/tests/test_regexlexer.py index d919a950..adc05a93 100644 --- a/tests/test_regexlexer.py +++ b/tests/test_regexlexer.py @@ -3,7 +3,7 @@ Pygments regex lexer tests ~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -11,7 +11,6 @@ import unittest from pygments.token import Text from pygments.lexer import RegexLexer -from pygments.lexer import bygroups from pygments.lexer import default @@ -21,6 +20,8 @@ class TestLexer(RegexLexer): 'root': [ ('a', Text.Root, 'rag'), ('e', Text.Root), + ('#', Text.Root, '#pop'), + ('@', Text.Root, ('#pop', '#pop')), default(('beer', 'beer')) ], 'beer': [ @@ -37,18 +38,29 @@ class TupleTransTest(unittest.TestCase): def test(self): lx = TestLexer() toks = list(lx.get_tokens_unprocessed('abcde')) - self.assertEqual(toks, - [(0, Text.Root, 'a'), (1, Text.Rag, 'b'), (2, Text.Rag, 'c'), + self.assertEqual(toks, [ + (0, Text.Root, 'a'), (1, Text.Rag, 'b'), (2, Text.Rag, 'c'), (3, Text.Beer, 'd'), (4, Text.Root, 'e')]) def test_multiline(self): lx = TestLexer() toks = list(lx.get_tokens_unprocessed('a\ne')) - self.assertEqual(toks, - [(0, Text.Root, 'a'), (1, Text, u'\n'), - (2, Text.Root, 'e')]) + self.assertEqual(toks, [ + (0, Text.Root, 'a'), (1, Text, u'\n'), (2, Text.Root, 'e')]) def test_default(self): lx = TestLexer() toks = list(lx.get_tokens_unprocessed('d')) self.assertEqual(toks, [(0, Text.Beer, 'd')]) + + +class PopEmptyTest(unittest.TestCase): + def test_regular(self): + lx = TestLexer() + toks = list(lx.get_tokens_unprocessed('#e')) + self.assertEqual(toks, [(0, Text.Root, '#'), (1, Text.Root, 'e')]) + + def test_tuple(self): + lx = TestLexer() + toks = list(lx.get_tokens_unprocessed('@e')) + self.assertEqual(toks, [(0, Text.Root, '@'), (1, Text.Root, 'e')]) diff --git a/tests/test_regexopt.py b/tests/test_regexopt.py index 5cfb62a3..9c44f498 100644 --- a/tests/test_regexopt.py +++ b/tests/test_regexopt.py @@ -3,7 +3,7 @@ Tests for pygments.regexopt ~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_rtf_formatter.py b/tests/test_rtf_formatter.py index 756c03a9..80ce01f5 100644 --- a/tests/test_rtf_formatter.py +++ b/tests/test_rtf_formatter.py @@ -3,7 +3,7 @@ Pygments RTF formatter tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -80,7 +80,7 @@ class RtfFormatterTest(StringTests, unittest.TestCase): self.assertEndsWith(result, expected+self.foot, msg) def test_escape_characters(self): - t = u'\ {{' + t = u'\\ {{' result = self.format_rtf(t) expected = (r'\\ \{\{') if not result.endswith(self.foot): diff --git a/tests/test_ruby.py b/tests/test_ruby.py index b7d4110a..45a77469 100644 --- a/tests/test_ruby.py +++ b/tests/test_ruby.py @@ -3,7 +3,7 @@ Basic RubyLexer Test ~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_shell.py b/tests/test_shell.py index e283793e..6b24eb43 100644 --- a/tests/test_shell.py +++ b/tests/test_shell.py @@ -3,14 +3,14 @@ Basic Shell Tests ~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import unittest from pygments.token import Token -from pygments.lexers import BashLexer, BashSessionLexer +from pygments.lexers import BashLexer, BashSessionLexer, MSDOSSessionLexer class BashTest(unittest.TestCase): @@ -140,3 +140,20 @@ class BashSessionTest(unittest.TestCase): ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) +class MSDOSSessionTest(unittest.TestCase): + + def setUp(self): + self.lexer = MSDOSSessionLexer() + + def testGtOnlyPrompt(self): + fragment = u'> py\nhi\n' + tokens = [ + (Token.Text, u''), + (Token.Generic.Prompt, u'>'), + (Token.Text, u' '), + (Token.Text, u'py'), + (Token.Text, u''), + (Token.Text, u'\n'), + (Token.Generic.Output, u'hi\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_smarty.py b/tests/test_smarty.py index e1e079d9..fb15f7f4 100644 --- a/tests/test_smarty.py +++ b/tests/test_smarty.py @@ -3,7 +3,7 @@ Basic SmartyLexer Test ~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_sql.py b/tests/test_sql.py index c5f5c758..6be34006 100644 --- a/tests/test_sql.py +++ b/tests/test_sql.py @@ -8,7 +8,10 @@ """ import unittest -from pygments.lexers.sql import TransactSqlLexer +from pygments.lexers.sql import name_between_bracket_re, \ + name_between_backtick_re, tsql_go_re, tsql_declare_re, \ + tsql_variable_re, MySqlLexer, SqlLexer, TransactSqlLexer + from pygments.token import Comment, Name, Number, Punctuation, Whitespace @@ -72,3 +75,44 @@ class TransactSqlLexerTest(unittest.TestCase): (Comment.Multiline, '*/'), (Comment.Multiline, '*/'), )) + + +class SqlAnalyzeTextTest(unittest.TestCase): + def test_can_match_analyze_text_res(self): + self.assertEqual(['`a`', '`bc`'], + name_between_backtick_re.findall('select `a`, `bc` from some')) + self.assertEqual(['[a]', '[bc]'], + name_between_bracket_re.findall('select [a], [bc] from some')) + self.assertTrue(tsql_declare_re.search('--\nDeClaRe @some int;')) + self.assertTrue(tsql_go_re.search('select 1\ngo\n--')) + self.assertTrue(tsql_variable_re.search( + 'create procedure dbo.usp_x @a int, @b int')) + + def test_can_analyze_text(self): + mysql_lexer = MySqlLexer() + sql_lexer = SqlLexer() + tsql_lexer = TransactSqlLexer() + code_to_expected_lexer_map = { + 'select `a`, `bc` from some': mysql_lexer, + 'select a, bc from some': sql_lexer, + 'select [a], [bc] from some': tsql_lexer, + '-- `a`, `bc`\nselect [a], [bc] from some': tsql_lexer, + '-- `a`, `bc`\nselect [a], [bc] from some; go': tsql_lexer, + } + sql_lexers = set(code_to_expected_lexer_map.values()) + for code, expected_lexer in code_to_expected_lexer_map.items(): + ratings_and_lexers = list((lexer.analyse_text(code), lexer.name) for lexer in sql_lexers) + best_rating, best_lexer_name = sorted(ratings_and_lexers, reverse=True)[0] + expected_rating = expected_lexer.analyse_text(code) + message = ( + 'lexer must be %s (rating %.2f) instead of ' + '%s (rating %.2f) for analyse_text() on code:\n%s') % ( + expected_lexer.name, + expected_rating, + best_lexer_name, + best_rating, + code + ) + self.assertEqual( + expected_lexer.name, best_lexer_name, message + ) diff --git a/tests/test_string_asserts.py b/tests/test_string_asserts.py index 5e9e5617..737ba200 100644 --- a/tests/test_string_asserts.py +++ b/tests/test_string_asserts.py @@ -3,7 +3,7 @@ Pygments string assert utility tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_terminal_formatter.py b/tests/test_terminal_formatter.py index ee0ac380..e5a13431 100644 --- a/tests/test_terminal_formatter.py +++ b/tests/test_terminal_formatter.py @@ -3,7 +3,7 @@ Pygments terminal formatter tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -61,10 +61,10 @@ class TerminalFormatterTest(unittest.TestCase): class MyStyle(Style): styles = { - Token.Comment: '#ansidarkgray', - Token.String: '#ansiblue bg:#ansidarkred', - Token.Number: '#ansigreen bg:#ansidarkgreen', - Token.Number.Hex: '#ansidarkgreen bg:#ansired', + Token.Comment: 'ansibrightblack', + Token.String: 'ansibrightblue bg:ansired', + Token.Number: 'ansibrightgreen bg:ansigreen', + Token.Number.Hex: 'ansigreen bg:ansibrightred', } @@ -90,13 +90,13 @@ async def function(a,b,c, *d, **kwarg:Bool)->Bool: def test_256esc_seq(self): """ - test that a few escape sequences are actualy used when using #ansi<> color codes + test that a few escape sequences are actually used when using ansi<> color codes """ def termtest(x): return highlight(x, Python3Lexer(), Terminal256Formatter(style=MyStyle)) - self.assertTrue('32;41' in termtest('0x123')) - self.assertTrue('32;42' in termtest('123')) - self.assertTrue('30;01' in termtest('#comment')) - self.assertTrue('34;41' in termtest('"String"')) + self.assertTrue('32;101' in termtest('0x123')) + self.assertTrue('92;42' in termtest('123')) + self.assertTrue('90' in termtest('#comment')) + self.assertTrue('94;41' in termtest('"String"')) diff --git a/tests/test_textfmts.py b/tests/test_textfmts.py index 453dd61f..8a1b8eda 100644 --- a/tests/test_textfmts.py +++ b/tests/test_textfmts.py @@ -3,7 +3,7 @@ Basic Tests for textfmts ~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_token.py b/tests/test_token.py index 94522373..fdbcabd1 100644 --- a/tests/test_token.py +++ b/tests/test_token.py @@ -3,7 +3,7 @@ Test suite for the token module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_unistring.py b/tests/test_unistring.py index c56b68c7..82d74ed6 100644 --- a/tests/test_unistring.py +++ b/tests/test_unistring.py @@ -3,7 +3,7 @@ Test suite for the unistring module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_using_api.py b/tests/test_using_api.py index 7517ce7d..2ab70d09 100644 --- a/tests/test_using_api.py +++ b/tests/test_using_api.py @@ -3,7 +3,7 @@ Pygments tests for using() ~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_util.py b/tests/test_util.py index cdb58b3f..646a403b 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -3,7 +3,7 @@ Test suite for the util module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py33, py34 +envlist = py27, py35, py36, py37 [testenv] deps = nose |