diff options
| author | Georg Brandl <georg@python.org> | 2017-01-25 07:58:04 +0100 |
|---|---|---|
| committer | Georg Brandl <georg@python.org> | 2017-01-25 07:58:04 +0100 |
| commit | 7f1e0a5c97eb1ce6d9108d1d03fb648565830c14 (patch) | |
| tree | 9a2991e6452e179facc462b5af33a4f628ddab9a | |
| parent | 8bfdb2f31a732967823c183749d7bf901a22c033 (diff) | |
| parent | 0348972c229aee63d69921282455885489a75722 (diff) | |
| download | pygments-git-7f1e0a5c97eb1ce6d9108d1d03fb648565830c14.tar.gz | |
after release 2.2: merge default *into* stable
314 files changed, 22683 insertions, 2878 deletions
@@ -7,7 +7,7 @@ Other contributors, listed alphabetically, are: * Sam Aaron -- Ioke lexer * Ali Afshar -- image formatter -* Thomas Aglassinger -- Easytrieve, JCL and Rexx lexers +* Thomas Aglassinger -- Easytrieve, JCL, Rexx and Transact-SQL lexers * Muthiah Annamalai -- Ezhil lexer * Kumar Appaiah -- Debian control lexer * Andreas Amann -- AppleScript lexer @@ -33,10 +33,12 @@ Other contributors, listed alphabetically, are: * Adam Blinkinsop -- Haskell, Redcode lexers * Frits van Bommel -- assembler lexers * Pierre Bourdon -- bugfixes +* Matthias Bussonnier -- ANSI style handling for terminal-256 formatter * chebee7i -- Python traceback lexer improvements * Hiram Chirino -- Scaml and Jade lexers +* Mauricio Caceres -- SAS and Stata lexers. * Ian Cooper -- VGL lexer -* David Corbett -- Inform, Jasmin, and TADS 3 lexers +* David Corbett -- Inform, Jasmin, JSGF, Snowball, and TADS 3 lexers * Leaf Corcoran -- MoonScript lexer * Christopher Creutzig -- MuPAD lexer * Daniël W. Crompton -- Pike lexer @@ -64,6 +66,7 @@ Other contributors, listed alphabetically, are: * Alex Gilding -- BlitzBasic lexer * Bertrand Goetzmann -- Groovy lexer * Krzysiek Goj -- Scala lexer +* Andrey Golovizin -- BibTeX lexers * Matt Good -- Genshi, Cheetah lexers * Michał Górny -- vim modeline support * Alex Gosse -- TrafficScript lexer @@ -88,7 +91,8 @@ Other contributors, listed alphabetically, are: * Tim Howard -- BlitzMax lexer * Dustin Howett -- Logos lexer * Ivan Inozemtsev -- Fantom lexer -* Hiroaki Itoh -- Shell console rewrite, Lexers for PowerShell session, MSDOS session, BC +* Hiroaki Itoh -- Shell console rewrite, Lexers for PowerShell session, + MSDOS session, BC, WDiff * Brian R. Jackson -- Tea lexer * Christian Jann -- ShellSession lexer * Dennis Kaarsemaker -- sources.list lexer @@ -110,7 +114,9 @@ Other contributors, listed alphabetically, are: * Jon Larimer, Google Inc. -- Smali lexer * Olov Lassus -- Dart lexer * Matt Layman -- TAP lexer +* Kristian Lyngstøl -- Varnish lexers * Sylvestre Ledru -- Scilab lexer +* Chee Sing Lee -- Flatline lexer * Mark Lee -- Vala lexer * Valentin Lorentz -- C++ lexer improvements * Ben Mabey -- Gherkin lexer @@ -135,6 +141,7 @@ Other contributors, listed alphabetically, are: * Mher Movsisyan -- DTD lexer * Dejan Muhamedagic -- Crmsh lexer * Ana Nelson -- Ragel, ANTLR, R console lexers +* Kurt Neufeld -- Markdown lexer * Nam T. Nguyen -- Monokai style * Jesper Noehr -- HTML formatter "anchorlinenos" * Mike Nolta -- Julia lexer @@ -148,6 +155,8 @@ Other contributors, listed alphabetically, are: * Dominik Picheta -- Nimrod lexer * Andrew Pinkham -- RTF Formatter Refactoring * Clément Prévost -- UrbiScript lexer +* Tanner Prynn -- cmdline -x option and loading lexers from files +* Oleh Prypin -- Crystal lexer (based on Ruby lexer) * Elias Rabel -- Fortran fixed form lexer * raichoo -- Idris lexer * Kashif Rasul -- CUDA lexer @@ -156,6 +165,8 @@ Other contributors, listed alphabetically, are: * Corey Richardson -- Rust lexer updates * Lubomir Rintel -- GoodData MAQL and CL lexers * Andre Roberge -- Tango style +* Georg Rollinger -- HSAIL lexer +* Michiel Roos -- TypoScript lexer * Konrad Rudolph -- LaTeX formatter enhancements * Mario Ruggier -- Evoque lexers * Miikka Salminen -- Lovelace style, Hexdump lexer, lexer enhancements @@ -163,22 +174,28 @@ Other contributors, listed alphabetically, are: * Matteo Sasso -- Common Lisp lexer * Joe Schafer -- Ada lexer * Ken Schutte -- Matlab lexers +* René Schwaiger -- Rainbow Dash style +* Sebastian Schweizer -- Whiley lexer * Tassilo Schweyer -- Io, MOOCode lexers * Ted Shaw -- AutoIt lexer * Joerg Sieker -- ABAP lexer * Robert Simmons -- Standard ML lexer * Kirill Simonov -- YAML lexer +* Corbin Simpson -- Monte lexer * Alexander Smishlajev -- Visual FoxPro lexer * Steve Spigarelli -- XQuery lexer * Jerome St-Louis -- eC lexer +* Camil Staps -- Clean and NuSMV lexers * James Strachan -- Kotlin lexer * Tom Stuart -- Treetop lexer * Colin Sullivan -- SuperCollider lexer +* Ben Swift -- Extempore lexer * Edoardo Tenani -- Arduino lexer * Tiberius Teng -- default style overhaul * Jeremy Thurgood -- Erlang, Squid config lexers * Brian Tiffin -- OpenCOBOL lexer * Bob Tolbert -- Hy lexer +* Matthias Trute -- Forth lexer * Erick Tryzelaar -- Felix lexer * Alexander Udalov -- Kotlin lexer improvements * Thomas Van Doren -- Chapel lexer @@ -186,6 +203,7 @@ Other contributors, listed alphabetically, are: * Abe Voelker -- OpenEdge ABL lexer * Pepijn de Vos -- HTML formatter CTags support * Matthias Vallentin -- Bro lexer +* Benoît Vinot -- AMPL lexer * Linh Vu Hong -- RSL lexer * Nathan Weizenbaum -- Haml and Sass lexers * Nathan Whetsell -- Csound lexers @@ -6,6 +6,57 @@ Issue numbers refer to the tracker at pull request numbers to the requests at <https://bitbucket.org/birkenfeld/pygments-main/pull-requests/merged>. +Version 2.2.0 +------------- +(release Jan 22, 2017) + +- Added lexers: + + * AMPL + * TypoScript (#1173) + * Varnish config (PR#554) + * Clean (PR#503) + * WDiff (PR#513) + * Flatline (PR#551) + * Silver (PR#537) + * HSAIL (PR#518) + * JSGF (PR#546) + * NCAR command language (PR#536) + * Extempore (PR#530) + * Cap'n Proto (PR#595) + * Whiley (PR#573) + * Monte (PR#592) + * Crystal (PR#576) + * Snowball (PR#589) + * CapDL (PR#579) + * NuSMV (PR#564) + * SAS, Stata (PR#593) + +- Added the ability to load lexer and formatter classes directly from files + with the `-x` command line option and the `lexers.load_lexer_from_file()` + and `formatters.load_formatter_from_file()` functions. (PR#559) + +- Added `lexers.find_lexer_class_by_name()`. (#1203) + +- Added new token types and lexing for magic methods and variables in Python + and PHP. + +- Added a new token type for string affixes and lexing for them in Python, C++ + and Postgresql lexers. + +- Added a new token type for heredoc (and similar) string delimiters and + lexing for them in C++, Perl, PHP, Postgresql and Ruby lexers. + +- Styles can now define colors with ANSI colors for use in the 256-color + terminal formatter. (PR#531) + +- Improved the CSS lexer. (#1083, #1130) + +- Added "Rainbow Dash" style. (PR#623) + +- Delay loading `pkg_resources`, which takes a long while to import. (PR#690) + + Version 2.1.3 ------------- (released Mar 2, 2016) @@ -1,4 +1,4 @@ -Copyright (c) 2006-2015 by the respective authors (see AUTHORS file). +Copyright (c) 2006-2017 by the respective authors (see AUTHORS file). All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/MANIFEST.in b/MANIFEST.in index cfec4e94..7e1d320d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ include pygmentize include external/* -include Makefile CHANGES LICENSE AUTHORS TODO ez_setup.py +include Makefile CHANGES LICENSE AUTHORS TODO recursive-include tests * recursive-include doc * recursive-include scripts * @@ -4,7 +4,7 @@ # # Combines scripts for common tasks. # -# :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. +# :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. # :license: BSD, see LICENSE for details. # @@ -55,6 +55,9 @@ test: test-coverage: @$(PYTHON) tests/run.py -d --with-coverage --cover-package=pygments --cover-erase $(TEST) +test-examplefiles: + nosetests tests/test_examplefiles.py + tox-test: @tox -- $(TEST) diff --git a/doc/_themes/pygments14/layout.html b/doc/_themes/pygments14/layout.html index 2cc03e03..e8860827 100644 --- a/doc/_themes/pygments14/layout.html +++ b/doc/_themes/pygments14/layout.html @@ -82,7 +82,7 @@ {% block footer %} <div class="footer" role="contentinfo"> - © Copyright 2006-2015, Georg Brandl and Pygments contributors. + © Copyright 2006-2017, Georg Brandl and Pygments contributors. Created using <a href="http://sphinx-doc.org/">Sphinx</a> {{ sphinx_version }}. <br/> Pygments logo created by <a href="http://joelunger.com">Joel Unger</a>. diff --git a/doc/_themes/pygments14/static/pygments14.css_t b/doc/_themes/pygments14/static/pygments14.css_t index 5c37aaf9..7f09f623 100644 --- a/doc/_themes/pygments14/static/pygments14.css_t +++ b/doc/_themes/pygments14/static/pygments14.css_t @@ -4,7 +4,7 @@ * * Sphinx stylesheet -- pygments14 theme. Heavily copied from sphinx13. * - * :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + * :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ diff --git a/doc/docs/api.rst b/doc/docs/api.rst index 123a4643..a6b242dd 100644 --- a/doc/docs/api.rst +++ b/doc/docs/api.rst @@ -62,6 +62,18 @@ Functions from :mod:`pygments.lexers`: Will raise :exc:`pygments.util.ClassNotFound` if not lexer for that mimetype is found. +.. function:: load_lexer_from_file(filename, lexername="CustomLexer", **options) + + Return a `Lexer` subclass instance loaded from the provided file, relative + to the current directory. The file is expected to contain a Lexer class + named `lexername` (by default, CustomLexer). Users should be very careful with + the input, because this method is equivalent to running eval on the input file. + The lexer is given the `options` at its instantiation. + + :exc:`ClassNotFound` is raised if there are any errors loading the Lexer + + .. versionadded:: 2.2 + .. function:: guess_lexer(text, **options) Return a `Lexer` subclass instance that's guessed from the text in @@ -89,6 +101,21 @@ Functions from :mod:`pygments.lexers`: .. versionadded:: 0.6 +.. function:: find_lexer_class_by_name(alias) + + Return the `Lexer` subclass that has `alias` in its aliases list, without + instantiating it. + + Will raise :exc:`pygments.util.ClassNotFound` if no lexer with that alias is + found. + + .. versionadded:: 2.2 + +.. function:: find_lexer_class(name) + + Return the `Lexer` subclass that with the *name* attribute as given by + the *name* argument. + .. module:: pygments.formatters @@ -110,6 +137,17 @@ Functions from :mod:`pygments.formatters`: Will raise :exc:`pygments.util.ClassNotFound` if no formatter for that filename is found. +.. function:: load_formatter_from_file(filename, formattername="CustomFormatter", **options) + + Return a `Formatter` subclass instance loaded from the provided file, relative + to the current directory. The file is expected to contain a Formatter class + named ``formattername`` (by default, CustomFormatter). Users should be very + careful with the input, because this method is equivalent to running eval + on the input file. The formatter is given the `options` at its instantiation. + + :exc:`ClassNotFound` is raised if there are any errors loading the Formatter + + .. versionadded:: 2.2 .. module:: pygments.styles diff --git a/doc/docs/cmdline.rst b/doc/docs/cmdline.rst index 165af969..e4f94ea5 100644 --- a/doc/docs/cmdline.rst +++ b/doc/docs/cmdline.rst @@ -99,6 +99,23 @@ The ``-N`` option guesses a lexer name for a given filename, so that :: will print out ``python``. It won't highlight anything yet. If no specific lexer is known for that filename, ``text`` is printed. +Custom Lexers and Formatters +---------------------------- + +.. versionadded:: 2.2 + +The ``-x`` flag enables custom lexers and formatters to be loaded +from files relative to the current directory. Create a file with a class named +CustomLexer or CustomFormatter, then specify it on the command line:: + + $ pygmentize -l your_lexer.py -f your_formatter.py -x + +You can also specify the name of your class with a colon:: + + $ pygmentize -l your_lexer.py:SomeLexer -x + +For more information, see :doc:`the Pygments documentation on Lexer development +<lexerdevelopment>`. Getting help ------------ diff --git a/doc/docs/lexerdevelopment.rst b/doc/docs/lexerdevelopment.rst index 2c868440..63bd01a3 100644 --- a/doc/docs/lexerdevelopment.rst +++ b/doc/docs/lexerdevelopment.rst @@ -88,10 +88,55 @@ one. Adding and testing a new lexer ============================== -To make Pygments aware of your new lexer, you have to perform the following -steps: +The easiest way to use a new lexer is to use Pygments' support for loading +the lexer from a file relative to your current directory. -First, change to the current directory containing the Pygments source code: +First, change the name of your lexer class to CustomLexer: + +.. code-block:: python + + from pygments.lexer import RegexLexer + from pygments.token import * + + class CustomLexer(RegexLexer): + """All your lexer code goes here!""" + +Then you can load the lexer from the command line with the additional +flag ``-x``: + +.. code-block:: console + + $ pygmentize -l your_lexer_file.py -x + +To specify a class name other than CustomLexer, append it with a colon: + +.. code-block:: console + + $ pygmentize -l your_lexer.py:SomeLexer -x + +Or, using the Python API: + +.. code-block:: python + + # For a lexer named CustomLexer + your_lexer = load_lexer_from_file(filename, **options) + + # For a lexer named MyNewLexer + your_named_lexer = load_lexer_from_file(filename, "MyNewLexer", **options) + +When loading custom lexers and formatters, be extremely careful to use only +trusted files; Pygments will perform the equivalent of ``eval`` on them. + +If you only want to use your lexer with the Pygments API, you can import and +instantiate the lexer yourself, then pass it to :func:`pygments.highlight`. + +To prepare your new lexer for inclusion in the Pygments distribution, so that it +will be found when passing filenames or lexer aliases from the command line, you +have to perform the following steps. + +First, change to the current directory containing the Pygments source code. You +will need to have either an unpacked source tarball, or (preferably) a copy +cloned from BitBucket. .. code-block:: console @@ -101,11 +146,13 @@ Select a matching module under ``pygments/lexers``, or create a new module for your lexer class. Next, make sure the lexer is known from outside of the module. All modules in -the ``pygments.lexers`` specify ``__all__``. For example, ``esoteric.py`` sets:: +the ``pygments.lexers`` package specify ``__all__``. For example, +``esoteric.py`` sets:: __all__ = ['BrainfuckLexer', 'BefungeLexer', ...] -Simply add the name of your lexer class to this list. +Add the name of your lexer class to this list (or create the list if your lexer +is the only class in the module). Finally the lexer can be made publicly known by rebuilding the lexer mapping: @@ -353,7 +400,7 @@ There are a few more things you can do with states: tokens = {...} def get_tokens_unprocessed(self, text, stack=('root', 'otherstate')): - for item in RegexLexer.get_tokens_unprocessed(text, stack): + for item in RegexLexer.get_tokens_unprocessed(self, text, stack): yield item Some lexers like the `PhpLexer` use this to make the leading ``<?php`` diff --git a/doc/docs/lexers.rst b/doc/docs/lexers.rst index 9262efb0..ef40f140 100644 --- a/doc/docs/lexers.rst +++ b/doc/docs/lexers.rst @@ -31,7 +31,7 @@ Currently, **all lexers** support these options: If this option is set to ``"guess"``, a simple UTF-8 vs. Latin-1 detection is used, if it is set to ``"chardet"``, the - `chardet library <http://chardet.feedparser.org/>`_ is used to + `chardet library <https://chardet.github.io/>`_ is used to guess the encoding of the input. .. versionadded:: 0.6 diff --git a/doc/docs/styles.rst b/doc/docs/styles.rst index d56db0db..1094a270 100644 --- a/doc/docs/styles.rst +++ b/doc/docs/styles.rst @@ -143,3 +143,59 @@ a way to iterate over all styles: >>> from pygments.styles import get_all_styles >>> styles = list(get_all_styles()) + + +.. _AnsiTerminalStyle: + +Terminal Styles +=============== + +.. versionadded:: 2.2 + +Custom styles used with the 256-color terminal formatter can also map colors to +use the 8 default ANSI colors. To do so, use ``#ansigreen``, ``#ansired`` or +any other colors defined in :attr:`pygments.style.ansicolors`. Foreground ANSI +colors will be mapped to the corresponding `escape codes 30 to 37 +<https://en.wikipedia.org/wiki/ANSI_escape_code#Colors>`_ thus respecting any +custom color mapping and themes provided by many terminal emulators. Light +variants are treated as foreground color with and an added bold flag. +``bg:#ansi<color>`` will also be respected, except the light variant will be the +same shade as their dark variant. + +See the following example where the color of the string ``"hello world"`` is +governed by the escape sequence ``\x1b[34;01m`` (Ansi Blue, Bold, 41 being red +background) instead of an extended foreground & background color. + +.. sourcecode:: pycon + + >>> from pygments import highlight + >>> from pygments.style import Style + >>> from pygments.token import Token + >>> from pygments.lexers import Python3Lexer + >>> from pygments.formatters import Terminal256Formatter + + >>> class MyStyle(Style): + styles = { + Token.String: '#ansiblue bg:#ansired', + } + + >>> code = 'print("Hello World")' + >>> result = highlight(code, Python3Lexer(), Terminal256Formatter(style=MyStyle)) + >>> print(result.encode()) + b'\x1b[34;41;01m"\x1b[39;49;00m\x1b[34;41;01mHello World\x1b[39;49;00m\x1b[34;41;01m"\x1b[39;49;00m' + +Colors specified using ``#ansi*`` are converted to a default set of RGB colors +when used with formatters other than the terminal-256 formatter. + +By definition of ANSI, the following colors are considered "light" colors, and +will be rendered by most terminals as bold: + +- "darkgray", "red", "green", "yellow", "blue", "fuchsia", "turquoise", "white" + +The following are considered "dark" colors and will be rendered as non-bold: + +- "black", "darkred", "darkgreen", "brown", "darkblue", "purple", "teal", + "lightgray" + +Exact behavior might depends on the terminal emulator you are using, and its +settings. diff --git a/doc/docs/tokens.rst b/doc/docs/tokens.rst index 6455a501..801fc638 100644 --- a/doc/docs/tokens.rst +++ b/doc/docs/tokens.rst @@ -174,6 +174,10 @@ Name Tokens `Name.Function` Token type for function names. +`Name.Function.Magic` + same as `Name.Function` but for special function names that have an implicit use + in a language (e.g. ``__init__`` method in Python). + `Name.Label` Token type for label names (e.g. in languages that support ``goto``). @@ -201,6 +205,10 @@ Name Tokens `Name.Variable.Instance` same as `Name.Variable` but for instance variables. +`Name.Variable.Magic` + same as `Name.Variable` but for special variable names that have an implicit use + in a language (e.g. ``__doc__`` in Python). + Literals ======== @@ -215,12 +223,20 @@ Literals `String` For any string literal. +`String.Affix` + Token type for affixes that further specify the type of the string they're + attached to (e.g. the prefixes ``r`` and ``u8`` in ``r"foo"`` and ``u8"foo"``). + `String.Backtick` Token type for strings enclosed in backticks. `String.Char` Token type for single characters (e.g. Java, C). +`String.Delimiter` + Token type for delimiting identifiers in "heredoc", raw and other similar + strings (e.g. the word ``END`` in Perl code ``print <<'END';``). + `String.Doc` Token type for documentation strings (for example Python). diff --git a/doc/docs/unicode.rst b/doc/docs/unicode.rst index 17853a36..dca91116 100644 --- a/doc/docs/unicode.rst +++ b/doc/docs/unicode.rst @@ -55,4 +55,4 @@ encoding is handled differently, see :doc:`the command line docs <cmdline>`. options dict with lexers and formatters, and still have different input and output encodings. -.. _chardet: http://chardet.feedparser.org/ +.. _chardet: https://chardet.github.io/ diff --git a/doc/languages.rst b/doc/languages.rst index a495d15c..7fa8eb2f 100644 --- a/doc/languages.rst +++ b/doc/languages.rst @@ -26,11 +26,13 @@ Programming languages * Common Lisp * Coq * Cryptol (incl. Literate Cryptol) +* `Crystal <http://crystal-lang.org>`_ * `Cython <http://cython.org>`_ * `D <http://dlang.org>`_ * Dart * Delphi * Dylan +* `Elm <http://elm-lang.org/>`_ * Erlang * `Ezhil <http://ezhillang.org>`_ Ezhil - A Tamil programming language * Factor diff --git a/external/autopygmentize b/external/autopygmentize index d2f969a1..d2d05970 100755 --- a/external/autopygmentize +++ b/external/autopygmentize @@ -1,6 +1,6 @@ #!/bin/bash # Best effort auto-pygmentization with transparent decompression -# by Reuben Thomas 2008-2015 +# by Reuben Thomas 2008-2016 # This program is in the public domain. # Strategy: first see if pygmentize can find a lexer; if not, ask file; if that finds nothing, fail @@ -25,6 +25,7 @@ if [[ "$lexer" == text ]]; then text/x-awk) lexer=awk;; text/x-c) lexer=c;; text/x-c++) lexer=cpp;; + text/x-crystal) lexer=crystal;; text/x-diff) lexer=diff;; text/x-fortran) lexer=fortran;; text/x-gawk) lexer=gawk;; @@ -65,19 +66,36 @@ if [[ "$lexer" == text ]]; then esac fi +# Find a preprocessor for compressed files +concat=cat +case $(file $file_common_opts --mime-type "$file") in + application/x-gzip) concat=zcat;; + application/x-bzip2) concat=bzcat;; + application/x-xz) concat=xzcat;; +esac + +# Find a suitable lexer, preceded by a hex dump for binary files +prereader="" encoding=$(file --mime-encoding --uncompress $file_common_opts "$file") -if [[ $encoding == "us-asciibinarybinary" ]]; then - encoding="us-ascii" +if [[ $encoding == "binary" ]]; then + prereader="od -x" # POSIX fallback + if [[ -n $(which hd) ]]; then + prereader="hd" # preferred + fi + lexer=hexdump + encoding=latin1 fi - if [[ -n "$lexer" ]]; then - concat=cat - case $(file $file_common_opts --mime-type "$file") in - application/x-gzip) concat=zcat;; - application/x-bzip2) concat=bzcat;; - application/x-xz) concat=xzcat;; - esac - exec $concat "$file" | pygmentize -O inencoding=$encoding $PYGMENTIZE_OPTS $options -l $lexer + reader="pygmentize -O inencoding=$encoding $PYGMENTIZE_OPTS $options -l $lexer" +fi + +# If we found a reader, run it +if [[ -n "$reader" ]]; then + if [[ -n "$prereader" ]]; then + exec $concat "$file" | $prereader | $reader + else + exec $concat "$file" | $reader + fi fi exit 1 diff --git a/external/markdown-processor.py b/external/markdown-processor.py index a3e178ec..cb8d793d 100644 --- a/external/markdown-processor.py +++ b/external/markdown-processor.py @@ -22,7 +22,7 @@ .. _Markdown: https://pypi.python.org/pypi/Markdown - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/external/moin-parser.py b/external/moin-parser.py index 9cb082a2..03a7c3c3 100644 --- a/external/moin-parser.py +++ b/external/moin-parser.py @@ -31,7 +31,7 @@ If you do not want to do that and are willing to accept larger HTML output, you can set the INLINESTYLES option below to True. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/external/rst-directive.py b/external/rst-directive.py index f81677b6..de26dd03 100644 --- a/external/rst-directive.py +++ b/external/rst-directive.py @@ -31,7 +31,7 @@ .. _directive documentation: http://docutils.sourceforge.net/docs/howto/rst-directives.html - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/ez_setup.py b/ez_setup.py deleted file mode 100644 index 9dc2c872..00000000 --- a/ez_setup.py +++ /dev/null @@ -1,382 +0,0 @@ -#!python -"""Bootstrap setuptools installation - -If you want to use setuptools in your package's setup.py, just include this -file in the same directory with it, and add this to the top of your setup.py:: - - from ez_setup import use_setuptools - use_setuptools() - -If you want to require a specific version of setuptools, set a download -mirror, or use an alternate download directory, you can do so by supplying -the appropriate options to ``use_setuptools()``. - -This file can also be run as a script to install or upgrade setuptools. -""" -import os -import shutil -import sys -import tempfile -import tarfile -import optparse -import subprocess -import platform - -from distutils import log - -try: - from site import USER_SITE -except ImportError: - USER_SITE = None - -DEFAULT_VERSION = "1.4.2" -DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" - -def _python_cmd(*args): - args = (sys.executable,) + args - return subprocess.call(args) == 0 - -def _check_call_py24(cmd, *args, **kwargs): - res = subprocess.call(cmd, *args, **kwargs) - class CalledProcessError(Exception): - pass - if not res == 0: - msg = "Command '%s' return non-zero exit status %d" % (cmd, res) - raise CalledProcessError(msg) -vars(subprocess).setdefault('check_call', _check_call_py24) - -def _install(tarball, install_args=()): - # extracting the tarball - tmpdir = tempfile.mkdtemp() - log.warn('Extracting in %s', tmpdir) - old_wd = os.getcwd() - try: - os.chdir(tmpdir) - tar = tarfile.open(tarball) - _extractall(tar) - tar.close() - - # going in the directory - subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) - os.chdir(subdir) - log.warn('Now working in %s', subdir) - - # installing - log.warn('Installing Setuptools') - if not _python_cmd('setup.py', 'install', *install_args): - log.warn('Something went wrong during the installation.') - log.warn('See the error message above.') - # exitcode will be 2 - return 2 - finally: - os.chdir(old_wd) - shutil.rmtree(tmpdir) - - -def _build_egg(egg, tarball, to_dir): - # extracting the tarball - tmpdir = tempfile.mkdtemp() - log.warn('Extracting in %s', tmpdir) - old_wd = os.getcwd() - try: - os.chdir(tmpdir) - tar = tarfile.open(tarball) - _extractall(tar) - tar.close() - - # going in the directory - subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) - os.chdir(subdir) - log.warn('Now working in %s', subdir) - - # building an egg - log.warn('Building a Setuptools egg in %s', to_dir) - _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) - - finally: - os.chdir(old_wd) - shutil.rmtree(tmpdir) - # returning the result - log.warn(egg) - if not os.path.exists(egg): - raise IOError('Could not build the egg.') - - -def _do_download(version, download_base, to_dir, download_delay): - egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' - % (version, sys.version_info[0], sys.version_info[1])) - if not os.path.exists(egg): - tarball = download_setuptools(version, download_base, - to_dir, download_delay) - _build_egg(egg, tarball, to_dir) - sys.path.insert(0, egg) - - # Remove previously-imported pkg_resources if present (see - # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). - if 'pkg_resources' in sys.modules: - del sys.modules['pkg_resources'] - - import setuptools - setuptools.bootstrap_install_from = egg - - -def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, download_delay=15): - # making sure we use the absolute path - to_dir = os.path.abspath(to_dir) - was_imported = 'pkg_resources' in sys.modules or \ - 'setuptools' in sys.modules - try: - import pkg_resources - except ImportError: - return _do_download(version, download_base, to_dir, download_delay) - try: - pkg_resources.require("setuptools>=" + version) - return - except pkg_resources.VersionConflict: - e = sys.exc_info()[1] - if was_imported: - sys.stderr.write( - "The required version of setuptools (>=%s) is not available,\n" - "and can't be installed while this script is running. Please\n" - "install a more recent version first, using\n" - "'easy_install -U setuptools'." - "\n\n(Currently using %r)\n" % (version, e.args[0])) - sys.exit(2) - else: - del pkg_resources, sys.modules['pkg_resources'] # reload ok - return _do_download(version, download_base, to_dir, - download_delay) - except pkg_resources.DistributionNotFound: - return _do_download(version, download_base, to_dir, - download_delay) - -def _clean_check(cmd, target): - """ - Run the command to download target. If the command fails, clean up before - re-raising the error. - """ - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - if os.access(target, os.F_OK): - os.unlink(target) - raise - -def download_file_powershell(url, target): - """ - Download the file at url to target using Powershell (which will validate - trust). Raise an exception if the command cannot complete. - """ - target = os.path.abspath(target) - cmd = [ - 'powershell', - '-Command', - "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" % vars(), - ] - _clean_check(cmd, target) - -def has_powershell(): - if platform.system() != 'Windows': - return False - cmd = ['powershell', '-Command', 'echo test'] - devnull = open(os.path.devnull, 'wb') - try: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except: - return False - finally: - devnull.close() - return True - -download_file_powershell.viable = has_powershell - -def download_file_curl(url, target): - cmd = ['curl', url, '--silent', '--output', target] - _clean_check(cmd, target) - -def has_curl(): - cmd = ['curl', '--version'] - devnull = open(os.path.devnull, 'wb') - try: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except: - return False - finally: - devnull.close() - return True - -download_file_curl.viable = has_curl - -def download_file_wget(url, target): - cmd = ['wget', url, '--quiet', '--output-document', target] - _clean_check(cmd, target) - -def has_wget(): - cmd = ['wget', '--version'] - devnull = open(os.path.devnull, 'wb') - try: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except: - return False - finally: - devnull.close() - return True - -download_file_wget.viable = has_wget - -def download_file_insecure(url, target): - """ - Use Python to download the file, even though it cannot authenticate the - connection. - """ - try: - from urllib.request import urlopen - except ImportError: - from urllib2 import urlopen - src = dst = None - try: - src = urlopen(url) - # Read/write all in one block, so we don't create a corrupt file - # if the download is interrupted. - data = src.read() - dst = open(target, "wb") - dst.write(data) - finally: - if src: - src.close() - if dst: - dst.close() - -download_file_insecure.viable = lambda: True - -def get_best_downloader(): - downloaders = [ - download_file_powershell, - download_file_curl, - download_file_wget, - download_file_insecure, - ] - - for dl in downloaders: - if dl.viable(): - return dl - -def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, delay=15, - downloader_factory=get_best_downloader): - """Download setuptools from a specified location and return its filename - - `version` should be a valid setuptools version number that is available - as an egg for download under the `download_base` URL (which should end - with a '/'). `to_dir` is the directory where the egg will be downloaded. - `delay` is the number of seconds to pause before an actual download - attempt. - - ``downloader_factory`` should be a function taking no arguments and - returning a function for downloading a URL to a target. - """ - # making sure we use the absolute path - to_dir = os.path.abspath(to_dir) - tgz_name = "setuptools-%s.tar.gz" % version - url = download_base + tgz_name - saveto = os.path.join(to_dir, tgz_name) - if not os.path.exists(saveto): # Avoid repeated downloads - log.warn("Downloading %s", url) - downloader = downloader_factory() - downloader(url, saveto) - return os.path.realpath(saveto) - - -def _extractall(self, path=".", members=None): - """Extract all members from the archive to the current working - directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). - """ - import copy - import operator - from tarfile import ExtractError - directories = [] - - if members is None: - members = self - - for tarinfo in members: - if tarinfo.isdir(): - # Extract directories with a safe mode. - directories.append(tarinfo) - tarinfo = copy.copy(tarinfo) - tarinfo.mode = 448 # decimal for oct 0700 - self.extract(tarinfo, path) - - # Reverse sort directories. - if sys.version_info < (2, 4): - def sorter(dir1, dir2): - return cmp(dir1.name, dir2.name) - directories.sort(sorter) - directories.reverse() - else: - directories.sort(key=operator.attrgetter('name'), reverse=True) - - # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) - try: - self.chown(tarinfo, dirpath) - self.utime(tarinfo, dirpath) - self.chmod(tarinfo, dirpath) - except ExtractError: - e = sys.exc_info()[1] - if self.errorlevel > 1: - raise - else: - self._dbg(1, "tarfile: %s" % e) - - -def _build_install_args(options): - """ - Build the arguments to 'python setup.py install' on the setuptools package - """ - install_args = [] - if options.user_install: - if sys.version_info < (2, 6): - log.warn("--user requires Python 2.6 or later") - raise SystemExit(1) - install_args.append('--user') - return install_args - -def _parse_args(): - """ - Parse the command line for options - """ - parser = optparse.OptionParser() - parser.add_option( - '--user', dest='user_install', action='store_true', default=False, - help='install in user site package (requires Python 2.6 or later)') - parser.add_option( - '--download-base', dest='download_base', metavar="URL", - default=DEFAULT_URL, - help='alternative URL from where to download the setuptools package') - parser.add_option( - '--insecure', dest='downloader_factory', action='store_const', - const=lambda: download_file_insecure, default=get_best_downloader, - help='Use internal, non-validating downloader' - ) - options, args = parser.parse_args() - # positional arguments are ignored - return options - -def main(version=DEFAULT_VERSION): - """Install or upgrade setuptools and EasyInstall""" - options = _parse_args() - tarball = download_setuptools(download_base=options.download_base, - downloader_factory=options.downloader_factory) - return _install(tarball, _build_install_args(options)) - -if __name__ == '__main__': - sys.exit(main()) diff --git a/pygments/__init__.py b/pygments/__init__.py index c6234404..394a85f2 100644 --- a/pygments/__init__.py +++ b/pygments/__init__.py @@ -22,21 +22,19 @@ .. _Pygments tip: http://bitbucket.org/birkenfeld/pygments-main/get/tip.zip#egg=Pygments-dev - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ +import sys + +from pygments.util import StringIO, BytesIO -__version__ = '2.1.3' +__version__ = '2.2.0' __docformat__ = 'restructuredtext' __all__ = ['lex', 'format', 'highlight'] -import sys - -from pygments.util import StringIO, BytesIO - - def lex(code, lexer): """ Lex ``code`` with ``lexer`` and return an iterable of tokens. @@ -44,9 +42,9 @@ def lex(code, lexer): try: return lexer.get_tokens(code) except TypeError as err: - if isinstance(err.args[0], str) and \ - ('unbound method get_tokens' in err.args[0] or - 'missing 1 required positional argument' in err.args[0]): + if (isinstance(err.args[0], str) and + ('unbound method get_tokens' in err.args[0] or + 'missing 1 required positional argument' in err.args[0])): raise TypeError('lex() argument must be a lexer instance, ' 'not a class') raise @@ -68,9 +66,9 @@ def format(tokens, formatter, outfile=None): # pylint: disable=redefined-builti else: formatter.format(tokens, outfile) except TypeError as err: - if isinstance(err.args[0], str) and \ - ('unbound method format' in err.args[0] or - 'missing 1 required positional argument' in err.args[0]): + if (isinstance(err.args[0], str) and + ('unbound method format' in err.args[0] or + 'missing 1 required positional argument' in err.args[0])): raise TypeError('format() argument must be a formatter instance, ' 'not a class') raise diff --git a/pygments/cmdline.py b/pygments/cmdline.py index 00745edc..5e1f39e2 100644 --- a/pygments/cmdline.py +++ b/pygments/cmdline.py @@ -5,7 +5,7 @@ Command line interface. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -19,11 +19,11 @@ from pygments import __version__, highlight from pygments.util import ClassNotFound, OptionError, docstring_headline, \ guess_decode, guess_decode_from_terminal, terminal_encoding from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \ - get_lexer_for_filename, find_lexer_class_for_filename + load_lexer_from_file, get_lexer_for_filename, find_lexer_class_for_filename from pygments.lexers.special import TextLexer from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter from pygments.formatters import get_all_formatters, get_formatter_by_name, \ - get_formatter_for_filename, find_formatter_class + load_formatter_from_file, get_formatter_for_filename, find_formatter_class from pygments.formatters.terminal import TerminalFormatter from pygments.filters import get_all_filters, find_filter_class from pygments.styles import get_all_styles, get_style_by_name @@ -31,7 +31,7 @@ from pygments.styles import get_all_styles, get_style_by_name USAGE = """\ Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>] - [-O <options>] [-P <option=value>] [-s] [-v] [-o <outfile>] [<infile>] + [-O <options>] [-P <option=value>] [-s] [-v] [-x] [-o <outfile>] [<infile>] %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>] %s -L [<which> ...] @@ -57,6 +57,14 @@ Likewise, <formatter> is a formatter name, and will be guessed from the extension of the output file name. If no output file is given, the terminal formatter will be used by default. +The additional option -x allows custom lexers and formatters to be +loaded from a .py file relative to the current working directory. For +example, ``-l ./customlexer.py -x``. By default, this option expects a +file with a class named CustomLexer or CustomFormatter; you can also +specify your own class name with a colon (``-l ./lexer.py:MyLexer``). +Users should be very careful not to use this option with untrusted files, +because it will import and run them. + With the -O option, you can give the lexer and formatter a comma- separated list of options, e.g. ``-O bg=light,python=cool``. @@ -223,7 +231,7 @@ def main_inner(popts, args, usage): return 0 if opts.pop('-V', None) is not None: - print('Pygments version %s, (c) 2006-2015 by Georg Brandl.' % __version__) + print('Pygments version %s, (c) 2006-2017 by Georg Brandl.' % __version__) return 0 # handle ``pygmentize -L`` @@ -314,17 +322,35 @@ def main_inner(popts, args, usage): F_opts = _parse_filters(F_opts) opts.pop('-F', None) + allow_custom_lexer_formatter = False + # -x: allow custom (eXternal) lexers and formatters + if opts.pop('-x', None) is not None: + allow_custom_lexer_formatter = True + # select lexer lexer = None # given by name? lexername = opts.pop('-l', None) if lexername: - try: - lexer = get_lexer_by_name(lexername, **parsed_opts) - except (OptionError, ClassNotFound) as err: - print('Error:', err, file=sys.stderr) - return 1 + # custom lexer, located relative to user's cwd + if allow_custom_lexer_formatter and '.py' in lexername: + try: + if ':' in lexername: + filename, name = lexername.rsplit(':', 1) + lexer = load_lexer_from_file(filename, name, + **parsed_opts) + else: + lexer = load_lexer_from_file(lexername, **parsed_opts) + except ClassNotFound as err: + print('Error:', err, file=sys.stderr) + return 1 + else: + try: + lexer = get_lexer_by_name(lexername, **parsed_opts) + except (OptionError, ClassNotFound) as err: + print('Error:', err, file=sys.stderr) + return 1 # read input code code = None @@ -401,11 +427,24 @@ def main_inner(popts, args, usage): outfn = opts.pop('-o', None) fmter = opts.pop('-f', None) if fmter: - try: - fmter = get_formatter_by_name(fmter, **parsed_opts) - except (OptionError, ClassNotFound) as err: - print('Error:', err, file=sys.stderr) - return 1 + # custom formatter, located relative to user's cwd + if allow_custom_lexer_formatter and '.py' in fmter: + try: + if ':' in fmter: + file, fmtername = fmter.rsplit(':', 1) + fmter = load_formatter_from_file(file, fmtername, + **parsed_opts) + else: + fmter = load_formatter_from_file(fmter, **parsed_opts) + except ClassNotFound as err: + print('Error:', err, file=sys.stderr) + return 1 + else: + try: + fmter = get_formatter_by_name(fmter, **parsed_opts) + except (OptionError, ClassNotFound) as err: + print('Error:', err, file=sys.stderr) + return 1 if outfn: if not fmter: @@ -495,7 +534,7 @@ def main(args=sys.argv): usage = USAGE % ((args[0],) * 6) try: - popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgs") + popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgsx") except getopt.GetoptError: print(usage, file=sys.stderr) return 2 diff --git a/pygments/console.py b/pygments/console.py index 4a2c9acb..31b6839d 100644 --- a/pygments/console.py +++ b/pygments/console.py @@ -5,25 +5,25 @@ Format colored console output. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ esc = "\x1b[" codes = {} -codes[""] = "" -codes["reset"] = esc + "39;49;00m" +codes[""] = "" +codes["reset"] = esc + "39;49;00m" -codes["bold"] = esc + "01m" -codes["faint"] = esc + "02m" -codes["standout"] = esc + "03m" +codes["bold"] = esc + "01m" +codes["faint"] = esc + "02m" +codes["standout"] = esc + "03m" codes["underline"] = esc + "04m" -codes["blink"] = esc + "05m" -codes["overline"] = esc + "06m" +codes["blink"] = esc + "05m" +codes["overline"] = esc + "06m" -dark_colors = ["black", "darkred", "darkgreen", "brown", "darkblue", - "purple", "teal", "lightgray"] +dark_colors = ["black", "darkred", "darkgreen", "brown", "darkblue", + "purple", "teal", "lightgray"] light_colors = ["darkgray", "red", "green", "yellow", "blue", "fuchsia", "turquoise", "white"] @@ -35,10 +35,10 @@ for d, l in zip(dark_colors, light_colors): del d, l, x -codes["darkteal"] = codes["turquoise"] +codes["darkteal"] = codes["turquoise"] codes["darkyellow"] = codes["brown"] -codes["fuscia"] = codes["fuchsia"] -codes["white"] = codes["bold"] +codes["fuscia"] = codes["fuchsia"] +codes["white"] = codes["bold"] def reset_color(): diff --git a/pygments/filter.py b/pygments/filter.py index c8176ed9..68be7ad7 100644 --- a/pygments/filter.py +++ b/pygments/filter.py @@ -5,7 +5,7 @@ Module that implements the default filter. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -34,10 +34,10 @@ def simplefilter(f): yield ttype, value.lower() """ return type(f.__name__, (FunctionFilter,), { - 'function': f, - '__module__': getattr(f, '__module__'), - '__doc__': f.__doc__ - }) + '__module__': getattr(f, '__module__'), + '__doc__': f.__doc__, + 'function': f, + }) class Filter(object): diff --git a/pygments/filters/__init__.py b/pygments/filters/__init__.py index 45bd49d5..45f9608c 100644 --- a/pygments/filters/__init__.py +++ b/pygments/filters/__init__.py @@ -6,7 +6,7 @@ Module containing filter lookup functions and default filters. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatter.py b/pygments/formatter.py index addd07d7..c0780f62 100644 --- a/pygments/formatter.py +++ b/pygments/formatter.py @@ -5,7 +5,7 @@ Base formatter class. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -65,7 +65,7 @@ class Formatter(object): def __init__(self, **options): self.style = _lookup_style(options.get('style', 'default')) - self.full = get_bool_opt(options, 'full', False) + self.full = get_bool_opt(options, 'full', False) self.title = options.get('title', '') self.encoding = options.get('encoding', None) or None if self.encoding in ('guess', 'chardet'): diff --git a/pygments/formatters/__init__.py b/pygments/formatters/__init__.py index 8ddf0ee3..965c5f1a 100644 --- a/pygments/formatters/__init__.py +++ b/pygments/formatters/__init__.py @@ -5,7 +5,7 @@ Pygments formatters. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -20,7 +20,7 @@ from pygments.plugin import find_plugin_formatters from pygments.util import ClassNotFound, itervalues __all__ = ['get_formatter_by_name', 'get_formatter_for_filename', - 'get_all_formatters'] + list(FORMATTERS) + 'get_all_formatters', 'load_formatter_from_file'] + list(FORMATTERS) _formatter_cache = {} # classes by name _pattern_cache = {} @@ -79,6 +79,41 @@ def get_formatter_by_name(_alias, **options): return cls(**options) +def load_formatter_from_file(filename, formattername="CustomFormatter", + **options): + """Load a formatter from a file. + + This method expects a file located relative to the current working + directory, which contains a class named CustomFormatter. By default, + it expects the Formatter to be named CustomFormatter; you can specify + your own class name as the second argument to this function. + + Users should be very careful with the input, because this method + is equivalent to running eval on the input file. + + Raises ClassNotFound if there are any problems importing the Formatter. + + .. versionadded:: 2.2 + """ + try: + # This empty dict will contain the namespace for the exec'd file + custom_namespace = {} + exec(open(filename, 'rb').read(), custom_namespace) + # Retrieve the class `formattername` from that namespace + if formattername not in custom_namespace: + raise ClassNotFound('no valid %s class found in %s' % + (formattername, filename)) + formatter_class = custom_namespace[formattername] + # And finally instantiate it with the options + return formatter_class(**options) + except IOError as err: + raise ClassNotFound('cannot read %s' % filename) + except ClassNotFound as err: + raise + except Exception as err: + raise ClassNotFound('error when loading custom formatter: %s' % err) + + def get_formatter_for_filename(fn, **options): """Lookup and instantiate a formatter by filename pattern. diff --git a/pygments/formatters/_mapping.py b/pygments/formatters/_mapping.py index 01d053dd..7bb3e71c 100755 --- a/pygments/formatters/_mapping.py +++ b/pygments/formatters/_mapping.py @@ -9,7 +9,7 @@ Do not alter the FORMATTERS dictionary by hand. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/bbcode.py b/pygments/formatters/bbcode.py index 580989f0..9fc9476d 100644 --- a/pygments/formatters/bbcode.py +++ b/pygments/formatters/bbcode.py @@ -5,7 +5,7 @@ BBcode formatter. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py index 2c6bb19e..2969d502 100644 --- a/pygments/formatters/html.py +++ b/pygments/formatters/html.py @@ -5,7 +5,7 @@ Formatter for HTML output. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/img.py b/pygments/formatters/img.py index cc95ce24..2fb0dea5 100644 --- a/pygments/formatters/img.py +++ b/pygments/formatters/img.py @@ -5,10 +5,11 @@ Formatter for Pixmap output. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ +import os import sys from pygments.formatter import Formatter @@ -47,6 +48,7 @@ STYLES = { # A sane default for modern systems DEFAULT_FONT_NAME_NIX = 'Bitstream Vera Sans Mono' DEFAULT_FONT_NAME_WIN = 'Courier New' +DEFAULT_FONT_NAME_MAC = 'Courier New' class PilNotAvailable(ImportError): @@ -71,6 +73,10 @@ class FontManager(object): if not font_name: self.font_name = DEFAULT_FONT_NAME_WIN self._create_win() + elif sys.platform.startswith('darwin'): + if not font_name: + self.font_name = DEFAULT_FONT_NAME_MAC + self._create_mac() else: if not font_name: self.font_name = DEFAULT_FONT_NAME_NIX @@ -111,6 +117,37 @@ class FontManager(object): else: self.fonts[style] = self.fonts['NORMAL'] + def _get_mac_font_path(self, font_map, name, style): + return font_map.get((name + ' ' + style).strip().lower()) + + def _create_mac(self): + font_map = {} + for font_dir in (os.path.join(os.getenv("HOME"), 'Library/Fonts/'), + '/Library/Fonts/', '/System/Library/Fonts/'): + font_map.update( + ((os.path.splitext(f)[0].lower(), os.path.join(font_dir, f)) + for f in os.listdir(font_dir) if f.lower().endswith('ttf'))) + + for name in STYLES['NORMAL']: + path = self._get_mac_font_path(font_map, self.font_name, name) + if path is not None: + self.fonts['NORMAL'] = ImageFont.truetype(path, self.font_size) + break + else: + raise FontNotFound('No usable fonts named: "%s"' % + self.font_name) + for style in ('ITALIC', 'BOLD', 'BOLDITALIC'): + for stylename in STYLES[style]: + path = self._get_mac_font_path(font_map, self.font_name, stylename) + if path is not None: + self.fonts[style] = ImageFont.truetype(path, self.font_size) + break + else: + if style == 'BOLDITALIC': + self.fonts[style] = self.fonts['BOLD'] + else: + self.fonts[style] = self.fonts['NORMAL'] + def _lookup_win(self, key, basename, styles, fail=False): for suffix in ('', ' (TrueType)'): for style in styles: diff --git a/pygments/formatters/irc.py b/pygments/formatters/irc.py index d1eed0ac..eb744d74 100644 --- a/pygments/formatters/irc.py +++ b/pygments/formatters/irc.py @@ -5,7 +5,7 @@ Formatter for IRC output - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/latex.py b/pygments/formatters/latex.py index 66d521f5..336b59de 100644 --- a/pygments/formatters/latex.py +++ b/pygments/formatters/latex.py @@ -5,7 +5,7 @@ Formatter for LaTeX fancyvrb output. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/other.py b/pygments/formatters/other.py index 4945d763..d6bfcacf 100644 --- a/pygments/formatters/other.py +++ b/pygments/formatters/other.py @@ -5,7 +5,7 @@ Other formatters: NullFormatter, RawTokenFormatter. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/rtf.py b/pygments/formatters/rtf.py index 27be225a..c6353c12 100644 --- a/pygments/formatters/rtf.py +++ b/pygments/formatters/rtf.py @@ -5,7 +5,7 @@ A formatter that generates RTF files. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/svg.py b/pygments/formatters/svg.py index 0efe9eea..944b25e0 100644 --- a/pygments/formatters/svg.py +++ b/pygments/formatters/svg.py @@ -5,7 +5,7 @@ Formatter for SVG output. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/terminal.py b/pygments/formatters/terminal.py index 2dbfde7f..b8fec52e 100644 --- a/pygments/formatters/terminal.py +++ b/pygments/formatters/terminal.py @@ -5,7 +5,7 @@ Formatter for terminal output with ANSI sequences. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/terminal256.py b/pygments/formatters/terminal256.py index af311955..b80dc7dd 100644 --- a/pygments/formatters/terminal256.py +++ b/pygments/formatters/terminal256.py @@ -11,7 +11,7 @@ Formatter version 1. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -27,6 +27,8 @@ import sys from pygments.formatter import Formatter +from pygments.console import codes +from pygments.style import ansicolors __all__ = ['Terminal256Formatter', 'TerminalTrueColorFormatter'] @@ -47,9 +49,21 @@ class EscapeSequence: def color_string(self): attrs = [] if self.fg is not None: - attrs.extend(("38", "5", "%i" % self.fg)) + if self.fg in ansicolors: + esc = codes[self.fg[5:]] + if ';01m' in esc: + self.bold = True + # extract fg color code. + attrs.append(esc[2:4]) + else: + attrs.extend(("38", "5", "%i" % self.fg)) if self.bg is not None: - attrs.extend(("48", "5", "%i" % self.bg)) + if self.bg in ansicolors: + esc = codes[self.bg[5:]] + # extract fg color code, add 10 for bg. + attrs.append(str(int(esc[2:4])+10)) + else: + attrs.extend(("48", "5", "%i" % self.bg)) if self.bold: attrs.append("01") if self.underline: @@ -91,6 +105,11 @@ class Terminal256Formatter(Formatter): .. versionadded:: 0.9 + .. versionchanged:: 2.2 + If the used style defines foreground colors in the form ``#ansi*``, then + `Terminal256Formatter` will map these to non extended foreground color. + See :ref:`AnsiTerminalStyle` for more information. + Options accepted: `style` @@ -169,6 +188,10 @@ class Terminal256Formatter(Formatter): def _color_index(self, color): index = self.best_match.get(color, None) + if color in ansicolors: + # strip the `#ansi` part and look up code + index = color + self.best_match[color] = index if index is None: try: rgb = int(str(color), 16) @@ -185,9 +208,14 @@ class Terminal256Formatter(Formatter): def _setup_styles(self): for ttype, ndef in self.style: escape = EscapeSequence() - if ndef['color']: + # get foreground from ansicolor if set + if ndef['ansicolor']: + escape.fg = self._color_index(ndef['ansicolor']) + elif ndef['color']: escape.fg = self._color_index(ndef['color']) - if ndef['bgcolor']: + if ndef['bgansicolor']: + escape.bg = self._color_index(ndef['bgansicolor']) + elif ndef['bgcolor']: escape.bg = self._color_index(ndef['bgcolor']) if self.usebold and ndef['bold']: escape.bold = True diff --git a/pygments/lexer.py b/pygments/lexer.py index dd6c01e4..90905ba5 100644 --- a/pygments/lexer.py +++ b/pygments/lexer.py @@ -5,7 +5,7 @@ Base lexer classes. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -319,8 +319,8 @@ def bygroups(*args): if data is not None: if ctx: ctx.pos = match.start(i + 1) - for item in action( - lexer, _PseudoMatch(match.start(i + 1), data), ctx): + for item in action(lexer, + _PseudoMatch(match.start(i + 1), data), ctx): if item: yield item if ctx: diff --git a/pygments/lexers/__init__.py b/pygments/lexers/__init__.py index 7d0b89d4..328e072c 100644 --- a/pygments/lexers/__init__.py +++ b/pygments/lexers/__init__.py @@ -5,7 +5,7 @@ Pygments lexers. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -22,7 +22,7 @@ from pygments.util import ClassNotFound, itervalues, guess_decode __all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class', - 'guess_lexer'] + list(LEXERS) + 'guess_lexer', 'load_lexer_from_file'] + list(LEXERS) _lexer_cache = {} _pattern_cache = {} @@ -72,6 +72,28 @@ def find_lexer_class(name): return cls +def find_lexer_class_by_name(_alias): + """Lookup a lexer class by alias. + + Like `get_lexer_by_name`, but does not instantiate the class. + + .. versionadded:: 2.2 + """ + if not _alias: + raise ClassNotFound('no lexer for alias %r found' % _alias) + # lookup builtin lexers + for module_name, name, aliases, _, _ in itervalues(LEXERS): + if _alias.lower() in aliases: + if name not in _lexer_cache: + _load_lexers(module_name) + return _lexer_cache[name] + # continue with lexers from setuptools entrypoints + for cls in find_plugin_lexers(): + if _alias.lower() in cls.aliases: + return cls + raise ClassNotFound('no lexer for alias %r found' % _alias) + + def get_lexer_by_name(_alias, **options): """Get a lexer by an alias. @@ -93,6 +115,40 @@ def get_lexer_by_name(_alias, **options): raise ClassNotFound('no lexer for alias %r found' % _alias) +def load_lexer_from_file(filename, lexername="CustomLexer", **options): + """Load a lexer from a file. + + This method expects a file located relative to the current working + directory, which contains a Lexer class. By default, it expects the + Lexer to be name CustomLexer; you can specify your own class name + as the second argument to this function. + + Users should be very careful with the input, because this method + is equivalent to running eval on the input file. + + Raises ClassNotFound if there are any problems importing the Lexer. + + .. versionadded:: 2.2 + """ + try: + # This empty dict will contain the namespace for the exec'd file + custom_namespace = {} + exec(open(filename, 'rb').read(), custom_namespace) + # Retrieve the class `lexername` from that namespace + if lexername not in custom_namespace: + raise ClassNotFound('no valid %s class found in %s' % + (lexername, filename)) + lexer_class = custom_namespace[lexername] + # And finally instantiate it with the options + return lexer_class(**options) + except IOError as err: + raise ClassNotFound('cannot read %s' % filename) + except ClassNotFound as err: + raise + except Exception as err: + raise ClassNotFound('error when loading custom lexer: %s' % err) + + def find_lexer_class_for_filename(_fn, code=None): """Get a lexer for a filename. @@ -127,8 +183,8 @@ def find_lexer_class_for_filename(_fn, code=None): # gets turned into 0.0. Run scripts/detect_missing_analyse_text.py # to find lexers which need it overridden. if code: - return cls.analyse_text(code) + bonus - return cls.priority + bonus + return cls.analyse_text(code) + bonus, cls.__name__ + return cls.priority + bonus, cls.__name__ if matches: matches.sort(key=get_rating) diff --git a/pygments/lexers/_asy_builtins.py b/pygments/lexers/_asy_builtins.py index 51716866..1f831cdb 100644 --- a/pygments/lexers/_asy_builtins.py +++ b/pygments/lexers/_asy_builtins.py @@ -10,7 +10,7 @@ TODO: perl/python script in Asymptote SVN similar to asy-list.pl but only for function and variable names. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_cl_builtins.py b/pygments/lexers/_cl_builtins.py index a2243647..ce5ad48e 100644 --- a/pygments/lexers/_cl_builtins.py +++ b/pygments/lexers/_cl_builtins.py @@ -5,7 +5,7 @@ ANSI Common Lisp builtins. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_cocoa_builtins.py b/pygments/lexers/_cocoa_builtins.py index a4f00d9d..064167ff 100644 --- a/pygments/lexers/_cocoa_builtins.py +++ b/pygments/lexers/_cocoa_builtins.py @@ -8,7 +8,7 @@ File may be also used as standalone generator for aboves. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_csound_builtins.py b/pygments/lexers/_csound_builtins.py index a88e0a83..e5a9aaf7 100644 --- a/pygments/lexers/_csound_builtins.py +++ b/pygments/lexers/_csound_builtins.py @@ -3,7 +3,7 @@ pygments.lexers._csound_builtins ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_lasso_builtins.py b/pygments/lexers/_lasso_builtins.py index 7c6fd6d4..d950cbe8 100644 --- a/pygments/lexers/_lasso_builtins.py +++ b/pygments/lexers/_lasso_builtins.py @@ -5,7 +5,7 @@ Built-in Lasso types, traits, methods, and members. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -471,6 +471,10 @@ BUILTINS = { 'curl_netrc_ignored', 'curl_netrc_optional', 'curl_netrc_required', + 'curl_sslversion_default', + 'curl_sslversion_sslv2', + 'curl_sslversion_sslv3', + 'curl_sslversion_tlsv1', 'curl_version_asynchdns', 'curl_version_debug', 'curl_version_gssnegotiate', @@ -1102,6 +1106,7 @@ BUILTINS = { 'json_open_array', 'json_open_object', 'json_period', + 'json_positive', 'json_quote_double', 'json_rpccall', 'json_serialize', @@ -1229,6 +1234,7 @@ BUILTINS = { 'lcapi_loadmodules', 'lcapi_updatedatasourceslist', 'ldap_scope_base', + 'ldap_scope_children', 'ldap_scope_onelevel', 'ldap_scope_subtree', 'library_once', @@ -4044,6 +4050,7 @@ MEMBERS = { 'iscntrl', 'isdigit', 'isdir', + 'isdirectory', 'isempty', 'isemptyelement', 'isfirststep', diff --git a/pygments/lexers/_lua_builtins.py b/pygments/lexers/_lua_builtins.py index 6d2929b6..c60bf5a2 100644 --- a/pygments/lexers/_lua_builtins.py +++ b/pygments/lexers/_lua_builtins.py @@ -9,60 +9,71 @@ Do not edit the MODULES dict by hand. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ from __future__ import print_function - MODULES = {'basic': ('_G', '_VERSION', 'assert', 'collectgarbage', 'dofile', 'error', - 'getfenv', 'getmetatable', 'ipairs', 'load', 'loadfile', - 'loadstring', 'next', 'pairs', 'pcall', 'print', 'rawequal', 'rawget', + 'rawlen', 'rawset', 'select', - 'setfenv', 'setmetatable', 'tonumber', 'tostring', 'type', - 'unpack', 'xpcall'), + 'bit32': ('bit32.arshift', + 'bit32.band', + 'bit32.bnot', + 'bit32.bor', + 'bit32.btest', + 'bit32.bxor', + 'bit32.extract', + 'bit32.lrotate', + 'bit32.lshift', + 'bit32.replace', + 'bit32.rrotate', + 'bit32.rshift'), 'coroutine': ('coroutine.create', + 'coroutine.isyieldable', 'coroutine.resume', 'coroutine.running', 'coroutine.status', 'coroutine.wrap', 'coroutine.yield'), 'debug': ('debug.debug', - 'debug.getfenv', 'debug.gethook', 'debug.getinfo', 'debug.getlocal', 'debug.getmetatable', 'debug.getregistry', 'debug.getupvalue', - 'debug.setfenv', + 'debug.getuservalue', 'debug.sethook', 'debug.setlocal', 'debug.setmetatable', 'debug.setupvalue', - 'debug.traceback'), + 'debug.setuservalue', + 'debug.traceback', + 'debug.upvalueid', + 'debug.upvaluejoin'), 'io': ('io.close', 'io.flush', 'io.input', @@ -71,17 +82,20 @@ MODULES = {'basic': ('_G', 'io.output', 'io.popen', 'io.read', + 'io.stderr', + 'io.stdin', + 'io.stdout', 'io.tmpfile', 'io.type', 'io.write'), 'math': ('math.abs', 'math.acos', 'math.asin', - 'math.atan2', 'math.atan', + 'math.atan2', 'math.ceil', - 'math.cosh', 'math.cos', + 'math.cosh', 'math.deg', 'math.exp', 'math.floor', @@ -89,29 +103,34 @@ MODULES = {'basic': ('_G', 'math.frexp', 'math.huge', 'math.ldexp', - 'math.log10', 'math.log', 'math.max', + 'math.maxinteger', 'math.min', + 'math.mininteger', 'math.modf', 'math.pi', 'math.pow', 'math.rad', 'math.random', 'math.randomseed', - 'math.sinh', 'math.sin', + 'math.sinh', 'math.sqrt', + 'math.tan', 'math.tanh', - 'math.tan'), - 'modules': ('module', - 'require', + 'math.tointeger', + 'math.type', + 'math.ult'), + 'modules': ('package.config', 'package.cpath', 'package.loaded', 'package.loadlib', 'package.path', 'package.preload', - 'package.seeall'), + 'package.searchers', + 'package.searchpath', + 'require'), 'os': ('os.clock', 'os.date', 'os.difftime', @@ -133,19 +152,37 @@ MODULES = {'basic': ('_G', 'string.len', 'string.lower', 'string.match', + 'string.pack', + 'string.packsize', 'string.rep', 'string.reverse', 'string.sub', + 'string.unpack', 'string.upper'), 'table': ('table.concat', 'table.insert', - 'table.maxn', + 'table.move', + 'table.pack', 'table.remove', - 'table.sort')} - + 'table.sort', + 'table.unpack'), + 'utf8': ('utf8.char', + 'utf8.charpattern', + 'utf8.codepoint', + 'utf8.codes', + 'utf8.len', + 'utf8.offset')} if __name__ == '__main__': # pragma: no cover import re + import sys + + # urllib ends up wanting to import a module called 'math' -- if + # pygments/lexers is in the path, this ends badly. + for i in range(len(sys.path)-1, -1, -1): + if sys.path[i].endswith('/lexers'): + del sys.path[i] + try: from urllib import urlopen except ImportError: @@ -196,7 +233,7 @@ if __name__ == '__main__': # pragma: no cover def get_newest_version(): f = urlopen('http://www.lua.org/manual/') - r = re.compile(r'^<A HREF="(\d\.\d)/">Lua \1</A>') + r = re.compile(r'^<A HREF="(\d\.\d)/">(Lua )?\1</A>') for line in f: m = r.match(line) if m is not None: @@ -204,7 +241,7 @@ if __name__ == '__main__': # pragma: no cover def get_lua_functions(version): f = urlopen('http://www.lua.org/manual/%s/' % version) - r = re.compile(r'^<A HREF="manual.html#pdf-(.+)">\1</A>') + r = re.compile(r'^<A HREF="manual.html#pdf-(?!lua|LUA)([^:]+)">\1</A>') functions = [] for line in f: m = r.match(line) @@ -236,15 +273,22 @@ if __name__ == '__main__': # pragma: no cover def run(): version = get_newest_version() - print('> Downloading function index for Lua %s' % version) - functions = get_lua_functions(version) - print('> %d functions found:' % len(functions)) + functions = set() + for v in ('5.2', version): + print('> Downloading function index for Lua %s' % v) + f = get_lua_functions(v) + print('> %d functions found, %d new:' % + (len(f), len(set(f) - functions))) + functions |= set(f) + + functions = sorted(functions) modules = {} for full_function_name in functions: print('>> %s' % full_function_name) m = get_function_module(full_function_name) modules.setdefault(m, []).append(full_function_name) + modules = {k: tuple(v) for k, v in modules.iteritems()} regenerate(__file__, modules) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index 090e7a92..ea54241c 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -9,7 +9,7 @@ Do not alter the LEXERS dictionary by hand. - :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2014, 2016 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -24,8 +24,12 @@ LEXERS = { 'AdaLexer': ('pygments.lexers.pascal', 'Ada', ('ada', 'ada95', 'ada2005'), ('*.adb', '*.ads', '*.ada'), ('text/x-ada',)), 'AdlLexer': ('pygments.lexers.archetype', 'ADL', ('adl',), ('*.adl', '*.adls', '*.adlf', '*.adlx'), ()), 'AgdaLexer': ('pygments.lexers.haskell', 'Agda', ('agda',), ('*.agda',), ('text/x-agda',)), + 'AheuiLexer': ('pygments.lexers.esoteric', 'Aheui', ('aheui',), ('*.aheui',), ()), 'AlloyLexer': ('pygments.lexers.dsls', 'Alloy', ('alloy',), ('*.als',), ('text/x-alloy',)), 'AmbientTalkLexer': ('pygments.lexers.ambient', 'AmbientTalk', ('at', 'ambienttalk', 'ambienttalk/2'), ('*.at',), ('text/x-ambienttalk',)), + 'AmplLexer': ('pygments.lexers.ampl', 'Ampl', ('ampl',), ('*.run',), ()), + 'Angular2HtmlLexer': ('pygments.lexers.templates', 'HTML + Angular2', ('html+ng2',), ('*.ng2',), ()), + 'Angular2Lexer': ('pygments.lexers.templates', 'Angular2', ('ng2',), (), ()), 'AntlrActionScriptLexer': ('pygments.lexers.parsers', 'ANTLR With ActionScript Target', ('antlr-as', 'antlr-actionscript'), ('*.G', '*.g'), ()), 'AntlrCSharpLexer': ('pygments.lexers.parsers', 'ANTLR With C# Target', ('antlr-csharp', 'antlr-c#'), ('*.G', '*.g'), ()), 'AntlrCppLexer': ('pygments.lexers.parsers', 'ANTLR With CPP Target', ('antlr-cpp',), ('*.G', '*.g'), ()), @@ -45,16 +49,18 @@ LEXERS = { 'AwkLexer': ('pygments.lexers.textedit', 'Awk', ('awk', 'gawk', 'mawk', 'nawk'), ('*.awk',), ('application/x-awk',)), 'BBCodeLexer': ('pygments.lexers.markup', 'BBCode', ('bbcode',), (), ('text/x-bbcode',)), 'BCLexer': ('pygments.lexers.algebra', 'BC', ('bc',), ('*.bc',), ()), + 'BSTLexer': ('pygments.lexers.bibtex', 'BST', ('bst', 'bst-pybtex'), ('*.bst',), ()), 'BaseMakefileLexer': ('pygments.lexers.make', 'Base Makefile', ('basemake',), (), ()), - 'BashLexer': ('pygments.lexers.shell', 'Bash', ('bash', 'sh', 'ksh', 'shell'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', '*.exheres-0', '*.exlib', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'PKGBUILD'), ('application/x-sh', 'application/x-shellscript')), + 'BashLexer': ('pygments.lexers.shell', 'Bash', ('bash', 'sh', 'ksh', 'zsh', 'shell'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', '*.exheres-0', '*.exlib', '*.zsh', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', 'PKGBUILD'), ('application/x-sh', 'application/x-shellscript')), 'BashSessionLexer': ('pygments.lexers.shell', 'Bash Session', ('console', 'shell-session'), ('*.sh-session', '*.shell-session'), ('application/x-shell-session', 'application/x-sh-session')), 'BatchLexer': ('pygments.lexers.shell', 'Batchfile', ('bat', 'batch', 'dosbatch', 'winbatch'), ('*.bat', '*.cmd'), ('application/x-dos-batch',)), 'BefungeLexer': ('pygments.lexers.esoteric', 'Befunge', ('befunge',), ('*.befunge',), ('application/x-befunge',)), + 'BibTeXLexer': ('pygments.lexers.bibtex', 'BibTeX', ('bib', 'bibtex'), ('*.bib',), ('text/x-bibtex',)), 'BlitzBasicLexer': ('pygments.lexers.basic', 'BlitzBasic', ('blitzbasic', 'b3d', 'bplus'), ('*.bb', '*.decls'), ('text/x-bb',)), 'BlitzMaxLexer': ('pygments.lexers.basic', 'BlitzMax', ('blitzmax', 'bmax'), ('*.bmx',), ('text/x-bmx',)), 'BnfLexer': ('pygments.lexers.grammar_notation', 'BNF', ('bnf',), ('*.bnf',), ('text/x-bnf',)), 'BooLexer': ('pygments.lexers.dotnet', 'Boo', ('boo',), ('*.boo',), ('text/x-boo',)), - 'BoogieLexer': ('pygments.lexers.esoteric', 'Boogie', ('boogie',), ('*.bpl',), ()), + 'BoogieLexer': ('pygments.lexers.verification', 'Boogie', ('boogie',), ('*.bpl',), ()), 'BrainfuckLexer': ('pygments.lexers.esoteric', 'Brainfuck', ('brainfuck', 'bf'), ('*.bf', '*.b'), ('application/x-brainfuck',)), 'BroLexer': ('pygments.lexers.dsls', 'Bro', ('bro',), ('*.bro',), ()), 'BugsLexer': ('pygments.lexers.modeling', 'BUGS', ('bugs', 'winbugs', 'openbugs'), ('*.bug',), ()), @@ -67,6 +73,8 @@ LEXERS = { 'CSharpLexer': ('pygments.lexers.dotnet', 'C#', ('csharp', 'c#'), ('*.cs',), ('text/x-csharp',)), 'Ca65Lexer': ('pygments.lexers.asm', 'ca65 assembler', ('ca65',), ('*.s',), ()), 'CadlLexer': ('pygments.lexers.archetype', 'cADL', ('cadl',), ('*.cadl',), ()), + 'CapDLLexer': ('pygments.lexers.esoteric', 'CapDL', ('capdl',), ('*.cdl',), ()), + 'CapnProtoLexer': ('pygments.lexers.capnproto', "Cap'n Proto", ('capnp',), ('*.capnp',), ()), 'CbmBasicV2Lexer': ('pygments.lexers.basic', 'CBM BASIC V2', ('cbmbas',), ('*.bas',), ()), 'CeylonLexer': ('pygments.lexers.jvm', 'Ceylon', ('ceylon',), ('*.ceylon',), ('text/x-ceylon',)), 'Cfengine3Lexer': ('pygments.lexers.configs', 'CFEngine3', ('cfengine3', 'cf3'), ('*.cf',), ()), @@ -78,6 +86,7 @@ LEXERS = { 'CheetahXmlLexer': ('pygments.lexers.templates', 'XML+Cheetah', ('xml+cheetah', 'xml+spitfire'), (), ('application/xml+cheetah', 'application/xml+spitfire')), 'CirruLexer': ('pygments.lexers.webmisc', 'Cirru', ('cirru',), ('*.cirru',), ('text/x-cirru',)), 'ClayLexer': ('pygments.lexers.c_like', 'Clay', ('clay',), ('*.clay',), ('text/x-clay',)), + 'CleanLexer': ('pygments.lexers.clean', 'Clean', ('clean',), ('*.icl', '*.dcl'), ()), 'ClojureLexer': ('pygments.lexers.jvm', 'Clojure', ('clojure', 'clj'), ('*.clj',), ('text/x-clojure', 'application/x-clojure')), 'ClojureScriptLexer': ('pygments.lexers.jvm', 'ClojureScript', ('clojurescript', 'cljs'), ('*.cljs',), ('text/x-clojurescript', 'application/x-clojurescript')), 'CobolFreeformatLexer': ('pygments.lexers.business', 'COBOLFree', ('cobolfree',), ('*.cbl', '*.CBL'), ()), @@ -94,6 +103,7 @@ LEXERS = { 'CrmshLexer': ('pygments.lexers.dsls', 'Crmsh', ('crmsh', 'pcmk'), ('*.crmsh', '*.pcmk'), ()), 'CrocLexer': ('pygments.lexers.d', 'Croc', ('croc',), ('*.croc',), ('text/x-crocsrc',)), 'CryptolLexer': ('pygments.lexers.haskell', 'Cryptol', ('cryptol', 'cry'), ('*.cry',), ('text/x-cryptol',)), + 'CrystalLexer': ('pygments.lexers.crystal', 'Crystal', ('cr', 'crystal'), ('*.cr',), ('text/x-crystal',)), 'CsoundDocumentLexer': ('pygments.lexers.csound', 'Csound Document', ('csound-document', 'csound-csd'), ('*.csd',), ()), 'CsoundOrchestraLexer': ('pygments.lexers.csound', 'Csound Orchestra', ('csound', 'csound-orc'), ('*.orc',), ()), 'CsoundScoreLexer': ('pygments.lexers.csound', 'Csound Score', ('csound-score', 'csound-sco'), ('*.sco',), ()), @@ -111,7 +121,7 @@ LEXERS = { 'DarcsPatchLexer': ('pygments.lexers.diff', 'Darcs Patch', ('dpatch',), ('*.dpatch', '*.darcspatch'), ()), 'DartLexer': ('pygments.lexers.javascript', 'Dart', ('dart',), ('*.dart',), ('text/x-dart',)), 'DebianControlLexer': ('pygments.lexers.installers', 'Debian Control file', ('control', 'debcontrol'), ('control',), ()), - 'DelphiLexer': ('pygments.lexers.pascal', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas',), ('text/x-pascal',)), + 'DelphiLexer': ('pygments.lexers.pascal', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas', '*.dpr'), ('text/x-pascal',)), 'DgLexer': ('pygments.lexers.python', 'dg', ('dg',), ('*.dg',), ('text/x-dg',)), 'DiffLexer': ('pygments.lexers.diff', 'Diff', ('diff', 'udiff'), ('*.diff', '*.patch'), ('text/x-diff', 'text/x-patch')), 'DjangoLexer': ('pygments.lexers.templates', 'Django/Jinja', ('django', 'jinja'), (), ('application/x-django-templating', 'application/x-jinja')), @@ -144,6 +154,8 @@ LEXERS = { 'FantomLexer': ('pygments.lexers.fantom', 'Fantom', ('fan',), ('*.fan',), ('application/x-fantom',)), 'FelixLexer': ('pygments.lexers.felix', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)), 'FishShellLexer': ('pygments.lexers.shell', 'Fish', ('fish', 'fishshell'), ('*.fish', '*.load'), ('application/x-fish',)), + 'FlatlineLexer': ('pygments.lexers.dsls', 'Flatline', ('flatline',), (), ('text/x-flatline',)), + 'ForthLexer': ('pygments.lexers.forth', 'Forth', ('forth',), ('*.frt', '*.fs'), ('application/x-forth',)), 'FortranFixedLexer': ('pygments.lexers.fortran', 'FortranFixed', ('fortranfixed',), ('*.f', '*.F'), ()), 'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f03', '*.f90', '*.F03', '*.F90'), ('text/x-fortran',)), 'FoxProLexer': ('pygments.lexers.foxpro', 'FoxPro', ('foxpro', 'vfp', 'clipper', 'xbase'), ('*.PRG', '*.prg'), ()), @@ -168,6 +180,7 @@ LEXERS = { 'HaskellLexer': ('pygments.lexers.haskell', 'Haskell', ('haskell', 'hs'), ('*.hs',), ('text/x-haskell',)), 'HaxeLexer': ('pygments.lexers.haxe', 'Haxe', ('hx', 'haxe', 'hxsl'), ('*.hx', '*.hxsl'), ('text/haxe', 'text/x-haxe', 'text/x-hx')), 'HexdumpLexer': ('pygments.lexers.hexdump', 'Hexdump', ('hexdump',), (), ()), + 'HsailLexer': ('pygments.lexers.asm', 'HSAIL', ('hsail', 'hsa'), ('*.hsail',), ('text/x-hsail',)), 'HtmlDjangoLexer': ('pygments.lexers.templates', 'HTML+Django/Jinja', ('html+django', 'html+jinja', 'htmldjango'), (), ('text/html+django', 'text/html+jinja')), 'HtmlGenshiLexer': ('pygments.lexers.templates', 'HTML+Genshi', ('html+genshi', 'html+kid'), (), ('text/html+genshi',)), 'HtmlLexer': ('pygments.lexers.html', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml', '*.xslt'), ('text/html', 'application/xhtml+xml')), @@ -189,7 +202,6 @@ LEXERS = { 'IrcLogsLexer': ('pygments.lexers.textfmts', 'IRC logs', ('irc',), ('*.weechatlog',), ('text/x-irclog',)), 'IsabelleLexer': ('pygments.lexers.theorem', 'Isabelle', ('isabelle',), ('*.thy',), ('text/x-isabelle',)), 'JLexer': ('pygments.lexers.j', 'J', ('j',), ('*.ijs',), ('text/x-j',)), - 'JadeLexer': ('pygments.lexers.html', 'Jade', ('jade',), ('*.jade',), ('text/x-jade',)), 'JagsLexer': ('pygments.lexers.modeling', 'JAGS', ('jags',), ('*.jag', '*.bug'), ()), 'JasminLexer': ('pygments.lexers.jvm', 'Jasmin', ('jasmin', 'jasminxt'), ('*.j',), ()), 'JavaLexer': ('pygments.lexers.jvm', 'Java', ('java',), ('*.java',), ('text/x-java',)), @@ -200,11 +212,14 @@ LEXERS = { 'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ('application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php')), 'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ('application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty')), 'JclLexer': ('pygments.lexers.scripting', 'JCL', ('jcl',), ('*.jcl',), ('text/x-jcl',)), + 'JsgfLexer': ('pygments.lexers.grammar_notation', 'JSGF', ('jsgf',), ('*.jsgf',), ('application/jsgf', 'application/x-jsgf', 'text/jsgf')), + 'JsonBareObjectLexer': ('pygments.lexers.data', 'JSONBareObject', ('json-object',), (), ('application/json-object',)), 'JsonLdLexer': ('pygments.lexers.data', 'JSON-LD', ('jsonld', 'json-ld'), ('*.jsonld',), ('application/ld+json',)), 'JsonLexer': ('pygments.lexers.data', 'JSON', ('json',), ('*.json',), ('application/json',)), 'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ('application/x-jsp',)), 'JuliaConsoleLexer': ('pygments.lexers.julia', 'Julia console', ('jlcon',), (), ()), 'JuliaLexer': ('pygments.lexers.julia', 'Julia', ('julia', 'jl'), ('*.jl',), ('text/x-julia', 'application/x-julia')), + 'JuttleLexer': ('pygments.lexers.javascript', 'Juttle', ('juttle', 'juttle'), ('*.juttle',), ('application/juttle', 'application/x-juttle', 'text/x-juttle', 'text/juttle')), 'KalLexer': ('pygments.lexers.javascript', 'Kal', ('kal',), ('*.kal',), ('text/kal', 'application/kal')), 'KconfigLexer': ('pygments.lexers.configs', 'Kconfig', ('kconfig', 'menuconfig', 'linux-config', 'kernel-config'), ('Kconfig', '*Config.in*', 'external.in*', 'standard-modules.in'), ('text/x-kconfig',)), 'KokaLexer': ('pygments.lexers.haskell', 'Koka', ('koka',), ('*.kk', '*.kki'), ('text/x-koka',)), @@ -238,6 +253,7 @@ LEXERS = { 'MakoLexer': ('pygments.lexers.templates', 'Mako', ('mako',), ('*.mao',), ('application/x-mako',)), 'MakoXmlLexer': ('pygments.lexers.templates', 'XML+Mako', ('xml+mako',), (), ('application/xml+mako',)), 'MaqlLexer': ('pygments.lexers.business', 'MAQL', ('maql',), ('*.maql',), ('text/x-gooddata-maql', 'application/x-gooddata-maql')), + 'MarkdownLexer': ('pygments.lexers.markup', 'markdown', ('md',), ('*.md',), ('text/x-markdown',)), 'MaskLexer': ('pygments.lexers.javascript', 'Mask', ('mask',), ('*.mask',), ('text/x-mask',)), 'MasonLexer': ('pygments.lexers.templates', 'Mason', ('mason',), ('*.m', '*.mhtml', '*.mc', '*.mi', 'autohandler', 'dhandler'), ('application/x-mason',)), 'MathematicaLexer': ('pygments.lexers.algebra', 'Mathematica', ('mathematica', 'mma', 'nb'), ('*.nb', '*.cdf', '*.nbp', '*.ma'), ('application/mathematica', 'application/vnd.wolfram.mathematica', 'application/vnd.wolfram.mathematica.package', 'application/vnd.wolfram.cdf')), @@ -248,6 +264,7 @@ LEXERS = { 'Modula2Lexer': ('pygments.lexers.modula2', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)), 'MoinWikiLexer': ('pygments.lexers.markup', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)), 'MonkeyLexer': ('pygments.lexers.basic', 'Monkey', ('monkey',), ('*.monkey',), ('text/x-monkey',)), + 'MonteLexer': ('pygments.lexers.monte', 'Monte', ('monte',), ('*.mt',), ()), 'MoonScriptLexer': ('pygments.lexers.scripting', 'MoonScript', ('moon', 'moonscript'), ('*.moon',), ('text/x-moonscript', 'application/x-moonscript')), 'MozPreprocCssLexer': ('pygments.lexers.markup', 'CSS+mozpreproc', ('css+mozpreproc',), ('*.css.in',), ()), 'MozPreprocHashLexer': ('pygments.lexers.markup', 'mozhashpreproc', ('mozhashpreproc',), (), ()), @@ -264,17 +281,19 @@ LEXERS = { 'MyghtyJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Myghty', ('js+myghty', 'javascript+myghty'), (), ('application/x-javascript+myghty', 'text/x-javascript+myghty', 'text/javascript+mygthy')), 'MyghtyLexer': ('pygments.lexers.templates', 'Myghty', ('myghty',), ('*.myt', 'autodelegate'), ('application/x-myghty',)), 'MyghtyXmlLexer': ('pygments.lexers.templates', 'XML+Myghty', ('xml+myghty',), (), ('application/xml+myghty',)), + 'NCLLexer': ('pygments.lexers.ncl', 'NCL', ('ncl',), ('*.ncl',), ('text/ncl',)), 'NSISLexer': ('pygments.lexers.installers', 'NSIS', ('nsis', 'nsi', 'nsh'), ('*.nsi', '*.nsh'), ('text/x-nsis',)), 'NasmLexer': ('pygments.lexers.asm', 'NASM', ('nasm',), ('*.asm', '*.ASM'), ('text/x-nasm',)), 'NasmObjdumpLexer': ('pygments.lexers.asm', 'objdump-nasm', ('objdump-nasm',), ('*.objdump-intel',), ('text/x-nasm-objdump',)), 'NemerleLexer': ('pygments.lexers.dotnet', 'Nemerle', ('nemerle',), ('*.n',), ('text/x-nemerle',)), 'NesCLexer': ('pygments.lexers.c_like', 'nesC', ('nesc',), ('*.nc',), ('text/x-nescsrc',)), - 'NewLispLexer': ('pygments.lexers.lisp', 'NewLisp', ('newlisp',), ('*.lsp', '*.nl'), ('text/x-newlisp', 'application/x-newlisp')), + 'NewLispLexer': ('pygments.lexers.lisp', 'NewLisp', ('newlisp',), ('*.lsp', '*.nl', '*.kif'), ('text/x-newlisp', 'application/x-newlisp')), 'NewspeakLexer': ('pygments.lexers.smalltalk', 'Newspeak', ('newspeak',), ('*.ns2',), ('text/x-newspeak',)), - 'NginxConfLexer': ('pygments.lexers.configs', 'Nginx configuration file', ('nginx',), (), ('text/x-nginx-conf',)), - 'NimrodLexer': ('pygments.lexers.nimrod', 'Nimrod', ('nimrod', 'nim'), ('*.nim', '*.nimrod'), ('text/x-nimrod',)), + 'NginxConfLexer': ('pygments.lexers.configs', 'Nginx configuration file', ('nginx',), ('nginx.conf',), ('text/x-nginx-conf',)), + 'NimrodLexer': ('pygments.lexers.nimrod', 'Nimrod', ('nim', 'nimrod'), ('*.nim', '*.nimrod'), ('text/x-nim',)), 'NitLexer': ('pygments.lexers.nit', 'Nit', ('nit',), ('*.nit',), ()), 'NixLexer': ('pygments.lexers.nix', 'Nix', ('nixos', 'nix'), ('*.nix',), ('text/x-nix',)), + 'NuSMVLexer': ('pygments.lexers.smv', 'NuSMV', ('nusmv',), ('*.smv',), ()), 'NumPyLexer': ('pygments.lexers.python', 'NumPy', ('numpy',), (), ()), 'ObjdumpLexer': ('pygments.lexers.asm', 'objdump', ('objdump',), ('*.objdump',), ('text/x-objdump',)), 'ObjectiveCLexer': ('pygments.lexers.objective', 'Objective-C', ('objective-c', 'objectivec', 'obj-c', 'objc'), ('*.m', '*.h'), ('text/x-objective-c',)), @@ -307,6 +326,7 @@ LEXERS = { 'PrologLexer': ('pygments.lexers.prolog', 'Prolog', ('prolog',), ('*.ecl', '*.prolog', '*.pro', '*.pl'), ('text/x-prolog',)), 'PropertiesLexer': ('pygments.lexers.configs', 'Properties', ('properties', 'jproperties'), ('*.properties',), ('text/x-java-properties',)), 'ProtoBufLexer': ('pygments.lexers.dsls', 'Protocol Buffer', ('protobuf', 'proto'), ('*.proto',), ()), + 'PugLexer': ('pygments.lexers.html', 'Pug', ('pug', 'jade'), ('*.pug', '*.jade'), ('text/x-pug', 'text/x-jade')), 'PuppetLexer': ('pygments.lexers.dsls', 'Puppet', ('puppet',), ('*.pp',), ()), 'PyPyLogLexer': ('pygments.lexers.console', 'PyPy Log', ('pypylog', 'pypy'), ('*.pypylog',), ('application/x-pypylog',)), 'Python3Lexer': ('pygments.lexers.python', 'Python 3', ('python3', 'py3'), (), ('text/x-python3', 'application/x-python3')), @@ -318,6 +338,7 @@ LEXERS = { 'QVToLexer': ('pygments.lexers.qvt', 'QVTO', ('qvto', 'qvt'), ('*.qvto',), ()), 'QmlLexer': ('pygments.lexers.webmisc', 'QML', ('qml', 'qbs'), ('*.qml', '*.qbs'), ('application/x-qml', 'application/x-qt.qbs+qml')), 'RConsoleLexer': ('pygments.lexers.r', 'RConsole', ('rconsole', 'rout'), ('*.Rout',), ()), + 'RNCCompactLexer': ('pygments.lexers.rnc', 'Relax-NG Compact', ('rnc', 'rng-compact'), ('*.rnc',), ()), 'RPMSpecLexer': ('pygments.lexers.installers', 'RPMSpec', ('spec',), ('*.spec',), ('text/x-rpm-spec',)), 'RacketLexer': ('pygments.lexers.lisp', 'Racket', ('racket', 'rkt'), ('*.rkt', '*.rktd', '*.rktl'), ('text/x-racket', 'application/x-racket')), 'RagelCLexer': ('pygments.lexers.parsers', 'Ragel in C Host', ('ragel-c',), ('*.rl',), ()), @@ -347,6 +368,7 @@ LEXERS = { 'RubyConsoleLexer': ('pygments.lexers.ruby', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)), 'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')), 'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs', '*.rs.in'), ('text/rust',)), + 'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')), 'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')), 'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')), 'SassLexer': ('pygments.lexers.css', 'Sass', ('sass',), ('*.sass',), ('text/x-sass',)), @@ -356,11 +378,13 @@ LEXERS = { 'ScilabLexer': ('pygments.lexers.matlab', 'Scilab', ('scilab',), ('*.sci', '*.sce', '*.tst'), ('text/scilab',)), 'ScssLexer': ('pygments.lexers.css', 'SCSS', ('scss',), ('*.scss',), ('text/x-scss',)), 'ShenLexer': ('pygments.lexers.lisp', 'Shen', ('shen',), ('*.shen',), ('text/x-shen', 'application/x-shen')), + 'SilverLexer': ('pygments.lexers.verification', 'Silver', ('silver',), ('*.sil', '*.vpr'), ()), 'SlimLexer': ('pygments.lexers.webmisc', 'Slim', ('slim',), ('*.slim',), ('text/x-slim',)), 'SmaliLexer': ('pygments.lexers.dalvik', 'Smali', ('smali',), ('*.smali',), ('text/smali',)), 'SmalltalkLexer': ('pygments.lexers.smalltalk', 'Smalltalk', ('smalltalk', 'squeak', 'st'), ('*.st',), ('text/x-smalltalk',)), 'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)), 'SnobolLexer': ('pygments.lexers.snobol', 'Snobol', ('snobol',), ('*.snobol',), ('text/x-snobol',)), + 'SnowballLexer': ('pygments.lexers.dsls', 'Snowball', ('snowball',), ('*.sbl',), ()), 'SourcePawnLexer': ('pygments.lexers.pawn', 'SourcePawn', ('sp',), ('*.sp',), ('text/x-sourcepawn',)), 'SourcesListLexer': ('pygments.lexers.installers', 'Debian Sourcelist', ('sourceslist', 'sources.list', 'debsources'), ('sources.list',), ()), 'SparqlLexer': ('pygments.lexers.rdf', 'SPARQL', ('sparql',), ('*.rq', '*.sparql'), ('application/sparql-query',)), @@ -369,12 +393,14 @@ LEXERS = { 'SquidConfLexer': ('pygments.lexers.configs', 'SquidConf', ('squidconf', 'squid.conf', 'squid'), ('squid.conf',), ('text/x-squidconf',)), 'SspLexer': ('pygments.lexers.templates', 'Scalate Server Page', ('ssp',), ('*.ssp',), ('application/x-ssp',)), 'StanLexer': ('pygments.lexers.modeling', 'Stan', ('stan',), ('*.stan',), ()), + 'StataLexer': ('pygments.lexers.stata', 'Stata', ('stata', 'do'), ('*.do', '*.ado'), ('text/x-stata', 'text/stata', 'application/x-stata')), 'SuperColliderLexer': ('pygments.lexers.supercollider', 'SuperCollider', ('sc', 'supercollider'), ('*.sc', '*.scd'), ('application/supercollider', 'text/supercollider')), 'SwiftLexer': ('pygments.lexers.objective', 'Swift', ('swift',), ('*.swift',), ('text/x-swift',)), 'SwigLexer': ('pygments.lexers.c_like', 'SWIG', ('swig',), ('*.swg', '*.i'), ('text/swig',)), 'SystemVerilogLexer': ('pygments.lexers.hdl', 'systemverilog', ('systemverilog', 'sv'), ('*.sv', '*.svh'), ('text/x-systemverilog',)), 'TAPLexer': ('pygments.lexers.testing', 'TAP', ('tap',), ('*.tap',), ()), 'Tads3Lexer': ('pygments.lexers.int_fiction', 'TADS 3', ('tads3',), ('*.t',), ()), + 'TasmLexer': ('pygments.lexers.asm', 'TASM', ('tasm',), ('*.asm', '*.ASM', '*.tasm'), ('text/x-tasm',)), 'TclLexer': ('pygments.lexers.tcl', 'Tcl', ('tcl',), ('*.tcl', '*.rvt'), ('text/x-tcl', 'text/x-script.tcl', 'application/x-tcl')), 'TcshLexer': ('pygments.lexers.shell', 'Tcsh', ('tcsh', 'csh'), ('*.tcsh', '*.csh'), ('application/x-csh',)), 'TcshSessionLexer': ('pygments.lexers.shell', 'Tcsh Session', ('tcshcon',), (), ()), @@ -386,12 +412,18 @@ LEXERS = { 'TextLexer': ('pygments.lexers.special', 'Text only', ('text',), ('*.txt',), ('text/plain',)), 'ThriftLexer': ('pygments.lexers.dsls', 'Thrift', ('thrift',), ('*.thrift',), ('application/x-thrift',)), 'TodotxtLexer': ('pygments.lexers.textfmts', 'Todotxt', ('todotxt',), ('todo.txt', '*.todotxt'), ('text/x-todo',)), + 'TransactSqlLexer': ('pygments.lexers.sql', 'Transact-SQL', ('tsql', 't-sql'), ('*.sql',), ('text/x-tsql',)), 'TreetopLexer': ('pygments.lexers.parsers', 'Treetop', ('treetop',), ('*.treetop', '*.tt'), ()), 'TurtleLexer': ('pygments.lexers.rdf', 'Turtle', ('turtle',), ('*.ttl',), ('text/turtle', 'application/x-turtle')), 'TwigHtmlLexer': ('pygments.lexers.templates', 'HTML+Twig', ('html+twig',), ('*.twig',), ('text/html+twig',)), 'TwigLexer': ('pygments.lexers.templates', 'Twig', ('twig',), (), ('application/x-twig',)), 'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts', 'typescript'), ('*.ts',), ('text/x-typescript',)), + 'TypoScriptCssDataLexer': ('pygments.lexers.typoscript', 'TypoScriptCssData', ('typoscriptcssdata',), (), ()), + 'TypoScriptHtmlDataLexer': ('pygments.lexers.typoscript', 'TypoScriptHtmlData', ('typoscripthtmldata',), (), ()), + 'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.ts', '*.txt'), ('text/x-typoscript',)), 'UrbiscriptLexer': ('pygments.lexers.urbi', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)), + 'VCLLexer': ('pygments.lexers.varnish', 'VCL', ('vcl',), ('*.vcl',), ('text/x-vclsrc',)), + 'VCLSnippetLexer': ('pygments.lexers.varnish', 'VCLSnippets', ('vclsnippets', 'vclsnippet'), (), ('text/x-vclsnippet',)), 'VCTreeStatusLexer': ('pygments.lexers.console', 'VCTreeStatus', ('vctreestatus',), (), ()), 'VGLLexer': ('pygments.lexers.dsls', 'VGL', ('vgl',), ('*.rpf',), ()), 'ValaLexer': ('pygments.lexers.c_like', 'Vala', ('vala', 'vapi'), ('*.vala', '*.vapi'), ('text/x-vala',)), @@ -403,6 +435,8 @@ LEXERS = { 'VerilogLexer': ('pygments.lexers.hdl', 'verilog', ('verilog', 'v'), ('*.v',), ('text/x-verilog',)), 'VhdlLexer': ('pygments.lexers.hdl', 'vhdl', ('vhdl',), ('*.vhdl', '*.vhd'), ('text/x-vhdl',)), 'VimLexer': ('pygments.lexers.textedit', 'VimL', ('vim',), ('*.vim', '.vimrc', '.exrc', '.gvimrc', '_vimrc', '_exrc', '_gvimrc', 'vimrc', 'gvimrc'), ('text/x-vim',)), + 'WDiffLexer': ('pygments.lexers.diff', 'WDiff', ('wdiff',), ('*.wdiff',), ()), + 'WhileyLexer': ('pygments.lexers.whiley', 'Whiley', ('whiley',), ('*.whiley',), ('text/x-whiley',)), 'X10Lexer': ('pygments.lexers.x10', 'X10', ('x10', 'xten'), ('*.x10',), ('text/x-x10',)), 'XQueryLexer': ('pygments.lexers.webmisc', 'XQuery', ('xquery', 'xqy', 'xq', 'xql', 'xqm'), ('*.xqy', '*.xquery', '*.xq', '*.xql', '*.xqm'), ('text/xquery', 'application/xquery')), 'XmlDjangoLexer': ('pygments.lexers.templates', 'XML+Django/Jinja', ('xml+django', 'xml+jinja'), (), ('application/xml+django', 'application/xml+jinja')), @@ -412,6 +446,7 @@ LEXERS = { 'XmlSmartyLexer': ('pygments.lexers.templates', 'XML+Smarty', ('xml+smarty',), (), ('application/xml+smarty',)), 'XsltLexer': ('pygments.lexers.html', 'XSLT', ('xslt',), ('*.xsl', '*.xslt', '*.xpl'), ('application/xsl+xml', 'application/xslt+xml')), 'XtendLexer': ('pygments.lexers.jvm', 'Xtend', ('xtend',), ('*.xtend',), ('text/x-xtend',)), + 'XtlangLexer': ('pygments.lexers.lisp', 'xtlang', ('extempore',), ('*.xtm',), ()), 'YamlJinjaLexer': ('pygments.lexers.templates', 'YAML+Jinja', ('yaml+jinja', 'salt', 'sls'), ('*.sls',), ('text/x-yaml+jinja', 'text/x-sls')), 'YamlLexer': ('pygments.lexers.data', 'YAML', ('yaml',), ('*.yaml', '*.yml'), ('text/x-yaml',)), 'ZephirLexer': ('pygments.lexers.php', 'Zephir', ('zephir',), ('*.zep',), ()), diff --git a/pygments/lexers/_mql_builtins.py b/pygments/lexers/_mql_builtins.py index 524a2ea2..6eb600c4 100644 --- a/pygments/lexers/_mql_builtins.py +++ b/pygments/lexers/_mql_builtins.py @@ -5,7 +5,7 @@ Builtins for the MqlLexer. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ types = ( diff --git a/pygments/lexers/_openedge_builtins.py b/pygments/lexers/_openedge_builtins.py index 46b6cc42..0fa7d1b2 100644 --- a/pygments/lexers/_openedge_builtins.py +++ b/pygments/lexers/_openedge_builtins.py @@ -5,7 +5,7 @@ Builtin list for the OpenEdgeLexer. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_php_builtins.py b/pygments/lexers/_php_builtins.py index f1b64ced..fec3286a 100644 --- a/pygments/lexers/_php_builtins.py +++ b/pygments/lexers/_php_builtins.py @@ -12,7 +12,7 @@ internet connection. don't run that at home, use a server ;-) - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_postgres_builtins.py b/pygments/lexers/_postgres_builtins.py index 671fa677..a71360f0 100644 --- a/pygments/lexers/_postgres_builtins.py +++ b/pygments/lexers/_postgres_builtins.py @@ -5,7 +5,7 @@ Self-updating data files for PostgreSQL lexer. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_scilab_builtins.py b/pygments/lexers/_scilab_builtins.py index 85c99966..ce0ac67d 100644 --- a/pygments/lexers/_scilab_builtins.py +++ b/pygments/lexers/_scilab_builtins.py @@ -5,7 +5,7 @@ Builtin list for the ScilabLexer. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_sourcemod_builtins.py b/pygments/lexers/_sourcemod_builtins.py index 9ebb1595..f08ea481 100644 --- a/pygments/lexers/_sourcemod_builtins.py +++ b/pygments/lexers/_sourcemod_builtins.py @@ -8,7 +8,7 @@ Do not edit the FUNCTIONS list by hand. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_stan_builtins.py b/pygments/lexers/_stan_builtins.py index 6585ad71..a189647a 100644 --- a/pygments/lexers/_stan_builtins.py +++ b/pygments/lexers/_stan_builtins.py @@ -6,7 +6,7 @@ This file contains the names of functions for Stan used by ``pygments.lexers.math.StanLexer. This is for Stan language version 2.8.0. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_stata_builtins.py b/pygments/lexers/_stata_builtins.py new file mode 100644 index 00000000..5f5f72a9 --- /dev/null +++ b/pygments/lexers/_stata_builtins.py @@ -0,0 +1,419 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers._stata_builtins + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Builtins for Stata + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + + +builtins_base = ( + "if", "else", "in", "foreach", "for", "forv", "forva", + "forval", "forvalu", "forvalue", "forvalues", "by", "bys", + "bysort", "quietly", "qui", "about", "ac", + "ac_7", "acprplot", "acprplot_7", "adjust", "ado", "adopath", + "adoupdate", "alpha", "ameans", "an", "ano", "anov", "anova", + "anova_estat", "anova_terms", "anovadef", "aorder", "ap", "app", + "appe", "appen", "append", "arch", "arch_dr", "arch_estat", + "arch_p", "archlm", "areg", "areg_p", "args", "arima", + "arima_dr", "arima_estat", "arima_p", "as", "asmprobit", + "asmprobit_estat", "asmprobit_lf", "asmprobit_mfx__dlg", + "asmprobit_p", "ass", "asse", "asser", "assert", "avplot", + "avplot_7", "avplots", "avplots_7", "bcskew0", "bgodfrey", + "binreg", "bip0_lf", "biplot", "bipp_lf", "bipr_lf", + "bipr_p", "biprobit", "bitest", "bitesti", "bitowt", "blogit", + "bmemsize", "boot", "bootsamp", "bootstrap", "bootstrap_8", + "boxco_l", "boxco_p", "boxcox", "boxcox_6", "boxcox_p", + "bprobit", "br", "break", "brier", "bro", "brow", "brows", + "browse", "brr", "brrstat", "bs", "bs_7", "bsampl_w", + "bsample", "bsample_7", "bsqreg", "bstat", "bstat_7", "bstat_8", + "bstrap", "bstrap_7", "ca", "ca_estat", "ca_p", "cabiplot", + "camat", "canon", "canon_8", "canon_8_p", "canon_estat", + "canon_p", "cap", "caprojection", "capt", "captu", "captur", + "capture", "cat", "cc", "cchart", "cchart_7", "cci", + "cd", "censobs_table", "centile", "cf", "char", "chdir", + "checkdlgfiles", "checkestimationsample", "checkhlpfiles", + "checksum", "chelp", "ci", "cii", "cl", "class", "classutil", + "clear", "cli", "clis", "clist", "clo", "clog", "clog_lf", + "clog_p", "clogi", "clogi_sw", "clogit", "clogit_lf", + "clogit_p", "clogitp", "clogl_sw", "cloglog", "clonevar", + "clslistarray", "cluster", "cluster_measures", "cluster_stop", + "cluster_tree", "cluster_tree_8", "clustermat", "cmdlog", + "cnr", "cnre", "cnreg", "cnreg_p", "cnreg_sw", "cnsreg", + "codebook", "collaps4", "collapse", "colormult_nb", + "colormult_nw", "compare", "compress", "conf", "confi", + "confir", "confirm", "conren", "cons", "const", "constr", + "constra", "constrai", "constrain", "constraint", "continue", + "contract", "copy", "copyright", "copysource", "cor", "corc", + "corr", "corr2data", "corr_anti", "corr_kmo", "corr_smc", + "corre", "correl", "correla", "correlat", "correlate", + "corrgram", "cou", "coun", "count", "cox", "cox_p", "cox_sw", + "coxbase", "coxhaz", "coxvar", "cprplot", "cprplot_7", + "crc", "cret", "cretu", "cretur", "creturn", "cross", "cs", + "cscript", "cscript_log", "csi", "ct", "ct_is", "ctset", + "ctst_5", "ctst_st", "cttost", "cumsp", "cumsp_7", "cumul", + "cusum", "cusum_7", "cutil", "d", "datasig", "datasign", + "datasigna", "datasignat", "datasignatu", "datasignatur", + "datasignature", "datetof", "db", "dbeta", "de", "dec", + "deco", "decod", "decode", "deff", "des", "desc", "descr", + "descri", "describ", "describe", "destring", "dfbeta", + "dfgls", "dfuller", "di", "di_g", "dir", "dirstats", "dis", + "discard", "disp", "disp_res", "disp_s", "displ", "displa", + "display", "distinct", "do", "doe", "doed", "doedi", + "doedit", "dotplot", "dotplot_7", "dprobit", "drawnorm", + "drop", "ds", "ds_util", "dstdize", "duplicates", "durbina", + "dwstat", "dydx", "e", "ed", "edi", "edit", "egen", + "eivreg", "emdef", "en", "enc", "enco", "encod", "encode", + "eq", "erase", "ereg", "ereg_lf", "ereg_p", "ereg_sw", + "ereghet", "ereghet_glf", "ereghet_glf_sh", "ereghet_gp", + "ereghet_ilf", "ereghet_ilf_sh", "ereghet_ip", "eret", + "eretu", "eretur", "ereturn", "err", "erro", "error", "est", + "est_cfexist", "est_cfname", "est_clickable", "est_expand", + "est_hold", "est_table", "est_unhold", "est_unholdok", + "estat", "estat_default", "estat_summ", "estat_vce_only", + "esti", "estimates", "etodow", "etof", "etomdy", "ex", + "exi", "exit", "expand", "expandcl", "fac", "fact", "facto", + "factor", "factor_estat", "factor_p", "factor_pca_rotated", + "factor_rotate", "factormat", "fcast", "fcast_compute", + "fcast_graph", "fdades", "fdadesc", "fdadescr", "fdadescri", + "fdadescrib", "fdadescribe", "fdasav", "fdasave", "fdause", + "fh_st", "open", "read", "close", + "file", "filefilter", "fillin", "find_hlp_file", "findfile", + "findit", "findit_7", "fit", "fl", "fli", "flis", "flist", + "for5_0", "form", "forma", "format", "fpredict", "frac_154", + "frac_adj", "frac_chk", "frac_cox", "frac_ddp", "frac_dis", + "frac_dv", "frac_in", "frac_mun", "frac_pp", "frac_pq", + "frac_pv", "frac_wgt", "frac_xo", "fracgen", "fracplot", + "fracplot_7", "fracpoly", "fracpred", "fron_ex", "fron_hn", + "fron_p", "fron_tn", "fron_tn2", "frontier", "ftodate", "ftoe", + "ftomdy", "ftowdate", "g", "gamhet_glf", "gamhet_gp", + "gamhet_ilf", "gamhet_ip", "gamma", "gamma_d2", "gamma_p", + "gamma_sw", "gammahet", "gdi_hexagon", "gdi_spokes", "ge", + "gen", "gene", "gener", "genera", "generat", "generate", + "genrank", "genstd", "genvmean", "gettoken", "gl", "gladder", + "gladder_7", "glim_l01", "glim_l02", "glim_l03", "glim_l04", + "glim_l05", "glim_l06", "glim_l07", "glim_l08", "glim_l09", + "glim_l10", "glim_l11", "glim_l12", "glim_lf", "glim_mu", + "glim_nw1", "glim_nw2", "glim_nw3", "glim_p", "glim_v1", + "glim_v2", "glim_v3", "glim_v4", "glim_v5", "glim_v6", + "glim_v7", "glm", "glm_6", "glm_p", "glm_sw", "glmpred", "glo", + "glob", "globa", "global", "glogit", "glogit_8", "glogit_p", + "gmeans", "gnbre_lf", "gnbreg", "gnbreg_5", "gnbreg_p", + "gomp_lf", "gompe_sw", "gomper_p", "gompertz", "gompertzhet", + "gomphet_glf", "gomphet_glf_sh", "gomphet_gp", "gomphet_ilf", + "gomphet_ilf_sh", "gomphet_ip", "gphdot", "gphpen", + "gphprint", "gprefs", "gprobi_p", "gprobit", "gprobit_8", "gr", + "gr7", "gr_copy", "gr_current", "gr_db", "gr_describe", + "gr_dir", "gr_draw", "gr_draw_replay", "gr_drop", "gr_edit", + "gr_editviewopts", "gr_example", "gr_example2", "gr_export", + "gr_print", "gr_qscheme", "gr_query", "gr_read", "gr_rename", + "gr_replay", "gr_save", "gr_set", "gr_setscheme", "gr_table", + "gr_undo", "gr_use", "graph", "graph7", "grebar", "greigen", + "greigen_7", "greigen_8", "grmeanby", "grmeanby_7", + "gs_fileinfo", "gs_filetype", "gs_graphinfo", "gs_stat", + "gsort", "gwood", "h", "hadimvo", "hareg", "hausman", + "haver", "he", "heck_d2", "heckma_p", "heckman", "heckp_lf", + "heckpr_p", "heckprob", "hel", "help", "hereg", "hetpr_lf", + "hetpr_p", "hetprob", "hettest", "hexdump", "hilite", + "hist", "hist_7", "histogram", "hlogit", "hlu", "hmeans", + "hotel", "hotelling", "hprobit", "hreg", "hsearch", "icd9", + "icd9_ff", "icd9p", "iis", "impute", "imtest", "inbase", + "include", "inf", "infi", "infil", "infile", "infix", "inp", + "inpu", "input", "ins", "insheet", "insp", "inspe", + "inspec", "inspect", "integ", "inten", "intreg", "intreg_7", + "intreg_p", "intrg2_ll", "intrg_ll", "intrg_ll2", "ipolate", + "iqreg", "ir", "irf", "irf_create", "irfm", "iri", "is_svy", + "is_svysum", "isid", "istdize", "ivprob_1_lf", "ivprob_lf", + "ivprobit", "ivprobit_p", "ivreg", "ivreg_footnote", + "ivtob_1_lf", "ivtob_lf", "ivtobit", "ivtobit_p", "jackknife", + "jacknife", "jknife", "jknife_6", "jknife_8", "jkstat", + "joinby", "kalarma1", "kap", "kap_3", "kapmeier", "kappa", + "kapwgt", "kdensity", "kdensity_7", "keep", "ksm", "ksmirnov", + "ktau", "kwallis", "l", "la", "lab", "labe", "label", + "labelbook", "ladder", "levels", "levelsof", "leverage", + "lfit", "lfit_p", "li", "lincom", "line", "linktest", + "lis", "list", "lloghet_glf", "lloghet_glf_sh", "lloghet_gp", + "lloghet_ilf", "lloghet_ilf_sh", "lloghet_ip", "llogi_sw", + "llogis_p", "llogist", "llogistic", "llogistichet", + "lnorm_lf", "lnorm_sw", "lnorma_p", "lnormal", "lnormalhet", + "lnormhet_glf", "lnormhet_glf_sh", "lnormhet_gp", + "lnormhet_ilf", "lnormhet_ilf_sh", "lnormhet_ip", "lnskew0", + "loadingplot", "loc", "loca", "local", "log", "logi", + "logis_lf", "logistic", "logistic_p", "logit", "logit_estat", + "logit_p", "loglogs", "logrank", "loneway", "lookfor", + "lookup", "lowess", "lowess_7", "lpredict", "lrecomp", "lroc", + "lroc_7", "lrtest", "ls", "lsens", "lsens_7", "lsens_x", + "lstat", "ltable", "ltable_7", "ltriang", "lv", "lvr2plot", + "lvr2plot_7", "m", "ma", "mac", "macr", "macro", "makecns", + "man", "manova", "manova_estat", "manova_p", "manovatest", + "mantel", "mark", "markin", "markout", "marksample", "mat", + "mat_capp", "mat_order", "mat_put_rr", "mat_rapp", "mata", + "mata_clear", "mata_describe", "mata_drop", "mata_matdescribe", + "mata_matsave", "mata_matuse", "mata_memory", "mata_mlib", + "mata_mosave", "mata_rename", "mata_which", "matalabel", + "matcproc", "matlist", "matname", "matr", "matri", + "matrix", "matrix_input__dlg", "matstrik", "mcc", "mcci", + "md0_", "md1_", "md1debug_", "md2_", "md2debug_", "mds", + "mds_estat", "mds_p", "mdsconfig", "mdslong", "mdsmat", + "mdsshepard", "mdytoe", "mdytof", "me_derd", "mean", + "means", "median", "memory", "memsize", "meqparse", "mer", + "merg", "merge", "mfp", "mfx", "mhelp", "mhodds", "minbound", + "mixed_ll", "mixed_ll_reparm", "mkassert", "mkdir", + "mkmat", "mkspline", "ml", "ml_5", "ml_adjs", "ml_bhhhs", + "ml_c_d", "ml_check", "ml_clear", "ml_cnt", "ml_debug", + "ml_defd", "ml_e0", "ml_e0_bfgs", "ml_e0_cycle", "ml_e0_dfp", + "ml_e0i", "ml_e1", "ml_e1_bfgs", "ml_e1_bhhh", "ml_e1_cycle", + "ml_e1_dfp", "ml_e2", "ml_e2_cycle", "ml_ebfg0", "ml_ebfr0", + "ml_ebfr1", "ml_ebh0q", "ml_ebhh0", "ml_ebhr0", "ml_ebr0i", + "ml_ecr0i", "ml_edfp0", "ml_edfr0", "ml_edfr1", "ml_edr0i", + "ml_eds", "ml_eer0i", "ml_egr0i", "ml_elf", "ml_elf_bfgs", + "ml_elf_bhhh", "ml_elf_cycle", "ml_elf_dfp", "ml_elfi", + "ml_elfs", "ml_enr0i", "ml_enrr0", "ml_erdu0", "ml_erdu0_bfgs", + "ml_erdu0_bhhh", "ml_erdu0_bhhhq", "ml_erdu0_cycle", + "ml_erdu0_dfp", "ml_erdu0_nrbfgs", "ml_exde", "ml_footnote", + "ml_geqnr", "ml_grad0", "ml_graph", "ml_hbhhh", "ml_hd0", + "ml_hold", "ml_init", "ml_inv", "ml_log", "ml_max", + "ml_mlout", "ml_mlout_8", "ml_model", "ml_nb0", "ml_opt", + "ml_p", "ml_plot", "ml_query", "ml_rdgrd", "ml_repor", + "ml_s_e", "ml_score", "ml_searc", "ml_technique", "ml_unhold", + "mleval", "mlf_", "mlmatbysum", "mlmatsum", "mlog", "mlogi", + "mlogit", "mlogit_footnote", "mlogit_p", "mlopts", "mlsum", + "mlvecsum", "mnl0_", "mor", "more", "mov", "move", "mprobit", + "mprobit_lf", "mprobit_p", "mrdu0_", "mrdu1_", "mvdecode", + "mvencode", "mvreg", "mvreg_estat", "n", "nbreg", + "nbreg_al", "nbreg_lf", "nbreg_p", "nbreg_sw", "nestreg", "net", + "newey", "newey_7", "newey_p", "news", "nl", "nl_7", "nl_9", + "nl_9_p", "nl_p", "nl_p_7", "nlcom", "nlcom_p", "nlexp2", + "nlexp2_7", "nlexp2a", "nlexp2a_7", "nlexp3", "nlexp3_7", + "nlgom3", "nlgom3_7", "nlgom4", "nlgom4_7", "nlinit", "nllog3", + "nllog3_7", "nllog4", "nllog4_7", "nlog_rd", "nlogit", + "nlogit_p", "nlogitgen", "nlogittree", "nlpred", "no", + "nobreak", "noi", "nois", "noisi", "noisil", "noisily", "note", + "notes", "notes_dlg", "nptrend", "numlabel", "numlist", "odbc", + "old_ver", "olo", "olog", "ologi", "ologi_sw", "ologit", + "ologit_p", "ologitp", "on", "one", "onew", "onewa", "oneway", + "op_colnm", "op_comp", "op_diff", "op_inv", "op_str", "opr", + "opro", "oprob", "oprob_sw", "oprobi", "oprobi_p", "oprobit", + "oprobitp", "opts_exclusive", "order", "orthog", "orthpoly", + "ou", "out", "outf", "outfi", "outfil", "outfile", "outs", + "outsh", "outshe", "outshee", "outsheet", "ovtest", "pac", + "pac_7", "palette", "parse", "parse_dissim", "pause", "pca", + "pca_8", "pca_display", "pca_estat", "pca_p", "pca_rotate", + "pcamat", "pchart", "pchart_7", "pchi", "pchi_7", "pcorr", + "pctile", "pentium", "pergram", "pergram_7", "permute", + "permute_8", "personal", "peto_st", "pkcollapse", "pkcross", + "pkequiv", "pkexamine", "pkexamine_7", "pkshape", "pksumm", + "pksumm_7", "pl", "plo", "plot", "plugin", "pnorm", + "pnorm_7", "poisgof", "poiss_lf", "poiss_sw", "poisso_p", + "poisson", "poisson_estat", "post", "postclose", "postfile", + "postutil", "pperron", "pr", "prais", "prais_e", "prais_e2", + "prais_p", "predict", "predictnl", "preserve", "print", + "pro", "prob", "probi", "probit", "probit_estat", "probit_p", + "proc_time", "procoverlay", "procrustes", "procrustes_estat", + "procrustes_p", "profiler", "prog", "progr", "progra", + "program", "prop", "proportion", "prtest", "prtesti", "pwcorr", + "pwd", "q", "s", "qby", "qbys", "qchi", "qchi_7", "qladder", + "qladder_7", "qnorm", "qnorm_7", "qqplot", "qqplot_7", "qreg", + "qreg_c", "qreg_p", "qreg_sw", "qu", "quadchk", "quantile", + "quantile_7", "que", "quer", "query", "range", "ranksum", + "ratio", "rchart", "rchart_7", "rcof", "recast", "reclink", + "recode", "reg", "reg3", "reg3_p", "regdw", "regr", "regre", + "regre_p2", "regres", "regres_p", "regress", "regress_estat", + "regriv_p", "remap", "ren", "rena", "renam", "rename", + "renpfix", "repeat", "replace", "report", "reshape", + "restore", "ret", "retu", "retur", "return", "rm", "rmdir", + "robvar", "roccomp", "roccomp_7", "roccomp_8", "rocf_lf", + "rocfit", "rocfit_8", "rocgold", "rocplot", "rocplot_7", + "roctab", "roctab_7", "rolling", "rologit", "rologit_p", + "rot", "rota", "rotat", "rotate", "rotatemat", "rreg", + "rreg_p", "ru", "run", "runtest", "rvfplot", "rvfplot_7", + "rvpplot", "rvpplot_7", "sa", "safesum", "sample", + "sampsi", "sav", "save", "savedresults", "saveold", "sc", + "sca", "scal", "scala", "scalar", "scatter", "scm_mine", + "sco", "scob_lf", "scob_p", "scobi_sw", "scobit", "scor", + "score", "scoreplot", "scoreplot_help", "scree", "screeplot", + "screeplot_help", "sdtest", "sdtesti", "se", "search", + "separate", "seperate", "serrbar", "serrbar_7", "serset", "set", + "set_defaults", "sfrancia", "sh", "she", "shel", "shell", + "shewhart", "shewhart_7", "signestimationsample", "signrank", + "signtest", "simul", "simul_7", "simulate", "simulate_8", + "sktest", "sleep", "slogit", "slogit_d2", "slogit_p", "smooth", + "snapspan", "so", "sor", "sort", "spearman", "spikeplot", + "spikeplot_7", "spikeplt", "spline_x", "split", "sqreg", + "sqreg_p", "sret", "sretu", "sretur", "sreturn", "ssc", "st", + "st_ct", "st_hc", "st_hcd", "st_hcd_sh", "st_is", "st_issys", + "st_note", "st_promo", "st_set", "st_show", "st_smpl", + "st_subid", "stack", "statsby", "statsby_8", "stbase", "stci", + "stci_7", "stcox", "stcox_estat", "stcox_fr", "stcox_fr_ll", + "stcox_p", "stcox_sw", "stcoxkm", "stcoxkm_7", "stcstat", + "stcurv", "stcurve", "stcurve_7", "stdes", "stem", "stepwise", + "stereg", "stfill", "stgen", "stir", "stjoin", "stmc", "stmh", + "stphplot", "stphplot_7", "stphtest", "stphtest_7", + "stptime", "strate", "strate_7", "streg", "streg_sw", "streset", + "sts", "sts_7", "stset", "stsplit", "stsum", "sttocc", + "sttoct", "stvary", "stweib", "su", "suest", "suest_8", + "sum", "summ", "summa", "summar", "summari", "summariz", + "summarize", "sunflower", "sureg", "survcurv", "survsum", + "svar", "svar_p", "svmat", "svy", "svy_disp", "svy_dreg", + "svy_est", "svy_est_7", "svy_estat", "svy_get", "svy_gnbreg_p", + "svy_head", "svy_header", "svy_heckman_p", "svy_heckprob_p", + "svy_intreg_p", "svy_ivreg_p", "svy_logistic_p", "svy_logit_p", + "svy_mlogit_p", "svy_nbreg_p", "svy_ologit_p", "svy_oprobit_p", + "svy_poisson_p", "svy_probit_p", "svy_regress_p", "svy_sub", + "svy_sub_7", "svy_x", "svy_x_7", "svy_x_p", "svydes", + "svydes_8", "svygen", "svygnbreg", "svyheckman", "svyheckprob", + "svyintreg", "svyintreg_7", "svyintrg", "svyivreg", "svylc", + "svylog_p", "svylogit", "svymarkout", "svymarkout_8", + "svymean", "svymlog", "svymlogit", "svynbreg", "svyolog", + "svyologit", "svyoprob", "svyoprobit", "svyopts", + "svypois", "svypois_7", "svypoisson", "svyprobit", "svyprobt", + "svyprop", "svyprop_7", "svyratio", "svyreg", "svyreg_p", + "svyregress", "svyset", "svyset_7", "svyset_8", "svytab", + "svytab_7", "svytest", "svytotal", "sw", "sw_8", "swcnreg", + "swcox", "swereg", "swilk", "swlogis", "swlogit", + "swologit", "swoprbt", "swpois", "swprobit", "swqreg", + "swtobit", "swweib", "symmetry", "symmi", "symplot", + "symplot_7", "syntax", "sysdescribe", "sysdir", "sysuse", + "szroeter", "ta", "tab", "tab1", "tab2", "tab_or", "tabd", + "tabdi", "tabdis", "tabdisp", "tabi", "table", "tabodds", + "tabodds_7", "tabstat", "tabu", "tabul", "tabula", "tabulat", + "tabulate", "te", "tempfile", "tempname", "tempvar", "tes", + "test", "testnl", "testparm", "teststd", "tetrachoric", + "time_it", "timer", "tis", "tob", "tobi", "tobit", "tobit_p", + "tobit_sw", "token", "tokeni", "tokeniz", "tokenize", + "tostring", "total", "translate", "translator", "transmap", + "treat_ll", "treatr_p", "treatreg", "trim", "trnb_cons", + "trnb_mean", "trpoiss_d2", "trunc_ll", "truncr_p", "truncreg", + "tsappend", "tset", "tsfill", "tsline", "tsline_ex", + "tsreport", "tsrevar", "tsrline", "tsset", "tssmooth", + "tsunab", "ttest", "ttesti", "tut_chk", "tut_wait", "tutorial", + "tw", "tware_st", "two", "twoway", "twoway__fpfit_serset", + "twoway__function_gen", "twoway__histogram_gen", + "twoway__ipoint_serset", "twoway__ipoints_serset", + "twoway__kdensity_gen", "twoway__lfit_serset", + "twoway__normgen_gen", "twoway__pci_serset", + "twoway__qfit_serset", "twoway__scatteri_serset", + "twoway__sunflower_gen", "twoway_ksm_serset", "ty", "typ", + "type", "typeof", "u", "unab", "unabbrev", "unabcmd", + "update", "us", "use", "uselabel", "var", "var_mkcompanion", + "var_p", "varbasic", "varfcast", "vargranger", "varirf", + "varirf_add", "varirf_cgraph", "varirf_create", "varirf_ctable", + "varirf_describe", "varirf_dir", "varirf_drop", "varirf_erase", + "varirf_graph", "varirf_ograph", "varirf_rename", "varirf_set", + "varirf_table", "varlist", "varlmar", "varnorm", "varsoc", + "varstable", "varstable_w", "varstable_w2", "varwle", + "vce", "vec", "vec_fevd", "vec_mkphi", "vec_p", "vec_p_w", + "vecirf_create", "veclmar", "veclmar_w", "vecnorm", + "vecnorm_w", "vecrank", "vecstable", "verinst", "vers", + "versi", "versio", "version", "view", "viewsource", "vif", + "vwls", "wdatetof", "webdescribe", "webseek", "webuse", + "weib1_lf", "weib2_lf", "weib_lf", "weib_lf0", "weibhet_glf", + "weibhet_glf_sh", "weibhet_glfa", "weibhet_glfa_sh", + "weibhet_gp", "weibhet_ilf", "weibhet_ilf_sh", "weibhet_ilfa", + "weibhet_ilfa_sh", "weibhet_ip", "weibu_sw", "weibul_p", + "weibull", "weibull_c", "weibull_s", "weibullhet", + "wh", "whelp", "whi", "which", "whil", "while", "wilc_st", + "wilcoxon", "win", "wind", "windo", "window", "winexec", + "wntestb", "wntestb_7", "wntestq", "xchart", "xchart_7", + "xcorr", "xcorr_7", "xi", "xi_6", "xmlsav", "xmlsave", + "xmluse", "xpose", "xsh", "xshe", "xshel", "xshell", + "xt_iis", "xt_tis", "xtab_p", "xtabond", "xtbin_p", + "xtclog", "xtcloglog", "xtcloglog_8", "xtcloglog_d2", + "xtcloglog_pa_p", "xtcloglog_re_p", "xtcnt_p", "xtcorr", + "xtdata", "xtdes", "xtfront_p", "xtfrontier", "xtgee", + "xtgee_elink", "xtgee_estat", "xtgee_makeivar", "xtgee_p", + "xtgee_plink", "xtgls", "xtgls_p", "xthaus", "xthausman", + "xtht_p", "xthtaylor", "xtile", "xtint_p", "xtintreg", + "xtintreg_8", "xtintreg_d2", "xtintreg_p", "xtivp_1", + "xtivp_2", "xtivreg", "xtline", "xtline_ex", "xtlogit", + "xtlogit_8", "xtlogit_d2", "xtlogit_fe_p", "xtlogit_pa_p", + "xtlogit_re_p", "xtmixed", "xtmixed_estat", "xtmixed_p", + "xtnb_fe", "xtnb_lf", "xtnbreg", "xtnbreg_pa_p", + "xtnbreg_refe_p", "xtpcse", "xtpcse_p", "xtpois", "xtpoisson", + "xtpoisson_d2", "xtpoisson_pa_p", "xtpoisson_refe_p", "xtpred", + "xtprobit", "xtprobit_8", "xtprobit_d2", "xtprobit_re_p", + "xtps_fe", "xtps_lf", "xtps_ren", "xtps_ren_8", "xtrar_p", + "xtrc", "xtrc_p", "xtrchh", "xtrefe_p", "xtreg", "xtreg_be", + "xtreg_fe", "xtreg_ml", "xtreg_pa_p", "xtreg_re", + "xtregar", "xtrere_p", "xtset", "xtsf_ll", "xtsf_llti", + "xtsum", "xttab", "xttest0", "xttobit", "xttobit_8", + "xttobit_p", "xttrans", "yx", "yxview__barlike_draw", + "yxview_area_draw", "yxview_bar_draw", "yxview_dot_draw", + "yxview_dropline_draw", "yxview_function_draw", + "yxview_iarrow_draw", "yxview_ilabels_draw", + "yxview_normal_draw", "yxview_pcarrow_draw", + "yxview_pcbarrow_draw", "yxview_pccapsym_draw", + "yxview_pcscatter_draw", "yxview_pcspike_draw", + "yxview_rarea_draw", "yxview_rbar_draw", "yxview_rbarm_draw", + "yxview_rcap_draw", "yxview_rcapsym_draw", + "yxview_rconnected_draw", "yxview_rline_draw", + "yxview_rscatter_draw", "yxview_rspike_draw", + "yxview_spike_draw", "yxview_sunflower_draw", "zap_s", "zinb", + "zinb_llf", "zinb_plf", "zip", "zip_llf", "zip_p", "zip_plf", + "zt_ct_5", "zt_hc_5", "zt_hcd_5", "zt_is_5", "zt_iss_5", + "zt_sho_5", "zt_smp_5", "ztbase_5", "ztcox_5", "ztdes_5", + "ztereg_5", "ztfill_5", "ztgen_5", "ztir_5", "ztjoin_5", "ztnb", + "ztnb_p", "ztp", "ztp_p", "zts_5", "ztset_5", "ztspli_5", + "ztsum_5", "zttoct_5", "ztvary_5", "ztweib_5" +) + +builtins_functions = ( + "Cdhms", "Chms", "Clock", "Cmdyhms", "Cofc", "Cofd", "F", + "Fden", "Ftail", "I", "J", "_caller", "abbrev", "abs", "acos", + "acosh", "asin", "asinh", "atan", "atan2", "atanh", + "autocode", "betaden", "binomial", "binomialp", "binomialtail", + "binormal", "bofd", "byteorder", "c", "ceil", "char", + "chi2", "chi2den", "chi2tail", "cholesky", "chop", "clip", + "clock", "cloglog", "cofC", "cofd", "colnumb", "colsof", "comb", + "cond", "corr", "cos", "cosh", "d", "daily", "date", "day", + "det", "dgammapda", "dgammapdada", "dgammapdadx", "dgammapdx", + "dgammapdxdx", "dhms", "diag", "diag0cnt", "digamma", + "dofC", "dofb", "dofc", "dofh", "dofm", "dofq", "dofw", + "dofy", "dow", "doy", "dunnettprob", "e", "el", "epsdouble", + "epsfloat", "exp", "fileexists", "fileread", "filereaderror", + "filewrite", "float", "floor", "fmtwidth", "gammaden", + "gammap", "gammaptail", "get", "group", "h", "hadamard", + "halfyear", "halfyearly", "has_eprop", "hh", "hhC", "hms", + "hofd", "hours", "hypergeometric", "hypergeometricp", "ibeta", + "ibetatail", "index", "indexnot", "inlist", "inrange", "int", + "inv", "invF", "invFtail", "invbinomial", "invbinomialtail", + "invchi2", "invchi2tail", "invcloglog", "invdunnettprob", + "invgammap", "invgammaptail", "invibeta", "invibetatail", + "invlogit", "invnFtail", "invnbinomial", "invnbinomialtail", + "invnchi2", "invnchi2tail", "invnibeta", "invnorm", "invnormal", + "invnttail", "invpoisson", "invpoissontail", "invsym", "invt", + "invttail", "invtukeyprob", "irecode", "issym", "issymmetric", + "itrim", "length", "ln", "lnfact", "lnfactorial", "lngamma", + "lnnormal", "lnnormalden", "log", "log10", "logit", "lower", + "ltrim", "m", "match", "matmissing", "matrix", "matuniform", + "max", "maxbyte", "maxdouble", "maxfloat", "maxint", "maxlong", + "mdy", "mdyhms", "mi", "min", "minbyte", "mindouble", + "minfloat", "minint", "minlong", "minutes", "missing", "mm", + "mmC", "mod", "mofd", "month", "monthly", "mreldif", + "msofhours", "msofminutes", "msofseconds", "nF", "nFden", + "nFtail", "nbetaden", "nbinomial", "nbinomialp", "nbinomialtail", + "nchi2", "nchi2den", "nchi2tail", "nibeta", "norm", "normal", + "normalden", "normd", "npnF", "npnchi2", "npnt", "nt", "ntden", + "nttail", "nullmat", "plural", "poisson", "poissonp", + "poissontail", "proper", "q", "qofd", "quarter", "quarterly", + "r", "rbeta", "rbinomial", "rchi2", "real", "recode", "regexm", + "regexr", "regexs", "reldif", "replay", "return", "reverse", + "rgamma", "rhypergeometric", "rnbinomial", "rnormal", "round", + "rownumb", "rowsof", "rpoisson", "rt", "rtrim", "runiform", "s", + "scalar", "seconds", "sign", "sin", "sinh", "smallestdouble", + "soundex", "soundex_nara", "sqrt", "ss", "ssC", "strcat", + "strdup", "string", "strlen", "strlower", "strltrim", "strmatch", + "strofreal", "strpos", "strproper", "strreverse", "strrtrim", + "strtoname", "strtrim", "strupper", "subinstr", "subinword", + "substr", "sum", "sweep", "syminv", "t", "tC", "tan", "tanh", + "tc", "td", "tden", "th", "tin", "tm", "tq", "trace", + "trigamma", "trim", "trunc", "ttail", "tukeyprob", "tw", + "twithin", "uniform", "upper", "vec", "vecdiag", "w", "week", + "weekly", "wofd", "word", "wordcount", "year", "yearly", + "yh", "ym", "yofd", "yq", "yw" +) + + diff --git a/pygments/lexers/_tsql_builtins.py b/pygments/lexers/_tsql_builtins.py new file mode 100644 index 00000000..e29ed34b --- /dev/null +++ b/pygments/lexers/_tsql_builtins.py @@ -0,0 +1,1004 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers._tsql_builtins + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + These are manually translated lists from https://msdn.microsoft.com. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +# See https://msdn.microsoft.com/en-us/library/ms174986.aspx. +OPERATORS = ( + '!<', + '!=', + '!>', + '<', + '<=', + '<>', + '=', + '>', + '>=', + '+', + '+=', + '-', + '-=', + '*', + '*=', + '/', + '/=', + '%', + '%=', + '&', + '&=', + '|', + '|=', + '^', + '^=', + '~', + '::', +) + +OPERATOR_WORDS = ( + 'all', + 'and', + 'any', + 'between', + 'except', + 'exists', + 'in', + 'intersect', + 'like', + 'not', + 'or', + 'some', + 'union', +) + +_KEYWORDS_SERVER = ( + 'add', + 'all', + 'alter', + 'and', + 'any', + 'as', + 'asc', + 'authorization', + 'backup', + 'begin', + 'between', + 'break', + 'browse', + 'bulk', + 'by', + 'cascade', + 'case', + 'catch', + 'check', + 'checkpoint', + 'close', + 'clustered', + 'coalesce', + 'collate', + 'column', + 'commit', + 'compute', + 'constraint', + 'contains', + 'containstable', + 'continue', + 'convert', + 'create', + 'cross', + 'current', + 'current_date', + 'current_time', + 'current_timestamp', + 'current_user', + 'cursor', + 'database', + 'dbcc', + 'deallocate', + 'declare', + 'default', + 'delete', + 'deny', + 'desc', + 'disk', + 'distinct', + 'distributed', + 'double', + 'drop', + 'dump', + 'else', + 'end', + 'errlvl', + 'escape', + 'except', + 'exec', + 'execute', + 'exists', + 'exit', + 'external', + 'fetch', + 'file', + 'fillfactor', + 'for', + 'foreign', + 'freetext', + 'freetexttable', + 'from', + 'full', + 'function', + 'goto', + 'grant', + 'group', + 'having', + 'holdlock', + 'identity', + 'identity_insert', + 'identitycol', + 'if', + 'in', + 'index', + 'inner', + 'insert', + 'intersect', + 'into', + 'is', + 'join', + 'key', + 'kill', + 'left', + 'like', + 'lineno', + 'load', + 'merge', + 'national', + 'nocheck', + 'nonclustered', + 'not', + 'null', + 'nullif', + 'of', + 'off', + 'offsets', + 'on', + 'open', + 'opendatasource', + 'openquery', + 'openrowset', + 'openxml', + 'option', + 'or', + 'order', + 'outer', + 'over', + 'percent', + 'pivot', + 'plan', + 'precision', + 'primary', + 'print', + 'proc', + 'procedure', + 'public', + 'raiserror', + 'read', + 'readtext', + 'reconfigure', + 'references', + 'replication', + 'restore', + 'restrict', + 'return', + 'revert', + 'revoke', + 'right', + 'rollback', + 'rowcount', + 'rowguidcol', + 'rule', + 'save', + 'schema', + 'securityaudit', + 'select', + 'semantickeyphrasetable', + 'semanticsimilaritydetailstable', + 'semanticsimilaritytable', + 'session_user', + 'set', + 'setuser', + 'shutdown', + 'some', + 'statistics', + 'system_user', + 'table', + 'tablesample', + 'textsize', + 'then', + 'throw', + 'to', + 'top', + 'tran', + 'transaction', + 'trigger', + 'truncate', + 'try', + 'try_convert', + 'tsequal', + 'union', + 'unique', + 'unpivot', + 'update', + 'updatetext', + 'use', + 'user', + 'values', + 'varying', + 'view', + 'waitfor', + 'when', + 'where', + 'while', + 'with', + 'within', + 'writetext', +) + +_KEYWORDS_FUTURE = ( + 'absolute', + 'action', + 'admin', + 'after', + 'aggregate', + 'alias', + 'allocate', + 'are', + 'array', + 'asensitive', + 'assertion', + 'asymmetric', + 'at', + 'atomic', + 'before', + 'binary', + 'bit', + 'blob', + 'boolean', + 'both', + 'breadth', + 'call', + 'called', + 'cardinality', + 'cascaded', + 'cast', + 'catalog', + 'char', + 'character', + 'class', + 'clob', + 'collation', + 'collect', + 'completion', + 'condition', + 'connect', + 'connection', + 'constraints', + 'constructor', + 'corr', + 'corresponding', + 'covar_pop', + 'covar_samp', + 'cube', + 'cume_dist', + 'current_catalog', + 'current_default_transform_group', + 'current_path', + 'current_role', + 'current_schema', + 'current_transform_group_for_type', + 'cycle', + 'data', + 'date', + 'day', + 'dec', + 'decimal', + 'deferrable', + 'deferred', + 'depth', + 'deref', + 'describe', + 'descriptor', + 'destroy', + 'destructor', + 'deterministic', + 'diagnostics', + 'dictionary', + 'disconnect', + 'domain', + 'dynamic', + 'each', + 'element', + 'end-exec', + 'equals', + 'every', + 'exception', + 'false', + 'filter', + 'first', + 'float', + 'found', + 'free', + 'fulltexttable', + 'fusion', + 'general', + 'get', + 'global', + 'go', + 'grouping', + 'hold', + 'host', + 'hour', + 'ignore', + 'immediate', + 'indicator', + 'initialize', + 'initially', + 'inout', + 'input', + 'int', + 'integer', + 'intersection', + 'interval', + 'isolation', + 'iterate', + 'language', + 'large', + 'last', + 'lateral', + 'leading', + 'less', + 'level', + 'like_regex', + 'limit', + 'ln', + 'local', + 'localtime', + 'localtimestamp', + 'locator', + 'map', + 'match', + 'member', + 'method', + 'minute', + 'mod', + 'modifies', + 'modify', + 'module', + 'month', + 'multiset', + 'names', + 'natural', + 'nchar', + 'nclob', + 'new', + 'next', + 'no', + 'none', + 'normalize', + 'numeric', + 'object', + 'occurrences_regex', + 'old', + 'only', + 'operation', + 'ordinality', + 'out', + 'output', + 'overlay', + 'pad', + 'parameter', + 'parameters', + 'partial', + 'partition', + 'path', + 'percent_rank', + 'percentile_cont', + 'percentile_disc', + 'position_regex', + 'postfix', + 'prefix', + 'preorder', + 'prepare', + 'preserve', + 'prior', + 'privileges', + 'range', + 'reads', + 'real', + 'recursive', + 'ref', + 'referencing', + 'regr_avgx', + 'regr_avgy', + 'regr_count', + 'regr_intercept', + 'regr_r2', + 'regr_slope', + 'regr_sxx', + 'regr_sxy', + 'regr_syy', + 'relative', + 'release', + 'result', + 'returns', + 'role', + 'rollup', + 'routine', + 'row', + 'rows', + 'savepoint', + 'scope', + 'scroll', + 'search', + 'second', + 'section', + 'sensitive', + 'sequence', + 'session', + 'sets', + 'similar', + 'size', + 'smallint', + 'space', + 'specific', + 'specifictype', + 'sql', + 'sqlexception', + 'sqlstate', + 'sqlwarning', + 'start', + 'state', + 'statement', + 'static', + 'stddev_pop', + 'stddev_samp', + 'structure', + 'submultiset', + 'substring_regex', + 'symmetric', + 'system', + 'temporary', + 'terminate', + 'than', + 'time', + 'timestamp', + 'timezone_hour', + 'timezone_minute', + 'trailing', + 'translate_regex', + 'translation', + 'treat', + 'true', + 'uescape', + 'under', + 'unknown', + 'unnest', + 'usage', + 'using', + 'value', + 'var_pop', + 'var_samp', + 'varchar', + 'variable', + 'whenever', + 'width_bucket', + 'window', + 'within', + 'without', + 'work', + 'write', + 'xmlagg', + 'xmlattributes', + 'xmlbinary', + 'xmlcast', + 'xmlcomment', + 'xmlconcat', + 'xmldocument', + 'xmlelement', + 'xmlexists', + 'xmlforest', + 'xmliterate', + 'xmlnamespaces', + 'xmlparse', + 'xmlpi', + 'xmlquery', + 'xmlserialize', + 'xmltable', + 'xmltext', + 'xmlvalidate', + 'year', + 'zone', +) + +_KEYWORDS_ODBC = ( + 'absolute', + 'action', + 'ada', + 'add', + 'all', + 'allocate', + 'alter', + 'and', + 'any', + 'are', + 'as', + 'asc', + 'assertion', + 'at', + 'authorization', + 'avg', + 'begin', + 'between', + 'bit', + 'bit_length', + 'both', + 'by', + 'cascade', + 'cascaded', + 'case', + 'cast', + 'catalog', + 'char', + 'char_length', + 'character', + 'character_length', + 'check', + 'close', + 'coalesce', + 'collate', + 'collation', + 'column', + 'commit', + 'connect', + 'connection', + 'constraint', + 'constraints', + 'continue', + 'convert', + 'corresponding', + 'count', + 'create', + 'cross', + 'current', + 'current_date', + 'current_time', + 'current_timestamp', + 'current_user', + 'cursor', + 'date', + 'day', + 'deallocate', + 'dec', + 'decimal', + 'declare', + 'default', + 'deferrable', + 'deferred', + 'delete', + 'desc', + 'describe', + 'descriptor', + 'diagnostics', + 'disconnect', + 'distinct', + 'domain', + 'double', + 'drop', + 'else', + 'end', + 'end-exec', + 'escape', + 'except', + 'exception', + 'exec', + 'execute', + 'exists', + 'external', + 'extract', + 'false', + 'fetch', + 'first', + 'float', + 'for', + 'foreign', + 'fortran', + 'found', + 'from', + 'full', + 'get', + 'global', + 'go', + 'goto', + 'grant', + 'group', + 'having', + 'hour', + 'identity', + 'immediate', + 'in', + 'include', + 'index', + 'indicator', + 'initially', + 'inner', + 'input', + 'insensitive', + 'insert', + 'int', + 'integer', + 'intersect', + 'interval', + 'into', + 'is', + 'isolation', + 'join', + 'key', + 'language', + 'last', + 'leading', + 'left', + 'level', + 'like', + 'local', + 'lower', + 'match', + 'max', + 'min', + 'minute', + 'module', + 'month', + 'names', + 'national', + 'natural', + 'nchar', + 'next', + 'no', + 'none', + 'not', + 'null', + 'nullif', + 'numeric', + 'octet_length', + 'of', + 'on', + 'only', + 'open', + 'option', + 'or', + 'order', + 'outer', + 'output', + 'overlaps', + 'pad', + 'partial', + 'pascal', + 'position', + 'precision', + 'prepare', + 'preserve', + 'primary', + 'prior', + 'privileges', + 'procedure', + 'public', + 'read', + 'real', + 'references', + 'relative', + 'restrict', + 'revoke', + 'right', + 'rollback', + 'rows', + 'schema', + 'scroll', + 'second', + 'section', + 'select', + 'session', + 'session_user', + 'set', + 'size', + 'smallint', + 'some', + 'space', + 'sql', + 'sqlca', + 'sqlcode', + 'sqlerror', + 'sqlstate', + 'sqlwarning', + 'substring', + 'sum', + 'system_user', + 'table', + 'temporary', + 'then', + 'time', + 'timestamp', + 'timezone_hour', + 'timezone_minute', + 'to', + 'trailing', + 'transaction', + 'translate', + 'translation', + 'trim', + 'true', + 'union', + 'unique', + 'unknown', + 'update', + 'upper', + 'usage', + 'user', + 'using', + 'value', + 'values', + 'varchar', + 'varying', + 'view', + 'when', + 'whenever', + 'where', + 'with', + 'work', + 'write', + 'year', + 'zone', +) + +# See https://msdn.microsoft.com/en-us/library/ms189822.aspx. +KEYWORDS = sorted(set(_KEYWORDS_FUTURE + _KEYWORDS_ODBC + _KEYWORDS_SERVER)) + +# See https://msdn.microsoft.com/en-us/library/ms187752.aspx. +TYPES = ( + 'bigint', + 'binary', + 'bit', + 'char', + 'cursor', + 'date', + 'datetime', + 'datetime2', + 'datetimeoffset', + 'decimal', + 'float', + 'hierarchyid', + 'image', + 'int', + 'money', + 'nchar', + 'ntext', + 'numeric', + 'nvarchar', + 'real', + 'smalldatetime', + 'smallint', + 'smallmoney', + 'sql_variant', + 'table', + 'text', + 'time', + 'timestamp', + 'tinyint', + 'uniqueidentifier', + 'varbinary', + 'varchar', + 'xml', +) + +# See https://msdn.microsoft.com/en-us/library/ms174318.aspx. +FUNCTIONS = ( + '$partition', + 'abs', + 'acos', + 'app_name', + 'applock_mode', + 'applock_test', + 'ascii', + 'asin', + 'assemblyproperty', + 'atan', + 'atn2', + 'avg', + 'binary_checksum', + 'cast', + 'ceiling', + 'certencoded', + 'certprivatekey', + 'char', + 'charindex', + 'checksum', + 'checksum_agg', + 'choose', + 'col_length', + 'col_name', + 'columnproperty', + 'compress', + 'concat', + 'connectionproperty', + 'context_info', + 'convert', + 'cos', + 'cot', + 'count', + 'count_big', + 'current_request_id', + 'current_timestamp', + 'current_transaction_id', + 'current_user', + 'cursor_status', + 'database_principal_id', + 'databasepropertyex', + 'dateadd', + 'datediff', + 'datediff_big', + 'datefromparts', + 'datename', + 'datepart', + 'datetime2fromparts', + 'datetimefromparts', + 'datetimeoffsetfromparts', + 'day', + 'db_id', + 'db_name', + 'decompress', + 'degrees', + 'dense_rank', + 'difference', + 'eomonth', + 'error_line', + 'error_message', + 'error_number', + 'error_procedure', + 'error_severity', + 'error_state', + 'exp', + 'file_id', + 'file_idex', + 'file_name', + 'filegroup_id', + 'filegroup_name', + 'filegroupproperty', + 'fileproperty', + 'floor', + 'format', + 'formatmessage', + 'fulltextcatalogproperty', + 'fulltextserviceproperty', + 'get_filestream_transaction_context', + 'getansinull', + 'getdate', + 'getutcdate', + 'grouping', + 'grouping_id', + 'has_perms_by_name', + 'host_id', + 'host_name', + 'iif', + 'index_col', + 'indexkey_property', + 'indexproperty', + 'is_member', + 'is_rolemember', + 'is_srvrolemember', + 'isdate', + 'isjson', + 'isnull', + 'isnumeric', + 'json_modify', + 'json_query', + 'json_value', + 'left', + 'len', + 'log', + 'log10', + 'lower', + 'ltrim', + 'max', + 'min', + 'min_active_rowversion', + 'month', + 'nchar', + 'newid', + 'newsequentialid', + 'ntile', + 'object_definition', + 'object_id', + 'object_name', + 'object_schema_name', + 'objectproperty', + 'objectpropertyex', + 'opendatasource', + 'openjson', + 'openquery', + 'openrowset', + 'openxml', + 'original_db_name', + 'original_login', + 'parse', + 'parsename', + 'patindex', + 'permissions', + 'pi', + 'power', + 'pwdcompare', + 'pwdencrypt', + 'quotename', + 'radians', + 'rand', + 'rank', + 'replace', + 'replicate', + 'reverse', + 'right', + 'round', + 'row_number', + 'rowcount_big', + 'rtrim', + 'schema_id', + 'schema_name', + 'scope_identity', + 'serverproperty', + 'session_context', + 'session_user', + 'sign', + 'sin', + 'smalldatetimefromparts', + 'soundex', + 'sp_helplanguage', + 'space', + 'sqrt', + 'square', + 'stats_date', + 'stdev', + 'stdevp', + 'str', + 'string_escape', + 'string_split', + 'stuff', + 'substring', + 'sum', + 'suser_id', + 'suser_name', + 'suser_sid', + 'suser_sname', + 'switchoffset', + 'sysdatetime', + 'sysdatetimeoffset', + 'system_user', + 'sysutcdatetime', + 'tan', + 'textptr', + 'textvalid', + 'timefromparts', + 'todatetimeoffset', + 'try_cast', + 'try_convert', + 'try_parse', + 'type_id', + 'type_name', + 'typeproperty', + 'unicode', + 'upper', + 'user_id', + 'user_name', + 'var', + 'varp', + 'xact_state', + 'year', +) diff --git a/pygments/lexers/_vim_builtins.py b/pygments/lexers/_vim_builtins.py index e9b5fa1e..82586289 100644 --- a/pygments/lexers/_vim_builtins.py +++ b/pygments/lexers/_vim_builtins.py @@ -5,7 +5,7 @@ This file is autogenerated by scripts/get_vimkw.py - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/actionscript.py b/pygments/lexers/actionscript.py index 9c687a57..84607e68 100644 --- a/pygments/lexers/actionscript.py +++ b/pygments/lexers/actionscript.py @@ -5,7 +5,7 @@ Lexers for ActionScript and MXML. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py index defa7b6e..cb200b9e 100644 --- a/pygments/lexers/agile.py +++ b/pygments/lexers/agile.py @@ -5,7 +5,7 @@ Just export lexer classes previously contained in this module. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/algebra.py b/pygments/lexers/algebra.py index fc54c3c3..15d68842 100644 --- a/pygments/lexers/algebra.py +++ b/pygments/lexers/algebra.py @@ -5,7 +5,7 @@ Lexers for computer algebra systems. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -104,9 +104,9 @@ class MathematicaLexer(RegexLexer): (r'#\d*', Name.Variable), (r'([a-zA-Z]+[a-zA-Z0-9]*)', Name), - (r'-?[0-9]+\.[0-9]*', Number.Float), - (r'-?[0-9]*\.[0-9]+', Number.Float), - (r'-?[0-9]+', Number.Integer), + (r'-?\d+\.\d*', Number.Float), + (r'-?\d*\.\d+', Number.Float), + (r'-?\d+', Number.Integer), (words(operators), Operator), (words(punctuation), Punctuation), diff --git a/pygments/lexers/ambient.py b/pygments/lexers/ambient.py index 7f622fbc..53f3a5e1 100644 --- a/pygments/lexers/ambient.py +++ b/pygments/lexers/ambient.py @@ -5,7 +5,7 @@ Lexers for AmbientTalk language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/ampl.py b/pygments/lexers/ampl.py new file mode 100644 index 00000000..d439cb19 --- /dev/null +++ b/pygments/lexers/ampl.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.ampl + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for the ampl language. <http://ampl.com/> + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, using, this, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['AmplLexer'] + + +class AmplLexer(RegexLexer): + """ + For AMPL source code. + + .. versionadded:: 2.2 + """ + name = 'Ampl' + aliases = ['ampl'] + filenames = ['*.run'] + + tokens = { + 'root': [ + (r'\n', Text), + (r'\s+', Text.Whitespace), + (r'#.*?\n', Comment.Single), + (r'/[*](.|\n)*?[*]/', Comment.Multiline), + (words(( + 'call', 'cd', 'close', 'commands', 'data', 'delete', 'display', + 'drop', 'end', 'environ', 'exit', 'expand', 'include', 'load', + 'model', 'objective', 'option', 'problem', 'purge', 'quit', + 'redeclare', 'reload', 'remove', 'reset', 'restore', 'shell', + 'show', 'solexpand', 'solution', 'solve', 'update', 'unload', + 'xref', 'coeff', 'coef', 'cover', 'obj', 'interval', 'default', + 'from', 'to', 'to_come', 'net_in', 'net_out', 'dimen', + 'dimension', 'check', 'complements', 'write', 'function', + 'pipe', 'format', 'if', 'then', 'else', 'in', 'while', 'repeat', + 'for'), suffix=r'\b'), Keyword.Reserved), + (r'(integer|binary|symbolic|ordered|circular|reversed|INOUT|IN|OUT|LOCAL)', + Keyword.Type), + (r'\".*?\"', String.Double), + (r'\'.*?\'', String.Single), + (r'[()\[\]{},;:]+', Punctuation), + (r'\b(\w+)(\.)(astatus|init0|init|lb0|lb1|lb2|lb|lrc|' + r'lslack|rc|relax|slack|sstatus|status|ub0|ub1|ub2|' + r'ub|urc|uslack|val)', + bygroups(Name.Variable, Punctuation, Keyword.Reserved)), + (r'(set|param|var|arc|minimize|maximize|subject to|s\.t\.|subj to|' + r'node|table|suffix|read table|write table)(\s+)(\w+)', + bygroups(Keyword.Declaration, Text, Name.Variable)), + (r'(param)(\s*)(:)(\s*)(\w+)(\s*)(:)(\s*)((\w|\s)+)', + bygroups(Keyword.Declaration, Text, Punctuation, Text, + Name.Variable, Text, Punctuation, Text, Name.Variable)), + (r'(let|fix|unfix)(\s*)((?:\{.*\})?)(\s*)(\w+)', + bygroups(Keyword.Declaration, Text, using(this), Text, Name.Variable)), + (words(( + 'abs', 'acos', 'acosh', 'alias', 'asin', 'asinh', 'atan', 'atan2', + 'atanh', 'ceil', 'ctime', 'cos', 'exp', 'floor', 'log', 'log10', + 'max', 'min', 'precision', 'round', 'sin', 'sinh', 'sqrt', 'tan', + 'tanh', 'time', 'trunc', 'Beta', 'Cauchy', 'Exponential', 'Gamma', + 'Irand224', 'Normal', 'Normal01', 'Poisson', 'Uniform', 'Uniform01', + 'num', 'num0', 'ichar', 'char', 'length', 'substr', 'sprintf', + 'match', 'sub', 'gsub', 'print', 'printf', 'next', 'nextw', 'prev', + 'prevw', 'first', 'last', 'ord', 'ord0', 'card', 'arity', + 'indexarity'), prefix=r'\b', suffix=r'\b'), Name.Builtin), + (r'(\+|\-|\*|/|\*\*|=|<=|>=|==|\||\^|<|>|\!|\.\.|:=|\&|\!=|<<|>>)', + Operator), + (words(( + 'or', 'exists', 'forall', 'and', 'in', 'not', 'within', 'union', + 'diff', 'difference', 'symdiff', 'inter', 'intersect', + 'intersection', 'cross', 'setof', 'by', 'less', 'sum', 'prod', + 'product', 'div', 'mod'), suffix=r'\b'), + Keyword.Reserved), # Operator.Name but not enough emphasized with that + (r'(\d+\.(?!\.)\d*|\.(?!.)\d+)([eE][+-]?\d+)?', Number.Float), + (r'\d+([eE][+-]?\d+)?', Number.Integer), + (r'[+-]?Infinity', Number.Integer), + (r'(\w+|(\.(?!\.)))', Text) + ] + + } diff --git a/pygments/lexers/apl.py b/pygments/lexers/apl.py index 61ea4c4b..b3414cc0 100644 --- a/pygments/lexers/apl.py +++ b/pygments/lexers/apl.py @@ -5,7 +5,7 @@ Lexers for APL. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/archetype.py b/pygments/lexers/archetype.py index e596b7be..5d4eb9aa 100644 --- a/pygments/lexers/archetype.py +++ b/pygments/lexers/archetype.py @@ -14,7 +14,7 @@ Contributed by Thomas Beale <https://github.com/wolandscat>, <https://bitbucket.org/thomas_beale>. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py index bbe04f69..9c58478e 100644 --- a/pygments/lexers/asm.py +++ b/pygments/lexers/asm.py @@ -5,21 +5,22 @@ Lexers for assembly languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re -from pygments.lexer import RegexLexer, include, bygroups, using, DelegatingLexer +from pygments.lexer import RegexLexer, include, bygroups, using, words, \ + DelegatingLexer from pygments.lexers.c_cpp import CppLexer, CLexer from pygments.lexers.d import DLexer from pygments.token import Text, Name, Number, String, Comment, Punctuation, \ Other, Keyword, Operator __all__ = ['GasLexer', 'ObjdumpLexer', 'DObjdumpLexer', 'CppObjdumpLexer', - 'CObjdumpLexer', 'LlvmLexer', 'NasmLexer', 'NasmObjdumpLexer', - 'Ca65Lexer'] + 'CObjdumpLexer', 'HsailLexer', 'LlvmLexer', 'NasmLexer', + 'NasmObjdumpLexer', 'TasmLexer', 'Ca65Lexer'] class GasLexer(RegexLexer): @@ -53,8 +54,6 @@ class GasLexer(RegexLexer): (number, Number.Integer), (r'[\r\n]+', Text, '#pop'), - (r'#.*?$', Comment, '#pop'), - include('punctuation'), include('whitespace') ], @@ -77,14 +76,14 @@ class GasLexer(RegexLexer): ('$'+number, Number.Integer), (r"$'(.|\\')'", String.Char), (r'[\r\n]+', Text, '#pop'), - (r'#.*?$', Comment, '#pop'), + include('punctuation'), include('whitespace') ], 'whitespace': [ (r'\n', Text), (r'\s+', Text), - (r'#.*?\n', Comment) + (r'[;#].*?\n', Comment) ], 'punctuation': [ (r'[-*,.()\[\]!:]+', Punctuation) @@ -198,6 +197,141 @@ class CObjdumpLexer(DelegatingLexer): super(CObjdumpLexer, self).__init__(CLexer, ObjdumpLexer, **options) +class HsailLexer(RegexLexer): + """ + For HSAIL assembly code. + + .. versionadded:: 2.2 + """ + name = 'HSAIL' + aliases = ['hsail', 'hsa'] + filenames = ['*.hsail'] + mimetypes = ['text/x-hsail'] + + string = r'"[^"]*?"' + identifier = r'[a-zA-Z_][\w.]*' + # Registers + register_number = r'[0-9]+' + register = r'(\$(c|s|d|q)' + register_number + ')' + # Qualifiers + alignQual = r'(align\(\d+\))' + widthQual = r'(width\((\d+|all)\))' + allocQual = r'(alloc\(agent\))' + # Instruction Modifiers + roundingMod = (r'((_ftz)?(_up|_down|_zero|_near))') + datatypeMod = (r'_(' + # packedTypes + r'u8x4|s8x4|u16x2|s16x2|u8x8|s8x8|u16x4|s16x4|u32x2|s32x2|' + r'u8x16|s8x16|u16x8|s16x8|u32x4|s32x4|u64x2|s64x2|' + r'f16x2|f16x4|f16x8|f32x2|f32x4|f64x2|' + # baseTypes + r'u8|s8|u16|s16|u32|s32|u64|s64|' + r'b128|b8|b16|b32|b64|b1|' + r'f16|f32|f64|' + # opaqueType + r'roimg|woimg|rwimg|samp|sig32|sig64)') + + # Numeric Constant + float = r'((\d+\.)|(\d*\.\d+))[eE][+-]?\d+' + hexfloat = r'0[xX](([0-9a-fA-F]+\.[0-9a-fA-F]*)|([0-9a-fA-F]*\.[0-9a-fA-F]+))[pP][+-]?\d+' + ieeefloat = r'0((h|H)[0-9a-fA-F]{4}|(f|F)[0-9a-fA-F]{8}|(d|D)[0-9a-fA-F]{16})' + + tokens = { + 'root': [ + include('whitespace'), + include('comments'), + + (string, String), + + (r'@' + identifier + ':?', Name.Label), + + (register, Name.Variable.Anonymous), + + include('keyword'), + + (r'&' + identifier, Name.Variable.Global), + (r'%' + identifier, Name.Variable), + + (hexfloat, Number.Hex), + (r'0[xX][a-fA-F0-9]+', Number.Hex), + (ieeefloat, Number.Float), + (float, Number.Float), + ('\d+', Number.Integer), + + (r'[=<>{}\[\]()*.,:;!]|x\b', Punctuation) + ], + 'whitespace': [ + (r'(\n|\s)+', Text), + ], + 'comments': [ + (r'/\*.*?\*/', Comment.Multiline), + (r'//.*?\n', Comment.Singleline), + ], + 'keyword': [ + # Types + (r'kernarg' + datatypeMod, Keyword.Type), + + # Regular keywords + (r'\$(full|base|small|large|default|zero|near)', Keyword), + (words(( + 'module', 'extension', 'pragma', 'prog', 'indirect', 'signature', + 'decl', 'kernel', 'function', 'enablebreakexceptions', + 'enabledetectexceptions', 'maxdynamicgroupsize', 'maxflatgridsize', + 'maxflatworkgroupsize', 'requireddim', 'requiredgridsize', + 'requiredworkgroupsize', 'requirenopartialworkgroups'), + suffix=r'\b'), Keyword), + + # instructions + (roundingMod, Keyword), + (datatypeMod, Keyword), + (r'_(' + alignQual + '|' + widthQual + ')', Keyword), + (r'_kernarg', Keyword), + (r'(nop|imagefence)\b', Keyword), + (words(( + 'cleardetectexcept', 'clock', 'cuid', 'debugtrap', 'dim', + 'getdetectexcept', 'groupbaseptr', 'kernargbaseptr', 'laneid', + 'maxcuid', 'maxwaveid', 'packetid', 'setdetectexcept', 'waveid', + 'workitemflatabsid', 'workitemflatid', 'nullptr', 'abs', 'bitrev', + 'currentworkgroupsize', 'currentworkitemflatid', 'fract', 'ncos', + 'neg', 'nexp2', 'nlog2', 'nrcp', 'nrsqrt', 'nsin', 'nsqrt', + 'gridgroups', 'gridsize', 'not', 'sqrt', 'workgroupid', + 'workgroupsize', 'workitemabsid', 'workitemid', 'ceil', 'floor', + 'rint', 'trunc', 'add', 'bitmask', 'borrow', 'carry', 'copysign', + 'div', 'rem', 'sub', 'shl', 'shr', 'and', 'or', 'xor', 'unpackhi', + 'unpacklo', 'max', 'min', 'fma', 'mad', 'bitextract', 'bitselect', + 'shuffle', 'cmov', 'bitalign', 'bytealign', 'lerp', 'nfma', 'mul', + 'mulhi', 'mul24hi', 'mul24', 'mad24', 'mad24hi', 'bitinsert', + 'combine', 'expand', 'lda', 'mov', 'pack', 'unpack', 'packcvt', + 'unpackcvt', 'sad', 'sementp', 'ftos', 'stof', 'cmp', 'ld', 'st', + '_eq', '_ne', '_lt', '_le', '_gt', '_ge', '_equ', '_neu', '_ltu', + '_leu', '_gtu', '_geu', '_num', '_nan', '_seq', '_sne', '_slt', + '_sle', '_sgt', '_sge', '_snum', '_snan', '_sequ', '_sneu', '_sltu', + '_sleu', '_sgtu', '_sgeu', 'atomic', '_ld', '_st', '_cas', '_add', + '_and', '_exch', '_max', '_min', '_or', '_sub', '_wrapdec', + '_wrapinc', '_xor', 'ret', 'cvt', '_readonly', '_kernarg', '_global', + 'br', 'cbr', 'sbr', '_scacq', '_screl', '_scar', '_rlx', '_wave', + '_wg', '_agent', '_system', 'ldimage', 'stimage', '_v2', '_v3', '_v4', + '_1d', '_2d', '_3d', '_1da', '_2da', '_1db', '_2ddepth', '_2dadepth', + '_width', '_height', '_depth', '_array', '_channelorder', + '_channeltype', 'querysampler', '_coord', '_filter', '_addressing', + 'barrier', 'wavebarrier', 'initfbar', 'joinfbar', 'waitfbar', + 'arrivefbar', 'leavefbar', 'releasefbar', 'ldf', 'activelaneid', + 'activelanecount', 'activelanemask', 'activelanepermute', 'call', + 'scall', 'icall', 'alloca', 'packetcompletionsig', + 'addqueuewriteindex', 'casqueuewriteindex', 'ldqueuereadindex', + 'stqueuereadindex', 'readonly', 'global', 'private', 'group', + 'spill', 'arg', '_upi', '_downi', '_zeroi', '_neari', '_upi_sat', + '_downi_sat', '_zeroi_sat', '_neari_sat', '_supi', '_sdowni', + '_szeroi', '_sneari', '_supi_sat', '_sdowni_sat', '_szeroi_sat', + '_sneari_sat', '_pp', '_ps', '_sp', '_ss', '_s', '_p', '_pp_sat', + '_ps_sat', '_sp_sat', '_ss_sat', '_s_sat', '_p_sat')), Keyword), + + # Integer types + (r'i[1-9]\d*', Keyword) + ] + } + + class LlvmLexer(RegexLexer): """ For LLVM assembly code. @@ -240,69 +374,60 @@ class LlvmLexer(RegexLexer): ], 'keyword': [ # Regular keywords - (r'(begin|end' - r'|true|false' - r'|declare|define' - r'|global|constant' - - r'|private|linker_private|internal|available_externally|linkonce' - r'|linkonce_odr|weak|weak_odr|appending|dllimport|dllexport' - r'|common|default|hidden|protected|extern_weak|external' - r'|thread_local|zeroinitializer|undef|null|to|tail|target|triple' - r'|datalayout|volatile|nuw|nsw|nnan|ninf|nsz|arcp|fast|exact|inbounds' - r'|align|addrspace|section|alias|module|asm|sideeffect|gc|dbg' - r'|linker_private_weak' - r'|attributes|blockaddress|initialexec|localdynamic|localexec' - r'|prefix|unnamed_addr' - - r'|ccc|fastcc|coldcc|x86_stdcallcc|x86_fastcallcc|arm_apcscc' - r'|arm_aapcscc|arm_aapcs_vfpcc|ptx_device|ptx_kernel' - r'|intel_ocl_bicc|msp430_intrcc|spir_func|spir_kernel' - r'|x86_64_sysvcc|x86_64_win64cc|x86_thiscallcc' - - r'|cc|c' - - r'|signext|zeroext|inreg|sret|nounwind|noreturn|noalias|nocapture' - r'|byval|nest|readnone|readonly' - r'|inlinehint|noinline|alwaysinline|optsize|ssp|sspreq|noredzone' - r'|noimplicitfloat|naked' - r'|builtin|cold|nobuiltin|noduplicate|nonlazybind|optnone' - r'|returns_twice|sanitize_address|sanitize_memory|sanitize_thread' - r'|sspstrong|uwtable|returned' - - r'|type|opaque' - - r'|eq|ne|slt|sgt|sle' - r'|sge|ult|ugt|ule|uge' - r'|oeq|one|olt|ogt|ole' - r'|oge|ord|uno|ueq|une' - r'|x' - r'|acq_rel|acquire|alignstack|atomic|catch|cleanup|filter' - r'|inteldialect|max|min|monotonic|nand|personality|release' - r'|seq_cst|singlethread|umax|umin|unordered|xchg' - - # instructions - r'|add|fadd|sub|fsub|mul|fmul|udiv|sdiv|fdiv|urem|srem|frem|shl' - r'|lshr|ashr|and|or|xor|icmp|fcmp' - - r'|phi|call|trunc|zext|sext|fptrunc|fpext|uitofp|sitofp|fptoui' - r'|fptosi|inttoptr|ptrtoint|bitcast|addrspacecast' - r'|select|va_arg|ret|br|switch' - r'|invoke|unwind|unreachable' - r'|indirectbr|landingpad|resume' - - r'|malloc|alloca|free|load|store|getelementptr' - - r'|extractelement|insertelement|shufflevector|getresult' - r'|extractvalue|insertvalue' - - r'|atomicrmw|cmpxchg|fence' - - r')\b', Keyword), + (words(( + 'begin', 'end', 'true', 'false', 'declare', 'define', 'global', + 'constant', 'private', 'linker_private', 'internal', + 'available_externally', 'linkonce', 'linkonce_odr', 'weak', + 'weak_odr', 'appending', 'dllimport', 'dllexport', 'common', + 'default', 'hidden', 'protected', 'extern_weak', 'external', + 'thread_local', 'zeroinitializer', 'undef', 'null', 'to', 'tail', + 'target', 'triple', 'datalayout', 'volatile', 'nuw', 'nsw', 'nnan', + 'ninf', 'nsz', 'arcp', 'fast', 'exact', 'inbounds', 'align', + 'addrspace', 'section', 'alias', 'module', 'asm', 'sideeffect', + 'gc', 'dbg', 'linker_private_weak', 'attributes', 'blockaddress', + 'initialexec', 'localdynamic', 'localexec', 'prefix', 'unnamed_addr', + 'ccc', 'fastcc', 'coldcc', 'x86_stdcallcc', 'x86_fastcallcc', + 'arm_apcscc', 'arm_aapcscc', 'arm_aapcs_vfpcc', 'ptx_device', + 'ptx_kernel', 'intel_ocl_bicc', 'msp430_intrcc', 'spir_func', + 'spir_kernel', 'x86_64_sysvcc', 'x86_64_win64cc', 'x86_thiscallcc', + 'cc', 'c', 'signext', 'zeroext', 'inreg', 'sret', 'nounwind', + 'noreturn', 'noalias', 'nocapture', 'byval', 'nest', 'readnone', + 'readonly', 'inlinehint', 'noinline', 'alwaysinline', 'optsize', 'ssp', + 'sspreq', 'noredzone', 'noimplicitfloat', 'naked', 'builtin', 'cold', + 'nobuiltin', 'noduplicate', 'nonlazybind', 'optnone', 'returns_twice', + 'sanitize_address', 'sanitize_memory', 'sanitize_thread', 'sspstrong', + 'uwtable', 'returned', 'type', 'opaque', 'eq', 'ne', 'slt', 'sgt', + 'sle', 'sge', 'ult', 'ugt', 'ule', 'uge', 'oeq', 'one', 'olt', 'ogt', + 'ole', 'oge', 'ord', 'uno', 'ueq', 'une', 'x', 'acq_rel', 'acquire', + 'alignstack', 'atomic', 'catch', 'cleanup', 'filter', 'inteldialect', + 'max', 'min', 'monotonic', 'nand', 'personality', 'release', 'seq_cst', + 'singlethread', 'umax', 'umin', 'unordered', 'xchg', 'add', 'fadd', + 'sub', 'fsub', 'mul', 'fmul', 'udiv', 'sdiv', 'fdiv', 'urem', 'srem', + 'frem', 'shl', 'lshr', 'ashr', 'and', 'or', 'xor', 'icmp', 'fcmp', + 'phi', 'call', 'trunc', 'zext', 'sext', 'fptrunc', 'fpext', 'uitofp', + 'sitofp', 'fptoui', 'fptosi', 'inttoptr', 'ptrtoint', 'bitcast', + 'addrspacecast', 'select', 'va_arg', 'ret', 'br', 'switch', 'invoke', + 'unwind', 'unreachable', 'indirectbr', 'landingpad', 'resume', + 'malloc', 'alloca', 'free', 'load', 'store', 'getelementptr', + 'extractelement', 'insertelement', 'shufflevector', 'getresult', + 'extractvalue', 'insertvalue', 'atomicrmw', 'cmpxchg', 'fence', + 'allocsize', 'amdgpu_cs', 'amdgpu_gs', 'amdgpu_kernel', 'amdgpu_ps', + 'amdgpu_vs', 'any', 'anyregcc', 'argmemonly', 'avr_intrcc', + 'avr_signalcc', 'caller', 'catchpad', 'catchret', 'catchswitch', + 'cleanuppad', 'cleanupret', 'comdat', 'convergent', 'cxx_fast_tlscc', + 'deplibs', 'dereferenceable', 'dereferenceable_or_null', 'distinct', + 'exactmatch', 'externally_initialized', 'from', 'ghccc', 'hhvm_ccc', + 'hhvmcc', 'ifunc', 'inaccessiblemem_or_argmemonly', 'inaccessiblememonly', + 'inalloca', 'jumptable', 'largest', 'local_unnamed_addr', 'minsize', + 'musttail', 'noduplicates', 'none', 'nonnull', 'norecurse', 'notail', + 'preserve_allcc', 'preserve_mostcc', 'prologue', 'safestack', 'samesize', + 'source_filename', 'swiftcc', 'swifterror', 'swiftself', 'webkit_jscc', + 'within', 'writeonly', 'x86_intrcc', 'x86_vectorcallcc'), + suffix=r'\b'), Keyword), # Types - (r'void|half|float|double|x86_fp80|fp128|ppc_fp128|label|metadata', - Keyword.Type), + (words(('void', 'half', 'float', 'double', 'x86_fp80', 'fp128', + 'ppc_fp128', 'label', 'metadata', 'token')), Keyword.Type), # Integer types (r'i[1-9]\d*', Keyword) @@ -397,6 +522,86 @@ class NasmObjdumpLexer(ObjdumpLexer): tokens = _objdump_lexer_tokens(NasmLexer) +class TasmLexer(RegexLexer): + """ + For Tasm (Turbo Assembler) assembly code. + """ + name = 'TASM' + aliases = ['tasm'] + filenames = ['*.asm', '*.ASM', '*.tasm'] + mimetypes = ['text/x-tasm'] + + identifier = r'[@a-z$._?][\w$.?#@~]*' + hexn = r'(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)' + octn = r'[0-7]+q' + binn = r'[01]+b' + decn = r'[0-9]+' + floatn = decn + r'\.e?' + decn + string = r'"(\\"|[^"\n])*"|' + r"'(\\'|[^'\n])*'|" + r"`(\\`|[^`\n])*`" + declkw = r'(?:res|d)[bwdqt]|times' + register = (r'r[0-9][0-5]?[bwd]|' + r'[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|' + r'mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]') + wordop = r'seg|wrt|strict' + type = r'byte|[dq]?word' + directives = (r'BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|' + r'ORG|ALIGN|STRUC|ENDSTRUC|ENDS|COMMON|CPU|GROUP|UPPERCASE|INCLUDE|' + r'EXPORT|LIBRARY|MODULE|PROC|ENDP|USES|ARG|DATASEG|UDATASEG|END|IDEAL|' + r'P386|MODEL|ASSUME|CODESEG|SIZE') + # T[A-Z][a-z] is more of a convention. Lexer should filter out STRUC definitions + # and then 'add' them to datatype somehow. + datatype = (r'db|dd|dw|T[A-Z][a-z]+') + + flags = re.IGNORECASE | re.MULTILINE + tokens = { + 'root': [ + (r'^\s*%', Comment.Preproc, 'preproc'), + include('whitespace'), + (identifier + ':', Name.Label), + (directives, Keyword, 'instruction-args'), + (r'(%s)(\s+)(%s)' % (identifier, datatype), + bygroups(Name.Constant, Keyword.Declaration, Keyword.Declaration), + 'instruction-args'), + (declkw, Keyword.Declaration, 'instruction-args'), + (identifier, Name.Function, 'instruction-args'), + (r'[\r\n]+', Text) + ], + 'instruction-args': [ + (string, String), + (hexn, Number.Hex), + (octn, Number.Oct), + (binn, Number.Bin), + (floatn, Number.Float), + (decn, Number.Integer), + include('punctuation'), + (register, Name.Builtin), + (identifier, Name.Variable), + # Do not match newline when it's preceeded by a backslash + (r'(\\\s*)(;.*)([\r\n])', bygroups(Text, Comment.Single, Text)), + (r'[\r\n]+', Text, '#pop'), + include('whitespace') + ], + 'preproc': [ + (r'[^;\n]+', Comment.Preproc), + (r';.*?\n', Comment.Single, '#pop'), + (r'\n', Comment.Preproc, '#pop'), + ], + 'whitespace': [ + (r'[\n\r]', Text), + (r'\\[\n\r]', Text), + (r'[ \t]+', Text), + (r';.*', Comment.Single) + ], + 'punctuation': [ + (r'[,():\[\]]+', Punctuation), + (r'[&|^<>+*=/%~-]+', Operator), + (r'[$]+', Keyword.Constant), + (wordop, Operator.Word), + (type, Keyword.Type) + ], + } + + class Ca65Lexer(RegexLexer): """ For ca65 assembler sources. diff --git a/pygments/lexers/automation.py b/pygments/lexers/automation.py index 2ebc4d24..be1ec129 100644 --- a/pygments/lexers/automation.py +++ b/pygments/lexers/automation.py @@ -5,7 +5,7 @@ Lexers for automation scripting languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/basic.py b/pygments/lexers/basic.py index a73ad8b4..e6545ee6 100644 --- a/pygments/lexers/basic.py +++ b/pygments/lexers/basic.py @@ -5,7 +5,7 @@ Lexers for BASIC like languages (other than VB.net). - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/bibtex.py b/pygments/lexers/bibtex.py new file mode 100644 index 00000000..a6159f81 --- /dev/null +++ b/pygments/lexers/bibtex.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.bibtex + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for BibTeX bibliography data and styles + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, ExtendedRegexLexer, include, default, \ + words +from pygments.token import Name, Comment, String, Error, Number, Text, \ + Keyword, Punctuation + +__all__ = ['BibTeXLexer', 'BSTLexer'] + + +class BibTeXLexer(ExtendedRegexLexer): + """ + A lexer for BibTeX bibliography data format. + + .. versionadded:: 2.2 + """ + + name = 'BibTeX' + aliases = ['bib', 'bibtex'] + filenames = ['*.bib'] + mimetypes = ["text/x-bibtex"] + flags = re.IGNORECASE + + ALLOWED_CHARS = r'@!$&*+\-./:;<>?\[\\\]^`|~' + IDENTIFIER = '[{0}][{1}]*'.format('a-z_' + ALLOWED_CHARS, r'\w' + ALLOWED_CHARS) + + def open_brace_callback(self, match, ctx): + opening_brace = match.group() + ctx.opening_brace = opening_brace + yield match.start(), Punctuation, opening_brace + ctx.pos = match.end() + + def close_brace_callback(self, match, ctx): + closing_brace = match.group() + if ( + ctx.opening_brace == '{' and closing_brace != '}' or + ctx.opening_brace == '(' and closing_brace != ')' + ): + yield match.start(), Error, closing_brace + else: + yield match.start(), Punctuation, closing_brace + del ctx.opening_brace + ctx.pos = match.end() + + tokens = { + 'root': [ + include('whitespace'), + ('@comment', Comment), + ('@preamble', Name.Class, ('closing-brace', 'value', 'opening-brace')), + ('@string', Name.Class, ('closing-brace', 'field', 'opening-brace')), + ('@' + IDENTIFIER, Name.Class, + ('closing-brace', 'command-body', 'opening-brace')), + ('.+', Comment), + ], + 'opening-brace': [ + include('whitespace'), + (r'[{(]', open_brace_callback, '#pop'), + ], + 'closing-brace': [ + include('whitespace'), + (r'[})]', close_brace_callback, '#pop'), + ], + 'command-body': [ + include('whitespace'), + (r'[^\s\,\}]+', Name.Label, ('#pop', 'fields')), + ], + 'fields': [ + include('whitespace'), + (',', Punctuation, 'field'), + default('#pop'), + ], + 'field': [ + include('whitespace'), + (IDENTIFIER, Name.Attribute, ('value', '=')), + default('#pop'), + ], + '=': [ + include('whitespace'), + ('=', Punctuation, '#pop'), + ], + 'value': [ + include('whitespace'), + (IDENTIFIER, Name.Variable), + ('"', String, 'quoted-string'), + (r'\{', String, 'braced-string'), + (r'[\d]+', Number), + ('#', Punctuation), + default('#pop'), + ], + 'quoted-string': [ + (r'\{', String, 'braced-string'), + ('"', String, '#pop'), + ('[^\{\"]+', String), + ], + 'braced-string': [ + (r'\{', String, '#push'), + (r'\}', String, '#pop'), + ('[^\{\}]+', String), + ], + 'whitespace': [ + (r'\s+', Text), + ], + } + + +class BSTLexer(RegexLexer): + """ + A lexer for BibTeX bibliography styles. + + .. versionadded:: 2.2 + """ + + name = 'BST' + aliases = ['bst', 'bst-pybtex'] + filenames = ['*.bst'] + flags = re.IGNORECASE | re.MULTILINE + + tokens = { + 'root': [ + include('whitespace'), + (words(['read', 'sort']), Keyword), + (words(['execute', 'integers', 'iterate', 'reverse', 'strings']), + Keyword, ('group')), + (words(['function', 'macro']), Keyword, ('group', 'group')), + (words(['entry']), Keyword, ('group', 'group', 'group')), + ], + 'group': [ + include('whitespace'), + (r'\{', Punctuation, ('#pop', 'group-end', 'body')), + ], + 'group-end': [ + include('whitespace'), + (r'\}', Punctuation, '#pop'), + ], + 'body': [ + include('whitespace'), + (r"\'[^#\"\{\}\s]+", Name.Function), + (r'[^#\"\{\}\s]+\$', Name.Builtin), + (r'[^#\"\{\}\s]+', Name.Variable), + (r'"[^\"]*"', String), + (r'#-?\d+', Number), + (r'\{', Punctuation, ('group-end', 'body')), + default('#pop'), + ], + 'whitespace': [ + ('\s+', Text), + ('%.*?$', Comment.SingleLine), + ], + } diff --git a/pygments/lexers/business.py b/pygments/lexers/business.py index ea888245..552f3d9c 100644 --- a/pygments/lexers/business.py +++ b/pygments/lexers/business.py @@ -5,7 +5,7 @@ Lexers for "business-oriented" languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -57,9 +57,9 @@ class CobolLexer(RegexLexer): ], 'core': [ # Figurative constants - (r'(^|(?<=[^0-9a-z_\-]))(ALL\s+)?' + (r'(^|(?<=[^\w\-]))(ALL\s+)?' r'((ZEROES)|(HIGH-VALUE|LOW-VALUE|QUOTE|SPACE|ZERO)(S)?)' - r'\s*($|(?=[^0-9a-z_\-]))', + r'\s*($|(?=[^\w\-]))', Name.Constant), # Reserved words STATEMENTS and other bolds @@ -79,8 +79,8 @@ class CobolLexer(RegexLexer): 'RETURN', 'REWRITE', 'SCREEN', 'SD', 'SEARCH', 'SECTION', 'SET', 'SORT', 'START', 'STOP', 'STRING', 'SUBTRACT', 'SUPPRESS', 'TERMINATE', 'THEN', 'UNLOCK', 'UNSTRING', 'USE', 'VALIDATE', - 'WORKING-STORAGE', 'WRITE'), prefix=r'(^|(?<=[^0-9a-z_\-]))', - suffix=r'\s*($|(?=[^0-9a-z_\-]))'), + 'WORKING-STORAGE', 'WRITE'), prefix=r'(^|(?<=[^\w\-]))', + suffix=r'\s*($|(?=[^\w\-]))'), Keyword.Reserved), # Reserved words @@ -89,33 +89,33 @@ class CobolLexer(RegexLexer): 'ALPHABET', 'ALPHABETIC', 'ALPHABETIC-LOWER', 'ALPHABETIC-UPPER', 'ALPHANUMERIC', 'ALPHANUMERIC-EDITED', 'ALSO', 'ALTER', 'ALTERNATE' 'ANY', 'ARE', 'AREA', 'AREAS', 'ARGUMENT-NUMBER', 'ARGUMENT-VALUE', 'AS', - 'ASCENDING', 'ASSIGN', 'AT', 'AUTO', 'AUTO-SKIP', 'AUTOMATIC', 'AUTOTERMINATE', - 'BACKGROUND-COLOR', 'BASED', 'BEEP', 'BEFORE', 'BELL', + 'ASCENDING', 'ASSIGN', 'AT', 'AUTO', 'AUTO-SKIP', 'AUTOMATIC', + 'AUTOTERMINATE', 'BACKGROUND-COLOR', 'BASED', 'BEEP', 'BEFORE', 'BELL', 'BLANK', 'BLINK', 'BLOCK', 'BOTTOM', 'BY', 'BYTE-LENGTH', 'CHAINING', - 'CHARACTER', 'CHARACTERS', 'CLASS', 'CODE', 'CODE-SET', 'COL', 'COLLATING', - 'COLS', 'COLUMN', 'COLUMNS', 'COMMA', 'COMMAND-LINE', 'COMMIT', 'COMMON', - 'CONSTANT', 'CONTAINS', 'CONTENT', 'CONTROL', + 'CHARACTER', 'CHARACTERS', 'CLASS', 'CODE', 'CODE-SET', 'COL', + 'COLLATING', 'COLS', 'COLUMN', 'COLUMNS', 'COMMA', 'COMMAND-LINE', + 'COMMIT', 'COMMON', 'CONSTANT', 'CONTAINS', 'CONTENT', 'CONTROL', 'CONTROLS', 'CONVERTING', 'COPY', 'CORR', 'CORRESPONDING', 'COUNT', 'CRT', - 'CURRENCY', 'CURSOR', 'CYCLE', 'DATE', 'DAY', 'DAY-OF-WEEK', 'DE', 'DEBUGGING', - 'DECIMAL-POINT', 'DECLARATIVES', 'DEFAULT', 'DELIMITED', + 'CURRENCY', 'CURSOR', 'CYCLE', 'DATE', 'DAY', 'DAY-OF-WEEK', 'DE', + 'DEBUGGING', 'DECIMAL-POINT', 'DECLARATIVES', 'DEFAULT', 'DELIMITED', 'DELIMITER', 'DEPENDING', 'DESCENDING', 'DETAIL', 'DISK', 'DOWN', 'DUPLICATES', 'DYNAMIC', 'EBCDIC', 'ENTRY', 'ENVIRONMENT-NAME', 'ENVIRONMENT-VALUE', 'EOL', 'EOP', 'EOS', 'ERASE', 'ERROR', 'ESCAPE', 'EXCEPTION', - 'EXCLUSIVE', 'EXTEND', 'EXTERNAL', - 'FILE-ID', 'FILLER', 'FINAL', 'FIRST', 'FIXED', 'FLOAT-LONG', 'FLOAT-SHORT', - 'FOOTING', 'FOR', 'FOREGROUND-COLOR', 'FORMAT', 'FROM', 'FULL', 'FUNCTION', - 'FUNCTION-ID', 'GIVING', 'GLOBAL', 'GROUP', + 'EXCLUSIVE', 'EXTEND', 'EXTERNAL', 'FILE-ID', 'FILLER', 'FINAL', + 'FIRST', 'FIXED', 'FLOAT-LONG', 'FLOAT-SHORT', + 'FOOTING', 'FOR', 'FOREGROUND-COLOR', 'FORMAT', 'FROM', 'FULL', + 'FUNCTION', 'FUNCTION-ID', 'GIVING', 'GLOBAL', 'GROUP', 'HEADING', 'HIGHLIGHT', 'I-O', 'ID', 'IGNORE', 'IGNORING', 'IN', 'INDEX', 'INDEXED', 'INDICATE', - 'INITIAL', 'INITIALIZED', 'INPUT', - 'INTO', 'INTRINSIC', 'INVALID', 'IS', 'JUST', 'JUSTIFIED', 'KEY', 'LABEL', + 'INITIAL', 'INITIALIZED', 'INPUT', 'INTO', 'INTRINSIC', 'INVALID', + 'IS', 'JUST', 'JUSTIFIED', 'KEY', 'LABEL', 'LAST', 'LEADING', 'LEFT', 'LENGTH', 'LIMIT', 'LIMITS', 'LINAGE', 'LINAGE-COUNTER', 'LINE', 'LINES', 'LOCALE', 'LOCK', - 'LOWLIGHT', 'MANUAL', 'MEMORY', 'MINUS', 'MODE', - 'MULTIPLE', 'NATIONAL', 'NATIONAL-EDITED', 'NATIVE', - 'NEGATIVE', 'NEXT', 'NO', 'NULL', 'NULLS', 'NUMBER', 'NUMBERS', 'NUMERIC', - 'NUMERIC-EDITED', 'OBJECT-COMPUTER', 'OCCURS', 'OF', 'OFF', 'OMITTED', 'ON', 'ONLY', + 'LOWLIGHT', 'MANUAL', 'MEMORY', 'MINUS', 'MODE', 'MULTIPLE', + 'NATIONAL', 'NATIONAL-EDITED', 'NATIVE', 'NEGATIVE', 'NEXT', 'NO', + 'NULL', 'NULLS', 'NUMBER', 'NUMBERS', 'NUMERIC', 'NUMERIC-EDITED', + 'OBJECT-COMPUTER', 'OCCURS', 'OF', 'OFF', 'OMITTED', 'ON', 'ONLY', 'OPTIONAL', 'ORDER', 'ORGANIZATION', 'OTHER', 'OUTPUT', 'OVERFLOW', 'OVERLINE', 'PACKED-DECIMAL', 'PADDING', 'PAGE', 'PARAGRAPH', 'PLUS', 'POINTER', 'POSITION', 'POSITIVE', 'PRESENT', 'PREVIOUS', @@ -137,40 +137,42 @@ class CobolLexer(RegexLexer): 'UNSIGNED-INT', 'UNSIGNED-LONG', 'UNSIGNED-SHORT', 'UNTIL', 'UP', 'UPDATE', 'UPON', 'USAGE', 'USING', 'VALUE', 'VALUES', 'VARYING', 'WAIT', 'WHEN', 'WITH', 'WORDS', 'YYYYDDD', 'YYYYMMDD'), - prefix=r'(^|(?<=[^0-9a-z_\-]))', suffix=r'\s*($|(?=[^0-9a-z_\-]))'), + prefix=r'(^|(?<=[^\w\-]))', suffix=r'\s*($|(?=[^\w\-]))'), Keyword.Pseudo), # inactive reserved words (words(( - 'ACTIVE-CLASS', 'ALIGNED', 'ANYCASE', 'ARITHMETIC', 'ATTRIBUTE', 'B-AND', - 'B-NOT', 'B-OR', 'B-XOR', 'BIT', 'BOOLEAN', 'CD', 'CENTER', 'CF', 'CH', 'CHAIN', 'CLASS-ID', - 'CLASSIFICATION', 'COMMUNICATION', 'CONDITION', 'DATA-POINTER', - 'DESTINATION', 'DISABLE', 'EC', 'EGI', 'EMI', 'ENABLE', 'END-RECEIVE', - 'ENTRY-CONVENTION', 'EO', 'ESI', 'EXCEPTION-OBJECT', 'EXPANDS', 'FACTORY', - 'FLOAT-BINARY-16', 'FLOAT-BINARY-34', 'FLOAT-BINARY-7', - 'FLOAT-DECIMAL-16', 'FLOAT-DECIMAL-34', 'FLOAT-EXTENDED', 'FORMAT', - 'FUNCTION-POINTER', 'GET', 'GROUP-USAGE', 'IMPLEMENTS', 'INFINITY', - 'INHERITS', 'INTERFACE', 'INTERFACE-ID', 'INVOKE', 'LC_ALL', 'LC_COLLATE', + 'ACTIVE-CLASS', 'ALIGNED', 'ANYCASE', 'ARITHMETIC', 'ATTRIBUTE', + 'B-AND', 'B-NOT', 'B-OR', 'B-XOR', 'BIT', 'BOOLEAN', 'CD', 'CENTER', + 'CF', 'CH', 'CHAIN', 'CLASS-ID', 'CLASSIFICATION', 'COMMUNICATION', + 'CONDITION', 'DATA-POINTER', 'DESTINATION', 'DISABLE', 'EC', 'EGI', + 'EMI', 'ENABLE', 'END-RECEIVE', 'ENTRY-CONVENTION', 'EO', 'ESI', + 'EXCEPTION-OBJECT', 'EXPANDS', 'FACTORY', 'FLOAT-BINARY-16', + 'FLOAT-BINARY-34', 'FLOAT-BINARY-7', 'FLOAT-DECIMAL-16', + 'FLOAT-DECIMAL-34', 'FLOAT-EXTENDED', 'FORMAT', 'FUNCTION-POINTER', + 'GET', 'GROUP-USAGE', 'IMPLEMENTS', 'INFINITY', 'INHERITS', + 'INTERFACE', 'INTERFACE-ID', 'INVOKE', 'LC_ALL', 'LC_COLLATE', 'LC_CTYPE', 'LC_MESSAGES', 'LC_MONETARY', 'LC_NUMERIC', 'LC_TIME', - 'LINE-COUNTER', 'MESSAGE', 'METHOD', 'METHOD-ID', 'NESTED', 'NONE', 'NORMAL', - 'OBJECT', 'OBJECT-REFERENCE', 'OPTIONS', 'OVERRIDE', 'PAGE-COUNTER', 'PF', 'PH', - 'PROPERTY', 'PROTOTYPE', 'PURGE', 'QUEUE', 'RAISE', 'RAISING', 'RECEIVE', - 'RELATION', 'REPLACE', 'REPRESENTS-NOT-A-NUMBER', 'RESET', 'RESUME', 'RETRY', - 'RF', 'RH', 'SECONDS', 'SEGMENT', 'SELF', 'SEND', 'SOURCES', 'STATEMENT', 'STEP', - 'STRONG', 'SUB-QUEUE-1', 'SUB-QUEUE-2', 'SUB-QUEUE-3', 'SUPER', 'SYMBOL', - 'SYSTEM-DEFAULT', 'TABLE', 'TERMINAL', 'TEXT', 'TYPEDEF', 'UCS-4', 'UNIVERSAL', - 'USER-DEFAULT', 'UTF-16', 'UTF-8', 'VAL-STATUS', 'VALID', 'VALIDATE', - 'VALIDATE-STATUS'), - prefix=r'(^|(?<=[^0-9a-z_\-]))', suffix=r'\s*($|(?=[^0-9a-z_\-]))'), + 'LINE-COUNTER', 'MESSAGE', 'METHOD', 'METHOD-ID', 'NESTED', 'NONE', + 'NORMAL', 'OBJECT', 'OBJECT-REFERENCE', 'OPTIONS', 'OVERRIDE', + 'PAGE-COUNTER', 'PF', 'PH', 'PROPERTY', 'PROTOTYPE', 'PURGE', + 'QUEUE', 'RAISE', 'RAISING', 'RECEIVE', 'RELATION', 'REPLACE', + 'REPRESENTS-NOT-A-NUMBER', 'RESET', 'RESUME', 'RETRY', 'RF', 'RH', + 'SECONDS', 'SEGMENT', 'SELF', 'SEND', 'SOURCES', 'STATEMENT', + 'STEP', 'STRONG', 'SUB-QUEUE-1', 'SUB-QUEUE-2', 'SUB-QUEUE-3', + 'SUPER', 'SYMBOL', 'SYSTEM-DEFAULT', 'TABLE', 'TERMINAL', 'TEXT', + 'TYPEDEF', 'UCS-4', 'UNIVERSAL', 'USER-DEFAULT', 'UTF-16', 'UTF-8', + 'VAL-STATUS', 'VALID', 'VALIDATE', 'VALIDATE-STATUS'), + prefix=r'(^|(?<=[^\w\-]))', suffix=r'\s*($|(?=[^\w\-]))'), Error), # Data Types - (r'(^|(?<=[^0-9a-z_\-]))' + (r'(^|(?<=[^\w\-]))' r'(PIC\s+.+?(?=(\s|\.\s))|PICTURE\s+.+?(?=(\s|\.\s))|' r'(COMPUTATIONAL)(-[1-5X])?|(COMP)(-[1-5X])?|' r'BINARY-C-LONG|' r'BINARY-CHAR|BINARY-DOUBLE|BINARY-LONG|BINARY-SHORT|' - r'BINARY)\s*($|(?=[^0-9a-z_\-]))', Keyword.Type), + r'BINARY)\s*($|(?=[^\w\-]))', Keyword.Type), # Operators (r'(\*\*|\*|\+|-|/|<=|>=|<|>|==|/=|=)', Operator), @@ -180,7 +182,7 @@ class CobolLexer(RegexLexer): (r'([(),;:&%.])', Punctuation), # Intrinsics - (r'(^|(?<=[^0-9a-z_\-]))(ABS|ACOS|ANNUITY|ASIN|ATAN|BYTE-LENGTH|' + (r'(^|(?<=[^\w\-]))(ABS|ACOS|ANNUITY|ASIN|ATAN|BYTE-LENGTH|' r'CHAR|COMBINED-DATETIME|CONCATENATE|COS|CURRENT-DATE|' r'DATE-OF-INTEGER|DATE-TO-YYYYMMDD|DAY-OF-INTEGER|DAY-TO-YYYYDDD|' r'EXCEPTION-(?:FILE|LOCATION|STATEMENT|STATUS)|EXP10|EXP|E|' @@ -192,13 +194,13 @@ class CobolLexer(RegexLexer): r'STANDARD-DEVIATION|STORED-CHAR-LENGTH|SUBSTITUTE(?:-CASE)?|' r'SUM|TAN|TEST-DATE-YYYYMMDD|TEST-DAY-YYYYDDD|TRIM|' r'UPPER-CASE|VARIANCE|WHEN-COMPILED|YEAR-TO-YYYY)\s*' - r'($|(?=[^0-9a-z_\-]))', Name.Function), + r'($|(?=[^\w\-]))', Name.Function), # Booleans - (r'(^|(?<=[^0-9a-z_\-]))(true|false)\s*($|(?=[^0-9a-z_\-]))', Name.Builtin), + (r'(^|(?<=[^\w\-]))(true|false)\s*($|(?=[^\w\-]))', Name.Builtin), # Comparing Operators - (r'(^|(?<=[^0-9a-z_\-]))(equal|equals|ne|lt|le|gt|ge|' - r'greater|less|than|not|and|or)\s*($|(?=[^0-9a-z_\-]))', Operator.Word), + (r'(^|(?<=[^\w\-]))(equal|equals|ne|lt|le|gt|ge|' + r'greater|less|than|not|and|or)\s*($|(?=[^\w\-]))', Operator.Word), ], # \"[^\"\n]*\"|\'[^\'\n]*\' @@ -254,6 +256,7 @@ class ABAPLexer(RegexLexer): (r'\s+', Text), (r'^\*.*$', Comment.Single), (r'\".*?\n', Comment.Single), + (r'##\w+', Comment.Special), ], 'variable-names': [ (r'<\S+>', Name.Variable), @@ -262,8 +265,8 @@ class ABAPLexer(RegexLexer): 'root': [ include('common'), # function calls - (r'(CALL\s+(?:BADI|CUSTOMER-FUNCTION|FUNCTION))(\s+)(\'?\S+\'?)', - bygroups(Keyword, Text, Name.Function)), + (r'CALL\s+(?:BADI|CUSTOMER-FUNCTION|FUNCTION)', + Keyword), (r'(CALL\s+(?:DIALOG|SCREEN|SUBSCREEN|SELECTION-SCREEN|' r'TRANSACTION|TRANSFORMATION))\b', Keyword), @@ -283,6 +286,12 @@ class ABAPLexer(RegexLexer): # call methodnames returning style (r'(?<=(=|-)>)([\w\-~]+)(?=\()', Name.Function), + # text elements + (r'(TEXT)(-)(\d{3})', + bygroups(Keyword, Punctuation, Number.Integer)), + (r'(TEXT)(-)(\w{3})', + bygroups(Keyword, Punctuation, Name.Variable)), + # keywords with dashes in them. # these need to be first, because for instance the -ID part # of MESSAGE-ID wouldn't get highlighted if MESSAGE was @@ -299,13 +308,13 @@ class ABAPLexer(RegexLexer): r'OUTPUT-LENGTH|PRINT-CONTROL|' r'SELECT-OPTIONS|START-OF-SELECTION|SUBTRACT-CORRESPONDING|' r'SYNTAX-CHECK|SYSTEM-EXCEPTIONS|' - r'TYPE-POOL|TYPE-POOLS' + r'TYPE-POOL|TYPE-POOLS|NO-DISPLAY' r')\b', Keyword), # keyword kombinations - (r'CREATE\s+(PUBLIC|PRIVATE|DATA|OBJECT)|' - r'((PUBLIC|PRIVATE|PROTECTED)\s+SECTION|' - r'(TYPE|LIKE)(\s+(LINE\s+OF|REF\s+TO|' + (r'(?<![-\>])(CREATE\s+(PUBLIC|PRIVATE|DATA|OBJECT)|' + r'(PUBLIC|PRIVATE|PROTECTED)\s+SECTION|' + r'(TYPE|LIKE)\s+((LINE\s+OF|REF\s+TO|' r'(SORTED|STANDARD|HASHED)\s+TABLE\s+OF))?|' r'FROM\s+(DATABASE|MEMORY)|CALL\s+METHOD|' r'(GROUP|ORDER) BY|HAVING|SEPARATED BY|' @@ -341,10 +350,16 @@ class ABAPLexer(RegexLexer): r'(BEGIN|END)\s+OF|' r'DELETE(\s+ADJACENT\s+DUPLICATES\sFROM)?|' r'COMPARING(\s+ALL\s+FIELDS)?|' - r'INSERT(\s+INITIAL\s+LINE\s+INTO|\s+LINES\s+OF)?|' + r'(INSERT|APPEND)(\s+INITIAL\s+LINE\s+(IN)?TO|\s+LINES\s+OF)?|' r'IN\s+((BYTE|CHARACTER)\s+MODE|PROGRAM)|' r'END-OF-(DEFINITION|PAGE|SELECTION)|' r'WITH\s+FRAME(\s+TITLE)|' + r'(REPLACE|FIND)\s+((FIRST|ALL)\s+OCCURRENCES?\s+OF\s+)?(SUBSTRING|REGEX)?|' + r'MATCH\s+(LENGTH|COUNT|LINE|OFFSET)|' + r'(RESPECTING|IGNORING)\s+CASE|' + r'IN\s+UPDATE\s+TASK|' + r'(SOURCE|RESULT)\s+(XML)?|' + r'REFERENCE\s+INTO|' # simple kombinations r'AND\s+(MARK|RETURN)|CLIENT\s+SPECIFIED|CORRESPONDING\s+FIELDS\s+OF|' @@ -353,39 +368,41 @@ class ABAPLexer(RegexLexer): r'MODIFY\s+SCREEN|NESTING\s+LEVEL|NO\s+INTERVALS|OF\s+STRUCTURE|' r'RADIOBUTTON\s+GROUP|RANGE\s+OF|REF\s+TO|SUPPRESS DIALOG|' r'TABLE\s+OF|UPPER\s+CASE|TRANSPORTING\s+NO\s+FIELDS|' - r'VALUE\s+CHECK|VISIBLE\s+LENGTH|HEADER\s+LINE)\b', Keyword), + r'VALUE\s+CHECK|VISIBLE\s+LENGTH|HEADER\s+LINE|COMMON\s+PART)\b', Keyword), # single word keywords. - (r'(^|(?<=(\s|\.)))(ABBREVIATED|ADD|ALIASES|APPEND|ASSERT|' - r'ASSIGN(ING)?|AT(\s+FIRST)?|' + (r'(^|(?<=(\s|\.)))(ABBREVIATED|ABSTRACT|ADD|ALIASES|ALIGN|ALPHA|' + r'ASSERT|AS|ASSIGN(ING)?|AT(\s+FIRST)?|' r'BACK|BLOCK|BREAK-POINT|' r'CASE|CATCH|CHANGING|CHECK|CLASS|CLEAR|COLLECT|COLOR|COMMIT|' r'CREATE|COMMUNICATION|COMPONENTS?|COMPUTE|CONCATENATE|CONDENSE|' - r'CONSTANTS|CONTEXTS|CONTINUE|CONTROLS|' - r'DATA|DECIMALS|DEFAULT|DEFINE|DEFINITION|DEFERRED|DEMAND|' - r'DETAIL|DIRECTORY|DIVIDE|DO|' - r'ELSE(IF)?|ENDAT|ENDCASE|ENDCLASS|ENDDO|ENDFORM|ENDFUNCTION|' - r'ENDIF|ENDLOOP|ENDMETHOD|ENDMODULE|ENDSELECT|ENDTRY|' - r'ENHANCEMENT|EVENTS|EXCEPTIONS|EXIT|EXPORT|EXPORTING|EXTRACT|' - r'FETCH|FIELDS?|FIND|FOR|FORM|FORMAT|FREE|FROM|' + r'CONSTANTS|CONTEXTS|CONTINUE|CONTROLS|COUNTRY|CURRENCY|' + r'DATA|DATE|DECIMALS|DEFAULT|DEFINE|DEFINITION|DEFERRED|DEMAND|' + r'DETAIL|DIRECTORY|DIVIDE|DO|DUMMY|' + r'ELSE(IF)?|ENDAT|ENDCASE|ENDCATCH|ENDCLASS|ENDDO|ENDFORM|ENDFUNCTION|' + r'ENDIF|ENDINTERFACE|ENDLOOP|ENDMETHOD|ENDMODULE|ENDSELECT|ENDTRY|ENDWHILE|' + r'ENHANCEMENT|EVENTS|EXACT|EXCEPTIONS?|EXIT|EXPONENT|EXPORT|EXPORTING|EXTRACT|' + r'FETCH|FIELDS?|FOR|FORM|FORMAT|FREE|FROM|FUNCTION|' r'HIDE|' r'ID|IF|IMPORT|IMPLEMENTATION|IMPORTING|IN|INCLUDE|INCLUDING|' r'INDEX|INFOTYPES|INITIALIZATION|INTERFACE|INTERFACES|INTO|' - r'LENGTH|LINES|LOAD|LOCAL|' + r'LANGUAGE|LEAVE|LENGTH|LINES|LOAD|LOCAL|' r'JOIN|' r'KEY|' - r'MAXIMUM|MESSAGE|METHOD[S]?|MINIMUM|MODULE|MODIFY|MOVE|MULTIPLY|' - r'NODES|' - r'OBLIGATORY|OF|OFF|ON|OVERLAY|' - r'PACK|PARAMETERS|PERCENTAGE|POSITION|PROGRAM|PROVIDE|PUBLIC|PUT|' - r'RAISE|RAISING|RANGES|READ|RECEIVE|REFRESH|REJECT|REPORT|RESERVE|' - r'RESUME|RETRY|RETURN|RETURNING|RIGHT|ROLLBACK|' - r'SCROLL|SEARCH|SELECT|SHIFT|SINGLE|SKIP|SORT|SPLIT|STATICS|STOP|' - r'SUBMIT|SUBTRACT|SUM|SUMMARY|SUMMING|SUPPLY|' - r'TABLE|TABLES|TIMES|TITLE|TO|TOP-OF-PAGE|TRANSFER|TRANSLATE|TRY|TYPES|' + r'NEXT|' + r'MAXIMUM|MESSAGE|METHOD[S]?|MINIMUM|MODULE|MODIFIER|MODIFY|MOVE|MULTIPLY|' + r'NODES|NUMBER|' + r'OBLIGATORY|OBJECT|OF|OFF|ON|OTHERS|OVERLAY|' + r'PACK|PAD|PARAMETERS|PERCENTAGE|POSITION|PROGRAM|PROVIDE|PUBLIC|PUT|PF\d\d|' + r'RAISE|RAISING|RANGES?|READ|RECEIVE|REDEFINITION|REFRESH|REJECT|REPORT|RESERVE|' + r'RESUME|RETRY|RETURN|RETURNING|RIGHT|ROLLBACK|REPLACE|' + r'SCROLL|SEARCH|SELECT|SHIFT|SIGN|SINGLE|SIZE|SKIP|SORT|SPLIT|STATICS|STOP|' + r'STYLE|SUBMATCHES|SUBMIT|SUBTRACT|SUM(?!\()|SUMMARY|SUMMING|SUPPLY|' + r'TABLE|TABLES|TIMESTAMP|TIMES?|TIMEZONE|TITLE|\??TO|' + r'TOP-OF-PAGE|TRANSFER|TRANSLATE|TRY|TYPES|' r'ULINE|UNDER|UNPACK|UPDATE|USING|' - r'VALUE|VALUES|VIA|' - r'WAIT|WHEN|WHERE|WHILE|WITH|WINDOW|WRITE)\b', Keyword), + r'VALUE|VALUES|VIA|VARYING|VARY|' + r'WAIT|WHEN|WHERE|WIDTH|WHILE|WITH|WINDOW|WRITE|XSD|ZERO)\b', Keyword), # builtins (r'(abs|acos|asin|atan|' @@ -411,18 +428,21 @@ class ABAPLexer(RegexLexer): # operators which look like variable names before # parsing variable names. - (r'(?<=(\s|.))(AND|EQ|NE|GT|LT|GE|LE|CO|CN|CA|NA|CS|NOT|NS|CP|NP|' + (r'(?<=(\s|.))(AND|OR|EQ|NE|GT|LT|GE|LE|CO|CN|CA|NA|CS|NOT|NS|CP|NP|' r'BYTE-CO|BYTE-CN|BYTE-CA|BYTE-NA|BYTE-CS|BYTE-NS|' - r'IS\s+(NOT\s+)?(INITIAL|ASSIGNED|REQUESTED|BOUND))\b', Operator), + r'IS\s+(NOT\s+)?(INITIAL|ASSIGNED|REQUESTED|BOUND))\b', Operator.Word), include('variable-names'), - # standard oparators after variable names, + # standard operators after variable names, # because < and > are part of field symbols. - (r'[?*<>=\-+]', Operator), + (r'[?*<>=\-+&]', Operator), (r"'(''|[^'])*'", String.Single), (r"`([^`])*`", String.Single), - (r'[/;:()\[\],.]', Punctuation) + (r"([|}])([^{}|]*?)([|{])", + bygroups(Punctuation, String.Single, Punctuation)), + (r'[/;:()\[\],.]', Punctuation), + (r'(!)(\w+)', bygroups(Operator, Name)), ], } @@ -439,15 +459,15 @@ class OpenEdgeLexer(RegexLexer): filenames = ['*.p', '*.cls'] mimetypes = ['text/x-openedge', 'application/x-openedge'] - types = (r'(?i)(^|(?<=[^0-9a-z_\-]))(CHARACTER|CHAR|CHARA|CHARAC|CHARACT|CHARACTE|' + types = (r'(?i)(^|(?<=[^\w\-]))(CHARACTER|CHAR|CHARA|CHARAC|CHARACT|CHARACTE|' r'COM-HANDLE|DATE|DATETIME|DATETIME-TZ|' r'DECIMAL|DEC|DECI|DECIM|DECIMA|HANDLE|' r'INT64|INTEGER|INT|INTE|INTEG|INTEGE|' - r'LOGICAL|LONGCHAR|MEMPTR|RAW|RECID|ROWID)\s*($|(?=[^0-9a-z_\-]))') + r'LOGICAL|LONGCHAR|MEMPTR|RAW|RECID|ROWID)\s*($|(?=[^\w\-]))') keywords = words(OPENEDGEKEYWORDS, - prefix=r'(?i)(^|(?<=[^0-9a-z_\-]))', - suffix=r'\s*($|(?=[^0-9a-z_\-]))') + prefix=r'(?i)(^|(?<=[^\w\-]))', + suffix=r'\s*($|(?=[^\w\-]))') tokens = { 'root': [ diff --git a/pygments/lexers/c_cpp.py b/pygments/lexers/c_cpp.py index 5c724d03..691f5ab4 100644 --- a/pygments/lexers/c_cpp.py +++ b/pygments/lexers/c_cpp.py @@ -5,7 +5,7 @@ Lexers for C/C++ languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -46,12 +46,15 @@ class CFamilyLexer(RegexLexer): (r'\n', Text), (r'\s+', Text), (r'\\\n', Text), # line continuation - (r'//(\n|(.|\n)*?[^\\]\n)', Comment.Single), - (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), + (r'//(\n|[\w\W]*?[^\\]\n)', Comment.Single), + (r'/(\\\n)?[*][\w\W]*?[*](\\\n)?/', Comment.Multiline), + # Open until EOF, so no ending delimeter + (r'/(\\\n)?[*][\w\W]*', Comment.Multiline), ], 'statements': [ - (r'L?"', String, 'string'), - (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char), + (r'(L?)(")', bygroups(String.Affix, String), 'string'), + (r"(L?)(')(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])(')", + bygroups(String.Affix, String.Char, String.Char, String.Char)), (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float), (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float), (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex), @@ -60,10 +63,11 @@ class CFamilyLexer(RegexLexer): (r'\*/', Error), (r'[~!%^&*+=|?:<>/-]', Operator), (r'[()\[\],.]', Punctuation), - (words(('auto', 'break', 'case', 'const', 'continue', 'default', 'do', - 'else', 'enum', 'extern', 'for', 'goto', 'if', 'register', - 'restricted', 'return', 'sizeof', 'static', 'struct', - 'switch', 'typedef', 'union', 'volatile', 'while'), + (words(('asm', 'auto', 'break', 'case', 'const', 'continue', + 'default', 'do', 'else', 'enum', 'extern', 'for', 'goto', + 'if', 'register', 'restricted', 'return', 'sizeof', + 'static', 'struct', 'switch', 'typedef', 'union', + 'volatile', 'while'), suffix=r'\b'), Keyword), (r'(bool|int|long|float|short|double|char|unsigned|signed|void)\b', Keyword.Type), @@ -123,7 +127,8 @@ class CFamilyLexer(RegexLexer): (r'\\', String), # stray backslash ], 'macro': [ - (r'(include)(' + _ws1 + ')([^\n]+)', bygroups(Comment.Preproc, Text, Comment.PreprocFile)), + (r'(include)(' + _ws1 + r')([^\n]+)', + bygroups(Comment.Preproc, Text, Comment.PreprocFile)), (r'[^/\n]+', Comment.Preproc), (r'/[*](.|\n)*?[*]/', Comment.Multiline), (r'//.*?\n', Comment.Single, '#pop'), @@ -206,7 +211,7 @@ class CppLexer(CFamilyLexer): tokens = { 'statements': [ (words(( - 'asm', 'catch', 'const_cast', 'delete', 'dynamic_cast', 'explicit', + 'catch', 'const_cast', 'delete', 'dynamic_cast', 'explicit', 'export', 'friend', 'mutable', 'namespace', 'new', 'operator', 'private', 'protected', 'public', 'reinterpret_cast', 'restrict', 'static_cast', 'template', 'this', 'throw', 'throws', @@ -217,7 +222,11 @@ class CppLexer(CFamilyLexer): (r'char(16_t|32_t)\b', Keyword.Type), (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'), # C++11 raw strings - (r'R"\(', String, 'rawstring'), + (r'(R)(")([^\\()\s]{,16})(\()((?:.|\n)*?)(\)\3)(")', + bygroups(String.Affix, String, String.Delimiter, String.Delimiter, + String, String.Delimiter, String)), + # C++11 UTF-8/16/32 strings + (r'(u8|u|U)(")', bygroups(String.Affix, String), 'string'), inherit, ], 'root': [ @@ -234,11 +243,6 @@ class CppLexer(CFamilyLexer): # template specification (r'\s*(?=>)', Text, '#pop'), ], - 'rawstring': [ - (r'\)"', String, '#pop'), - (r'[^)]+', String), - (r'\)', String), - ], } def analyse_text(text): diff --git a/pygments/lexers/c_like.py b/pygments/lexers/c_like.py index d894818d..f7ba7e8f 100644 --- a/pygments/lexers/c_like.py +++ b/pygments/lexers/c_like.py @@ -5,7 +5,7 @@ Lexers for other C-like languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -427,115 +427,115 @@ class ArduinoLexer(CppLexer): filenames = ['*.ino'] mimetypes = ['text/x-arduino'] - # Language constants - constants = set(('DIGITAL_MESSAGE', 'FIRMATA_STRING', 'ANALOG_MESSAGE', - 'REPORT_DIGITAL', 'REPORT_ANALOG', 'INPUT_PULLUP', - 'SET_PIN_MODE', 'INTERNAL2V56', 'SYSTEM_RESET', 'LED_BUILTIN', - 'INTERNAL1V1', 'SYSEX_START', 'INTERNAL', 'EXTERNAL', - 'DEFAULT', 'OUTPUT', 'INPUT', 'HIGH', 'LOW')) - # Language sketch main structure functions structure = set(('setup', 'loop')) - # Language variable types - storage = set(('boolean', 'const', 'byte', 'word', 'string', 'String', 'array')) + # Language operators + operators = set(('not', 'or', 'and', 'xor')) + + # Language 'variables' + variables = set(( + 'DIGITAL_MESSAGE', 'FIRMATA_STRING', 'ANALOG_MESSAGE', 'REPORT_DIGITAL', + 'REPORT_ANALOG', 'INPUT_PULLUP', 'SET_PIN_MODE', 'INTERNAL2V56', 'SYSTEM_RESET', + 'LED_BUILTIN', 'INTERNAL1V1', 'SYSEX_START', 'INTERNAL', 'EXTERNAL', 'HIGH', + 'LOW', 'INPUT', 'OUTPUT', 'INPUT_PULLUP', 'LED_BUILTIN', 'true', 'false', + 'void', 'boolean', 'char', 'unsigned char', 'byte', 'int', 'unsigned int', + 'word', 'long', 'unsigned long', 'short', 'float', 'double', 'string', 'String', + 'array', 'static', 'volatile', 'const', 'boolean', 'byte', 'word', 'string', + 'String', 'array', 'int', 'float', 'private', 'char', 'virtual', 'operator', + 'sizeof', 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'int8_t', 'int16_t', + 'int32_t', 'int64_t', 'dynamic_cast', 'typedef', 'const_cast', 'const', + 'struct', 'static_cast', 'union', 'unsigned', 'long', 'volatile', 'static', + 'protected', 'bool', 'public', 'friend', 'auto', 'void', 'enum', 'extern', + 'class', 'short', 'reinterpret_cast', 'double', 'register', 'explicit', + 'signed', 'inline', 'delete', '_Bool', 'complex', '_Complex', '_Imaginary', + 'atomic_bool', 'atomic_char', 'atomic_schar', 'atomic_uchar', 'atomic_short', + 'atomic_ushort', 'atomic_int', 'atomic_uint', 'atomic_long', 'atomic_ulong', + 'atomic_llong', 'atomic_ullong', 'PROGMEM')) # Language shipped functions and class ( ) - functions = set(('KeyboardController', 'MouseController', 'SoftwareSerial', - 'EthernetServer', 'EthernetClient', 'LiquidCrystal', - 'RobotControl', 'GSMVoiceCall', 'EthernetUDP', 'EsploraTFT', - 'HttpClient', 'RobotMotor', 'WiFiClient', 'GSMScanner', - 'FileSystem', 'Scheduler', 'GSMServer', 'YunClient', 'YunServer', - 'IPAddress', 'GSMClient', 'GSMModem', 'Keyboard', 'Ethernet', - 'Console', 'GSMBand', 'Esplora', 'Stepper', 'Process', - 'WiFiUDP', 'GSM_SMS', 'Mailbox', 'USBHost', 'Firmata', 'PImage', - 'Client', 'Server', 'GSMPIN', 'FileIO', 'Bridge', 'Serial', - 'EEPROM', 'Stream', 'Mouse', 'Audio', 'Servo', 'File', 'Task', - 'GPRS', 'WiFi', 'Wire', 'TFT', 'GSM', 'SPI', 'SD', - 'runShellCommandAsynchronously', 'analogWriteResolution', - 'retrieveCallingNumber', 'printFirmwareVersion', - 'analogReadResolution', 'sendDigitalPortPair', - 'noListenOnLocalhost', 'readJoystickButton', 'setFirmwareVersion', - 'readJoystickSwitch', 'scrollDisplayRight', 'getVoiceCallStatus', - 'scrollDisplayLeft', 'writeMicroseconds', 'delayMicroseconds', - 'beginTransmission', 'getSignalStrength', 'runAsynchronously', - 'getAsynchronously', 'listenOnLocalhost', 'getCurrentCarrier', - 'readAccelerometer', 'messageAvailable', 'sendDigitalPorts', - 'lineFollowConfig', 'countryNameWrite', 'runShellCommand', - 'readStringUntil', 'rewindDirectory', 'readTemperature', - 'setClockDivider', 'readLightSensor', 'endTransmission', - 'analogReference', 'detachInterrupt', 'countryNameRead', - 'attachInterrupt', 'encryptionType', 'readBytesUntil', - 'robotNameWrite', 'readMicrophone', 'robotNameRead', 'cityNameWrite', - 'userNameWrite', 'readJoystickY', 'readJoystickX', 'mouseReleased', - 'openNextFile', 'scanNetworks', 'noInterrupts', 'digitalWrite', - 'beginSpeaker', 'mousePressed', 'isActionDone', 'mouseDragged', - 'displayLogos', 'noAutoscroll', 'addParameter', 'remoteNumber', - 'getModifiers', 'keyboardRead', 'userNameRead', 'waitContinue', - 'processInput', 'parseCommand', 'printVersion', 'readNetworks', - 'writeMessage', 'blinkVersion', 'cityNameRead', 'readMessage', - 'setDataMode', 'parsePacket', 'isListening', 'setBitOrder', - 'beginPacket', 'isDirectory', 'motorsWrite', 'drawCompass', - 'digitalRead', 'clearScreen', 'serialEvent', 'rightToLeft', - 'setTextSize', 'leftToRight', 'requestFrom', 'keyReleased', - 'compassRead', 'analogWrite', 'interrupts', 'WiFiServer', - 'disconnect', 'playMelody', 'parseFloat', 'autoscroll', - 'getPINUsed', 'setPINUsed', 'setTimeout', 'sendAnalog', - 'readSlider', 'analogRead', 'beginWrite', 'createChar', - 'motorsStop', 'keyPressed', 'tempoWrite', 'readButton', - 'subnetMask', 'debugPrint', 'macAddress', 'writeGreen', - 'randomSeed', 'attachGPRS', 'readString', 'sendString', - 'remotePort', 'releaseAll', 'mouseMoved', 'background', - 'getXChange', 'getYChange', 'answerCall', 'getResult', - 'voiceCall', 'endPacket', 'constrain', 'getSocket', 'writeJSON', - 'getButton', 'available', 'connected', 'findUntil', 'readBytes', - 'exitValue', 'readGreen', 'writeBlue', 'startLoop', 'IPAddress', - 'isPressed', 'sendSysex', 'pauseMode', 'gatewayIP', 'setCursor', - 'getOemKey', 'tuneWrite', 'noDisplay', 'loadImage', 'switchPIN', - 'onRequest', 'onReceive', 'changePIN', 'playFile', 'noBuffer', - 'parseInt', 'overflow', 'checkPIN', 'knobRead', 'beginTFT', - 'bitClear', 'updateIR', 'bitWrite', 'position', 'writeRGB', - 'highByte', 'writeRed', 'setSpeed', 'readBlue', 'noStroke', - 'remoteIP', 'transfer', 'shutdown', 'hangCall', 'beginSMS', - 'endWrite', 'attached', 'maintain', 'noCursor', 'checkReg', - 'checkPUK', 'shiftOut', 'isValid', 'shiftIn', 'pulseIn', - 'connect', 'println', 'localIP', 'pinMode', 'getIMEI', - 'display', 'noBlink', 'process', 'getBand', 'running', 'beginSD', - 'drawBMP', 'lowByte', 'setBand', 'release', 'bitRead', 'prepare', - 'pointTo', 'readRed', 'setMode', 'noFill', 'remove', 'listen', - 'stroke', 'detach', 'attach', 'noTone', 'exists', 'buffer', - 'height', 'bitSet', 'circle', 'config', 'cursor', 'random', - 'IRread', 'sizeof', 'setDNS', 'endSMS', 'getKey', 'micros', - 'millis', 'begin', 'print', 'write', 'ready', 'flush', 'width', - 'isPIN', 'blink', 'clear', 'press', 'mkdir', 'rmdir', 'close', - 'point', 'yield', 'image', 'float', 'BSSID', 'click', 'delay', - 'read', 'text', 'move', 'peek', 'beep', 'rect', 'line', 'open', - 'seek', 'fill', 'size', 'turn', 'stop', 'home', 'find', 'char', - 'byte', 'step', 'word', 'long', 'tone', 'sqrt', 'RSSI', 'SSID', - 'end', 'bit', 'tan', 'cos', 'sin', 'pow', 'map', 'abs', 'max', - 'min', 'int', 'get', 'run', 'put')) - + functions = set(( + 'KeyboardController', 'MouseController', 'SoftwareSerial', 'EthernetServer', + 'EthernetClient', 'LiquidCrystal', 'RobotControl', 'GSMVoiceCall', + 'EthernetUDP', 'EsploraTFT', 'HttpClient', 'RobotMotor', 'WiFiClient', + 'GSMScanner', 'FileSystem', 'Scheduler', 'GSMServer', 'YunClient', 'YunServer', + 'IPAddress', 'GSMClient', 'GSMModem', 'Keyboard', 'Ethernet', 'Console', + 'GSMBand', 'Esplora', 'Stepper', 'Process', 'WiFiUDP', 'GSM_SMS', 'Mailbox', + 'USBHost', 'Firmata', 'PImage', 'Client', 'Server', 'GSMPIN', 'FileIO', + 'Bridge', 'Serial', 'EEPROM', 'Stream', 'Mouse', 'Audio', 'Servo', 'File', + 'Task', 'GPRS', 'WiFi', 'Wire', 'TFT', 'GSM', 'SPI', 'SD', + 'runShellCommandAsynchronously', 'analogWriteResolution', + 'retrieveCallingNumber', 'printFirmwareVersion', 'analogReadResolution', + 'sendDigitalPortPair', 'noListenOnLocalhost', 'readJoystickButton', + 'setFirmwareVersion', 'readJoystickSwitch', 'scrollDisplayRight', + 'getVoiceCallStatus', 'scrollDisplayLeft', 'writeMicroseconds', + 'delayMicroseconds', 'beginTransmission', 'getSignalStrength', + 'runAsynchronously', 'getAsynchronously', 'listenOnLocalhost', + 'getCurrentCarrier', 'readAccelerometer', 'messageAvailable', + 'sendDigitalPorts', 'lineFollowConfig', 'countryNameWrite', 'runShellCommand', + 'readStringUntil', 'rewindDirectory', 'readTemperature', 'setClockDivider', + 'readLightSensor', 'endTransmission', 'analogReference', 'detachInterrupt', + 'countryNameRead', 'attachInterrupt', 'encryptionType', 'readBytesUntil', + 'robotNameWrite', 'readMicrophone', 'robotNameRead', 'cityNameWrite', + 'userNameWrite', 'readJoystickY', 'readJoystickX', 'mouseReleased', + 'openNextFile', 'scanNetworks', 'noInterrupts', 'digitalWrite', 'beginSpeaker', + 'mousePressed', 'isActionDone', 'mouseDragged', 'displayLogos', 'noAutoscroll', + 'addParameter', 'remoteNumber', 'getModifiers', 'keyboardRead', 'userNameRead', + 'waitContinue', 'processInput', 'parseCommand', 'printVersion', 'readNetworks', + 'writeMessage', 'blinkVersion', 'cityNameRead', 'readMessage', 'setDataMode', + 'parsePacket', 'isListening', 'setBitOrder', 'beginPacket', 'isDirectory', + 'motorsWrite', 'drawCompass', 'digitalRead', 'clearScreen', 'serialEvent', + 'rightToLeft', 'setTextSize', 'leftToRight', 'requestFrom', 'keyReleased', + 'compassRead', 'analogWrite', 'interrupts', 'WiFiServer', 'disconnect', + 'playMelody', 'parseFloat', 'autoscroll', 'getPINUsed', 'setPINUsed', + 'setTimeout', 'sendAnalog', 'readSlider', 'analogRead', 'beginWrite', + 'createChar', 'motorsStop', 'keyPressed', 'tempoWrite', 'readButton', + 'subnetMask', 'debugPrint', 'macAddress', 'writeGreen', 'randomSeed', + 'attachGPRS', 'readString', 'sendString', 'remotePort', 'releaseAll', + 'mouseMoved', 'background', 'getXChange', 'getYChange', 'answerCall', + 'getResult', 'voiceCall', 'endPacket', 'constrain', 'getSocket', 'writeJSON', + 'getButton', 'available', 'connected', 'findUntil', 'readBytes', 'exitValue', + 'readGreen', 'writeBlue', 'startLoop', 'IPAddress', 'isPressed', 'sendSysex', + 'pauseMode', 'gatewayIP', 'setCursor', 'getOemKey', 'tuneWrite', 'noDisplay', + 'loadImage', 'switchPIN', 'onRequest', 'onReceive', 'changePIN', 'playFile', + 'noBuffer', 'parseInt', 'overflow', 'checkPIN', 'knobRead', 'beginTFT', + 'bitClear', 'updateIR', 'bitWrite', 'position', 'writeRGB', 'highByte', + 'writeRed', 'setSpeed', 'readBlue', 'noStroke', 'remoteIP', 'transfer', + 'shutdown', 'hangCall', 'beginSMS', 'endWrite', 'attached', 'maintain', + 'noCursor', 'checkReg', 'checkPUK', 'shiftOut', 'isValid', 'shiftIn', 'pulseIn', + 'connect', 'println', 'localIP', 'pinMode', 'getIMEI', 'display', 'noBlink', + 'process', 'getBand', 'running', 'beginSD', 'drawBMP', 'lowByte', 'setBand', + 'release', 'bitRead', 'prepare', 'pointTo', 'readRed', 'setMode', 'noFill', + 'remove', 'listen', 'stroke', 'detach', 'attach', 'noTone', 'exists', 'buffer', + 'height', 'bitSet', 'circle', 'config', 'cursor', 'random', 'IRread', 'setDNS', + 'endSMS', 'getKey', 'micros', 'millis', 'begin', 'print', 'write', 'ready', + 'flush', 'width', 'isPIN', 'blink', 'clear', 'press', 'mkdir', 'rmdir', 'close', + 'point', 'yield', 'image', 'BSSID', 'click', 'delay', 'read', 'text', 'move', + 'peek', 'beep', 'rect', 'line', 'open', 'seek', 'fill', 'size', 'turn', 'stop', + 'home', 'find', 'step', 'tone', 'sqrt', 'RSSI', 'SSID', 'end', 'bit', 'tan', + 'cos', 'sin', 'pow', 'map', 'abs', 'max', 'min', 'get', 'run', 'put', + 'isAlphaNumeric', 'isAlpha', 'isAscii', 'isWhitespace', 'isControl', 'isDigit', + 'isGraph', 'isLowerCase', 'isPrintable', 'isPunct', 'isSpace', 'isUpperCase', + 'isHexadecimalDigit')) + + # do not highlight + suppress_highlight = set(( + 'namespace', 'template', 'mutable', 'using', 'asm', 'typeid', + 'typename', 'this', 'alignof', 'constexpr', 'decltype', 'noexcept', + 'static_assert', 'thread_local', 'restrict')) + def get_tokens_unprocessed(self, text): for index, token, value in CppLexer.get_tokens_unprocessed(self, text): - if token is Name: - if value in self.constants: - yield index, Keyword.Constant, value - elif value in self.functions: - yield index, Name.Function, value - elif value in self.storage: - yield index, Keyword.Type, value - else: - yield index, token, value - elif token is Name.Function: - if value in self.structure: - yield index, Name.Other, value - else: - yield index, token, value - elif token is Keyword: - if value in self.storage: - yield index, Keyword.Type, value - else: - yield index, token, value + if value in self.structure: + yield index, Name.Builtin, value + elif value in self.operators: + yield index, Operator, value + elif value in self.variables: + yield index, Keyword.Reserved, value + elif value in self.suppress_highlight: + yield index, Name, value + elif value in self.functions: + yield index, Name.Function, value else: yield index, token, value diff --git a/pygments/lexers/capnproto.py b/pygments/lexers/capnproto.py new file mode 100644 index 00000000..203523a1 --- /dev/null +++ b/pygments/lexers/capnproto.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.capnproto + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for the Cap'n Proto schema language. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, default +from pygments.token import Text, Comment, Keyword, Name, Literal + +__all__ = ['CapnProtoLexer'] + + +class CapnProtoLexer(RegexLexer): + """ + For `Cap'n Proto <https://capnproto.org>`_ source. + + .. versionadded:: 2.2 + """ + name = 'Cap\'n Proto' + filenames = ['*.capnp'] + aliases = ['capnp'] + + flags = re.MULTILINE | re.UNICODE + + tokens = { + 'root': [ + (r'#.*?$', Comment.Single), + (r'@[0-9a-zA-Z]*', Name.Decorator), + (r'=', Literal, 'expression'), + (r':', Name.Class, 'type'), + (r'\$', Name.Attribute, 'annotation'), + (r'(struct|enum|interface|union|import|using|const|annotation|' + r'extends|in|of|on|as|with|from|fixed)\b', + Keyword), + (r'[\w.]+', Name), + (r'[^#@=:$\w]+', Text), + ], + 'type': [ + (r'[^][=;,(){}$]+', Name.Class), + (r'[[(]', Name.Class, 'parentype'), + default('#pop'), + ], + 'parentype': [ + (r'[^][;()]+', Name.Class), + (r'[[(]', Name.Class, '#push'), + (r'[])]', Name.Class, '#pop'), + default('#pop'), + ], + 'expression': [ + (r'[^][;,(){}$]+', Literal), + (r'[[(]', Literal, 'parenexp'), + default('#pop'), + ], + 'parenexp': [ + (r'[^][;()]+', Literal), + (r'[[(]', Literal, '#push'), + (r'[])]', Literal, '#pop'), + default('#pop'), + ], + 'annotation': [ + (r'[^][;,(){}=:]+', Name.Attribute), + (r'[[(]', Name.Attribute, 'annexp'), + default('#pop'), + ], + 'annexp': [ + (r'[^][;()]+', Name.Attribute), + (r'[[(]', Name.Attribute, '#push'), + (r'[])]', Name.Attribute, '#pop'), + default('#pop'), + ], + } diff --git a/pygments/lexers/chapel.py b/pygments/lexers/chapel.py index 9f9894cd..55bf0e1e 100644 --- a/pygments/lexers/chapel.py +++ b/pygments/lexers/chapel.py @@ -5,7 +5,7 @@ Lexer for the Chapel language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -42,17 +42,17 @@ class ChapelLexer(RegexLexer): (r'(bool|complex|imag|int|opaque|range|real|string|uint)\b', Keyword.Type), (words(( - 'align', 'atomic', 'begin', 'break', 'by', 'cobegin', 'coforall', - 'continue', 'delete', 'dmapped', 'do', 'domain', 'else', 'enum', - 'except', 'export', 'extern', 'for', 'forall', 'if', 'index', - 'inline', 'iter', 'label', 'lambda', 'let', 'local', 'new', - 'noinit', 'on', 'only', 'otherwise', 'pragma', 'private', - 'public', 'reduce', 'require', 'return', 'scan', 'select', - 'serial', 'single', 'sparse', 'subdomain', 'sync', 'then', - 'use', 'when', 'where', 'while', 'with', 'yield', 'zip'), - suffix=r'\b'), + 'align', 'as', 'atomic', 'begin', 'break', 'by', 'cobegin', + 'coforall', 'continue', 'delete', 'dmapped', 'do', 'domain', + 'else', 'enum', 'except', 'export', 'extern', 'for', 'forall', + 'if', 'index', 'inline', 'iter', 'label', 'lambda', 'let', + 'local', 'new', 'noinit', 'on', 'only', 'otherwise', 'pragma', + 'private', 'public', 'reduce', 'require', 'return', 'scan', + 'select', 'serial', 'single', 'sparse', 'subdomain', 'sync', + 'then', 'use', 'when', 'where', 'while', 'with', 'yield', + 'zip'), suffix=r'\b'), Keyword), - (r'(proc)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'procname'), + (r'(proc)((?:\s)+)', bygroups(Keyword, Text), 'procname'), (r'(class|module|record|union)(\s+)', bygroups(Keyword, Text), 'classname'), @@ -96,6 +96,7 @@ class ChapelLexer(RegexLexer): (r'[a-zA-Z_][\w$]*', Name.Class, '#pop'), ], 'procname': [ - (r'[a-zA-Z_][\w$]*', Name.Function, '#pop'), + (r'([a-zA-Z_][\w$]+|\~[a-zA-Z_][\w$]+|[+*/!~%<>=&^|\-]{1,2})', + Name.Function, '#pop'), ], } diff --git a/pygments/lexers/clean.py b/pygments/lexers/clean.py new file mode 100644 index 00000000..ba2569f6 --- /dev/null +++ b/pygments/lexers/clean.py @@ -0,0 +1,288 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.clean + ~~~~~~~~~~~~~~~~~~~~~ + + Lexer for the Clean language. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import ExtendedRegexLexer, LexerContext, \ + bygroups, words, include, default +from pygments.token import Comment, Keyword, Literal, Name, Number, Operator, \ + Punctuation, String, Text, Whitespace + +__all__ = ['CleanLexer'] + + +class CleanLexer(ExtendedRegexLexer): + """ + Lexer for the general purpose, state-of-the-art, pure and lazy functional + programming language Clean (http://clean.cs.ru.nl/Clean). + + .. versionadded: 2.2 + """ + name = 'Clean' + aliases = ['clean'] + filenames = ['*.icl', '*.dcl'] + + def get_tokens_unprocessed(self, text=None, context=None): + ctx = LexerContext(text, 0) + ctx.indent = 0 + return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context=ctx) + + def check_class_not_import(lexer, match, ctx): + if match.group(0) == 'import': + yield match.start(), Keyword.Namespace, match.group(0) + ctx.stack = ctx.stack[:-1] + ['fromimportfunc'] + else: + yield match.start(), Name.Class, match.group(0) + ctx.pos = match.end() + + def check_instance_class(lexer, match, ctx): + if match.group(0) == 'instance' or match.group(0) == 'class': + yield match.start(), Keyword, match.group(0) + else: + yield match.start(), Name.Function, match.group(0) + ctx.stack = ctx.stack + ['fromimportfunctype'] + ctx.pos = match.end() + + @staticmethod + def indent_len(text): + # Tabs are four spaces: + # https://svn.cs.ru.nl/repos/clean-platform/trunk/doc/STANDARDS.txt + text = text.replace('\n', '') + return len(text.replace('\t', ' ')), len(text) + + def store_indent(lexer, match, ctx): + ctx.indent, _ = CleanLexer.indent_len(match.group(0)) + ctx.pos = match.end() + yield match.start(), Text, match.group(0) + + def check_indent1(lexer, match, ctx): + indent, reallen = CleanLexer.indent_len(match.group(0)) + if indent > ctx.indent: + yield match.start(), Whitespace, match.group(0) + ctx.pos = match.start() + reallen + 1 + else: + ctx.indent = 0 + ctx.pos = match.start() + ctx.stack = ctx.stack[:-1] + yield match.start(), Whitespace, match.group(0)[1:] + + def check_indent2(lexer, match, ctx): + indent, reallen = CleanLexer.indent_len(match.group(0)) + if indent > ctx.indent: + yield match.start(), Whitespace, match.group(0) + ctx.pos = match.start() + reallen + 1 + else: + ctx.indent = 0 + ctx.pos = match.start() + ctx.stack = ctx.stack[:-2] + + def check_indent3(lexer, match, ctx): + indent, reallen = CleanLexer.indent_len(match.group(0)) + if indent > ctx.indent: + yield match.start(), Whitespace, match.group(0) + ctx.pos = match.start() + reallen + 1 + else: + ctx.indent = 0 + ctx.pos = match.start() + ctx.stack = ctx.stack[:-3] + yield match.start(), Whitespace, match.group(0)[1:] + if match.group(0) == '\n\n': + ctx.pos = ctx.pos + 1 + + def skip(lexer, match, ctx): + ctx.stack = ctx.stack[:-1] + ctx.pos = match.end() + yield match.start(), Comment, match.group(0) + + keywords = ('class', 'instance', 'where', 'with', 'let', 'let!', + 'in', 'case', 'of', 'infix', 'infixr', 'infixl', 'generic', + 'derive', 'otherwise', 'code', 'inline') + + tokens = { + 'common': [ + (r';', Punctuation, '#pop'), + (r'//', Comment, 'singlecomment'), + ], + 'root': [ + # Comments + (r'//.*\n', Comment.Single), + (r'(?s)/\*\*.*?\*/', Comment.Special), + (r'(?s)/\*.*?\*/', Comment.Multi), + + # Modules, imports, etc. + (r'\b((?:implementation|definition|system)\s+)?(module)(\s+)([\w`.]+)', + bygroups(Keyword.Namespace, Keyword.Namespace, Text, Name.Class)), + (r'(?<=\n)import(?=\s)', Keyword.Namespace, 'import'), + (r'(?<=\n)from(?=\s)', Keyword.Namespace, 'fromimport'), + + # Keywords + # We cannot use (?s)^|(?<=\s) as prefix, so need to repeat this + (words(keywords, prefix=r'(?<=\s)', suffix=r'(?=\s)'), Keyword), + (words(keywords, prefix=r'^', suffix=r'(?=\s)'), Keyword), + + # Function definitions + (r'(?=\{\|)', Whitespace, 'genericfunction'), + (r'(?<=\n)([ \t]*)([\w`$()=\-<>~*\^|+&%]+)((?:\s+\w)*)(\s*)(::)', + bygroups(store_indent, Name.Function, Keyword.Type, Whitespace, + Punctuation), + 'functiondefargs'), + + # Type definitions + (r'(?<=\n)([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'), + (r'^([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'), + + # Literals + (r'\'\\?.(?<!\\)\'', String.Char), + (r'\'\\\d+\'', String.Char), + (r'\'\\\\\'', String.Char), # (special case for '\\') + (r'[+\-~]?\s*\d+\.\d+(E[+\-~]?\d+)?\b', Number.Float), + (r'[+\-~]?\s*0[0-7]\b', Number.Oct), + (r'[+\-~]?\s*0x[0-9a-fA-F]\b', Number.Hex), + (r'[+\-~]?\s*\d+\b', Number.Integer), + (r'"', String.Double, 'doubleqstring'), + (words(('True', 'False'), prefix=r'(?<=\s)', suffix=r'(?=\s)'), + Literal), + + # Qualified names + (r'(\')([\w.]+)(\'\.)', + bygroups(Punctuation, Name.Namespace, Punctuation)), + + # Everything else is some name + (r'([\w`$%/?@]+\.?)*[\w`$%/?@]+', Name), + + # Punctuation + (r'[{}()\[\],:;.#]', Punctuation), + (r'[+\-=!<>|&~*\^/]', Operator), + (r'\\\\', Operator), + + # Lambda expressions + (r'\\.*?(->|\.|=)', Name.Function), + + # Whitespace + (r'\s', Whitespace), + + include('common'), + ], + 'fromimport': [ + include('common'), + (r'([\w`.]+)', check_class_not_import), + (r'\n', Whitespace, '#pop'), + (r'\s', Whitespace), + ], + 'fromimportfunc': [ + include('common'), + (r'(::)(\s+)([^,\s]+)', bygroups(Punctuation, Text, Keyword.Type)), + (r'([\w`$()=\-<>~*\^|+&%/]+)', check_instance_class), + (r',', Punctuation), + (r'\n', Whitespace, '#pop'), + (r'\s', Whitespace), + ], + 'fromimportfunctype': [ + include('common'), + (r'[{(\[]', Punctuation, 'combtype'), + (r',', Punctuation, '#pop'), + (r'[:;.#]', Punctuation), + (r'\n', Whitespace, '#pop:2'), + (r'[^\S\n]+', Whitespace), + (r'\S+', Keyword.Type), + ], + 'combtype': [ + include('common'), + (r'[})\]]', Punctuation, '#pop'), + (r'[{(\[]', Punctuation, '#pop'), + (r'[,:;.#]', Punctuation), + (r'\s+', Whitespace), + (r'\S+', Keyword.Type), + ], + 'import': [ + include('common'), + (words(('from', 'import', 'as', 'qualified'), + prefix='(?<=\s)', suffix='(?=\s)'), Keyword.Namespace), + (r'[\w`.]+', Name.Class), + (r'\n', Whitespace, '#pop'), + (r',', Punctuation), + (r'[^\S\n]+', Whitespace), + ], + 'singlecomment': [ + (r'(.)(?=\n)', skip), + (r'.+(?!\n)', Comment), + ], + 'doubleqstring': [ + (r'[^\\"]+', String.Double), + (r'"', String.Double, '#pop'), + (r'\\.', String.Double), + ], + 'typedef': [ + include('common'), + (r'[\w`]+', Keyword.Type), + (r'[:=|(),\[\]{}!*]', Punctuation), + (r'->', Punctuation), + (r'\n(?=[^\s|])', Whitespace, '#pop'), + (r'\s', Whitespace), + (r'.', Keyword.Type), + ], + 'genericfunction': [ + include('common'), + (r'\{\|', Punctuation), + (r'\|\}', Punctuation, '#pop'), + (r',', Punctuation), + (r'->', Punctuation), + (r'(\s+of\s+)(\{)', bygroups(Keyword, Punctuation), 'genericftypes'), + (r'\s', Whitespace), + (r'[\w`\[\]{}!]+', Keyword.Type), + (r'[*()]', Punctuation), + ], + 'genericftypes': [ + include('common'), + (r'[\w`]+', Keyword.Type), + (r',', Punctuation), + (r'\s', Whitespace), + (r'\}', Punctuation, '#pop'), + ], + 'functiondefargs': [ + include('common'), + (r'\n(\s*)', check_indent1), + (r'[!{}()\[\],:;.#]', Punctuation), + (r'->', Punctuation, 'functiondefres'), + (r'^(?=\S)', Whitespace, '#pop'), + (r'\S', Keyword.Type), + (r'\s', Whitespace), + ], + 'functiondefres': [ + include('common'), + (r'\n(\s*)', check_indent2), + (r'^(?=\S)', Whitespace, '#pop:2'), + (r'[!{}()\[\],:;.#]', Punctuation), + (r'\|', Punctuation, 'functiondefclasses'), + (r'\S', Keyword.Type), + (r'\s', Whitespace), + ], + 'functiondefclasses': [ + include('common'), + (r'\n(\s*)', check_indent3), + (r'^(?=\S)', Whitespace, '#pop:3'), + (r'[,&]', Punctuation), + (r'\[', Punctuation, 'functiondefuniquneq'), + (r'[\w`$()=\-<>~*\^|+&%/{}\[\]@]', Name.Function, 'functionname'), + (r'\s+', Whitespace), + ], + 'functiondefuniquneq': [ + include('common'), + (r'[a-z]+', Keyword.Type), + (r'\s+', Whitespace), + (r'<=|,', Punctuation), + (r'\]', Punctuation, '#pop') + ], + 'functionname': [ + include('common'), + (r'[\w`$()=\-<>~*\^|+&%/]+', Name.Function), + (r'(?=\{\|)', Punctuation, 'genericfunction'), + default('#pop'), + ] + } diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py index 1cf83d7f..ab52a370 100644 --- a/pygments/lexers/compiled.py +++ b/pygments/lexers/compiled.py @@ -5,7 +5,7 @@ Just export lexer classes previously contained in this module. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -29,5 +29,6 @@ from pygments.lexers.dylan import DylanLexer, DylanLidLexer, DylanConsoleLexer from pygments.lexers.ooc import OocLexer from pygments.lexers.felix import FelixLexer from pygments.lexers.nimrod import NimrodLexer +from pygments.lexers.crystal import CrystalLexer __all__ = [] diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py index c46d8bb8..1717a563 100644 --- a/pygments/lexers/configs.py +++ b/pygments/lexers/configs.py @@ -5,7 +5,7 @@ Lexers for configuration file formats. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -39,8 +39,10 @@ class IniLexer(RegexLexer): (r'[;#].*', Comment.Single), (r'\[.*?\]$', Keyword), (r'(.*?)([ \t]*)(=)([ \t]*)(.*(?:\n[ \t].+)*)', - bygroups(Name.Attribute, Text, Operator, Text, String)) - ] + bygroups(Name.Attribute, Text, Operator, Text, String)), + # standalone option, supported by some INI parsers + (r'(.+?)$', Name.Attribute), + ], } def analyse_text(text): @@ -98,6 +100,8 @@ class PropertiesLexer(RegexLexer): """ Lexer for configuration files in Java's properties format. + Note: trailing whitespace counts as part of the value as per spec + .. versionadded:: 1.4 """ @@ -108,10 +112,14 @@ class PropertiesLexer(RegexLexer): tokens = { 'root': [ - (r'\s+', Text), - (r'(?:[;#]|//).*$', Comment), + (r'^(\w+)([ \t])(\w+\s*)$', bygroups(Name.Attribute, Text, String)), + (r'^\w+(\\[ \t]\w*)*$', Name.Attribute), + (r'(^ *)([#!].*)', bygroups(Text, Comment)), + # More controversial comments + (r'(^ *)((?:;|//).*)', bygroups(Text, Comment)), (r'(.*?)([ \t]*)([=:])([ \t]*)(.*(?:(?<=\\)\n.*)*)', bygroups(Name.Attribute, Text, Operator, Text, String)), + (r'\s', Text), ], } @@ -454,7 +462,7 @@ class NginxConfLexer(RegexLexer): """ name = 'Nginx configuration file' aliases = ['nginx'] - filenames = [] + filenames = ['nginx.conf'] mimetypes = ['text/x-nginx-conf'] tokens = { @@ -598,7 +606,7 @@ class TerraformLexer(RegexLexer): (r'(".*")', bygroups(String.Double)), ], 'punctuation': [ - (r'[\[\]\(\),.]', Punctuation), + (r'[\[\](),.]', Punctuation), ], # Keep this seperate from punctuation - we sometimes want to use different # Tokens for { } @@ -629,9 +637,8 @@ class TermcapLexer(RegexLexer): .. versionadded:: 2.1 """ name = 'Termcap' - aliases = ['termcap',] - - filenames = ['termcap', 'termcap.src',] + aliases = ['termcap'] + filenames = ['termcap', 'termcap.src'] mimetypes = [] # NOTE: @@ -642,13 +649,13 @@ class TermcapLexer(RegexLexer): tokens = { 'root': [ (r'^#.*$', Comment), - (r'^[^\s#:\|]+', Name.Tag, 'names'), + (r'^[^\s#:|]+', Name.Tag, 'names'), ], 'names': [ (r'\n', Text, '#pop'), (r':', Punctuation, 'defs'), (r'\|', Punctuation), - (r'[^:\|]+', Name.Attribute), + (r'[^:|]+', Name.Attribute), ], 'defs': [ (r'\\\n[ \t]*', Text), @@ -676,9 +683,8 @@ class TerminfoLexer(RegexLexer): .. versionadded:: 2.1 """ name = 'Terminfo' - aliases = ['terminfo',] - - filenames = ['terminfo', 'terminfo.src',] + aliases = ['terminfo'] + filenames = ['terminfo', 'terminfo.src'] mimetypes = [] # NOTE: @@ -689,13 +695,13 @@ class TerminfoLexer(RegexLexer): tokens = { 'root': [ (r'^#.*$', Comment), - (r'^[^\s#,\|]+', Name.Tag, 'names'), + (r'^[^\s#,|]+', Name.Tag, 'names'), ], 'names': [ (r'\n', Text, '#pop'), (r'(,)([ \t]*)', bygroups(Punctuation, Text), 'defs'), (r'\|', Punctuation), - (r'[^,\|]+', Name.Attribute), + (r'[^,|]+', Name.Attribute), ], 'defs': [ (r'\n[ \t]+', Text), @@ -724,8 +730,8 @@ class PkgConfigLexer(RegexLexer): """ name = 'PkgConfig' - aliases = ['pkgconfig',] - filenames = ['*.pc',] + aliases = ['pkgconfig'] + filenames = ['*.pc'] mimetypes = [] tokens = { @@ -791,8 +797,8 @@ class PacmanConfLexer(RegexLexer): """ name = 'PacmanConf' - aliases = ['pacmanconf',] - filenames = ['pacman.conf',] + aliases = ['pacmanconf'] + filenames = ['pacman.conf'] mimetypes = [] tokens = { @@ -820,7 +826,7 @@ class PacmanConfLexer(RegexLexer): '%u', # url ), suffix=r'\b'), Name.Variable), - + # fallback (r'.', Text), ], diff --git a/pygments/lexers/console.py b/pygments/lexers/console.py index 1d89b770..77bb72e5 100644 --- a/pygments/lexers/console.py +++ b/pygments/lexers/console.py @@ -5,7 +5,7 @@ Lexers for misc console output. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/crystal.py b/pygments/lexers/crystal.py new file mode 100644 index 00000000..bea4833f --- /dev/null +++ b/pygments/lexers/crystal.py @@ -0,0 +1,393 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.crystal + ~~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Crystal. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import ExtendedRegexLexer, include, \ + bygroups, default, LexerContext, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Error + +__all__ = ['CrystalLexer'] + +line_re = re.compile('.*?\n') + + +CRYSTAL_OPERATORS = [ + '!=', '!~', '!', '%', '&&', '&', '**', '*', '+', '-', '/', '<=>', '<<', '<=', '<', + '===', '==', '=~', '=', '>=', '>>', '>', '[]=', '[]?', '[]', '^', '||', '|', '~' +] + + +class CrystalLexer(ExtendedRegexLexer): + """ + For `Crystal <http://crystal-lang.org>`_ source code. + + .. versionadded:: 2.2 + """ + + name = 'Crystal' + aliases = ['cr', 'crystal'] + filenames = ['*.cr'] + mimetypes = ['text/x-crystal'] + + flags = re.DOTALL | re.MULTILINE + + def heredoc_callback(self, match, ctx): + # okay, this is the hardest part of parsing Crystal... + # match: 1 = <<-?, 2 = quote? 3 = name 4 = quote? 5 = rest of line + + start = match.start(1) + yield start, Operator, match.group(1) # <<-? + yield match.start(2), String.Heredoc, match.group(2) # quote ", ', ` + yield match.start(3), String.Delimiter, match.group(3) # heredoc name + yield match.start(4), String.Heredoc, match.group(4) # quote again + + heredocstack = ctx.__dict__.setdefault('heredocstack', []) + outermost = not bool(heredocstack) + heredocstack.append((match.group(1) == '<<-', match.group(3))) + + ctx.pos = match.start(5) + ctx.end = match.end(5) + # this may find other heredocs + for i, t, v in self.get_tokens_unprocessed(context=ctx): + yield i, t, v + ctx.pos = match.end() + + if outermost: + # this is the outer heredoc again, now we can process them all + for tolerant, hdname in heredocstack: + lines = [] + for match in line_re.finditer(ctx.text, ctx.pos): + if tolerant: + check = match.group().strip() + else: + check = match.group().rstrip() + if check == hdname: + for amatch in lines: + yield amatch.start(), String.Heredoc, amatch.group() + yield match.start(), String.Delimiter, match.group() + ctx.pos = match.end() + break + else: + lines.append(match) + else: + # end of heredoc not found -- error! + for amatch in lines: + yield amatch.start(), Error, amatch.group() + ctx.end = len(ctx.text) + del heredocstack[:] + + def gen_crystalstrings_rules(): + def intp_regex_callback(self, match, ctx): + yield match.start(1), String.Regex, match.group(1) # begin + nctx = LexerContext(match.group(3), 0, ['interpolated-regex']) + for i, t, v in self.get_tokens_unprocessed(context=nctx): + yield match.start(3)+i, t, v + yield match.start(4), String.Regex, match.group(4) # end[imsx]* + ctx.pos = match.end() + + def intp_string_callback(self, match, ctx): + yield match.start(1), String.Other, match.group(1) + nctx = LexerContext(match.group(3), 0, ['interpolated-string']) + for i, t, v in self.get_tokens_unprocessed(context=nctx): + yield match.start(3)+i, t, v + yield match.start(4), String.Other, match.group(4) # end + ctx.pos = match.end() + + states = {} + states['strings'] = [ + (r'\:@{0,2}[a-zA-Z_]\w*[!?]?', String.Symbol), + (words(CRYSTAL_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol), + (r":'(\\\\|\\'|[^'])*'", String.Symbol), + # This allows arbitrary text after '\ for simplicity + (r"'(\\\\|\\'|[^']|\\[^'\\]+)'", String.Char), + (r':"', String.Symbol, 'simple-sym'), + # Crystal doesn't have "symbol:"s but this simplifies function args + (r'([a-zA-Z_]\w*)(:)(?!:)', bygroups(String.Symbol, Punctuation)), + (r'"', String.Double, 'simple-string'), + (r'(?<!\.)`', String.Backtick, 'simple-backtick'), + ] + + # double-quoted string and symbol + for name, ttype, end in ('string', String.Double, '"'), \ + ('sym', String.Symbol, '"'), \ + ('backtick', String.Backtick, '`'): + states['simple-'+name] = [ + include('string-escaped' if name == 'sym' else 'string-intp-escaped'), + (r'[^\\%s#]+' % end, ttype), + (r'[\\#]', ttype), + (end, ttype, '#pop'), + ] + + # braced quoted strings + for lbrace, rbrace, bracecc, name in \ + ('\\{', '\\}', '{}', 'cb'), \ + ('\\[', '\\]', '\\[\\]', 'sb'), \ + ('\\(', '\\)', '()', 'pa'), \ + ('<', '>', '<>', 'ab'): + states[name+'-intp-string'] = [ + (r'\\[' + lbrace + ']', String.Other), + (lbrace, String.Other, '#push'), + (rbrace, String.Other, '#pop'), + include('string-intp-escaped'), + (r'[\\#' + bracecc + ']', String.Other), + (r'[^\\#' + bracecc + ']+', String.Other), + ] + states['strings'].append((r'%' + lbrace, String.Other, + name+'-intp-string')) + states[name+'-string'] = [ + (r'\\[\\' + bracecc + ']', String.Other), + (lbrace, String.Other, '#push'), + (rbrace, String.Other, '#pop'), + (r'[\\#' + bracecc + ']', String.Other), + (r'[^\\#' + bracecc + ']+', String.Other), + ] + # http://crystal-lang.org/docs/syntax_and_semantics/literals/array.html + states['strings'].append((r'%[wi]' + lbrace, String.Other, + name+'-string')) + states[name+'-regex'] = [ + (r'\\[\\' + bracecc + ']', String.Regex), + (lbrace, String.Regex, '#push'), + (rbrace + '[imsx]*', String.Regex, '#pop'), + include('string-intp'), + (r'[\\#' + bracecc + ']', String.Regex), + (r'[^\\#' + bracecc + ']+', String.Regex), + ] + states['strings'].append((r'%r' + lbrace, String.Regex, + name+'-regex')) + + # these must come after %<brace>! + states['strings'] += [ + # %r regex + (r'(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[imsx]*)', + intp_regex_callback), + # regular fancy strings with qsw + (r'(%[wi]([\W_]))((?:\\\2|(?!\2).)*)(\2)', + intp_string_callback), + # special forms of fancy strings after operators or + # in method calls with braces + (r'(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)', + bygroups(Text, String.Other, None)), + # and because of fixed width lookbehinds the whole thing a + # second time for line startings... + (r'^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)', + bygroups(Text, String.Other, None)), + # all regular fancy strings without qsw + (r'(%([\[{(<]))((?:\\\2|(?!\2).)*)(\2)', + intp_string_callback), + ] + + return states + + tokens = { + 'root': [ + (r'#.*?$', Comment.Single), + # keywords + (words(''' + abstract asm as begin break case do else elsif end ensure extend ifdef if + include instance_sizeof next of pointerof private protected rescue return + require sizeof super then typeof unless until when while with yield + '''.split(), suffix=r'\b'), Keyword), + (words(['true', 'false', 'nil'], suffix=r'\b'), Keyword.Constant), + # start of function, class and module names + (r'(module|lib)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)', + bygroups(Keyword, Text, Name.Namespace)), + (r'(def|fun|macro)(\s+)((?:[a-zA-Z_]\w*::)*)', + bygroups(Keyword, Text, Name.Namespace), 'funcname'), + (r'def(?=[*%&^`~+-/\[<>=])', Keyword, 'funcname'), + (r'(class|struct|union|type|alias|enum)(\s+)((?:[a-zA-Z_]\w*::)*)', + bygroups(Keyword, Text, Name.Namespace), 'classname'), + (r'(self|out|uninitialized)\b|(is_a|responds_to)\?', Keyword.Pseudo), + # macros + (words(''' + debugger record pp assert_responds_to spawn parallel + getter setter property delegate def_hash def_equals def_equals_and_hash + forward_missing_to + '''.split(), suffix=r'\b'), Name.Builtin.Pseudo), + (r'getter[!?]|property[!?]|__(DIR|FILE|LINE)__\b', Name.Builtin.Pseudo), + # builtins + # http://crystal-lang.org/api/toplevel.html + (words(''' + Object Value Struct Reference Proc Class Nil Symbol Enum Void + Bool Number Int Int8 Int16 Int32 Int64 UInt8 UInt16 UInt32 UInt64 + Float Float32 Float64 Char String + Pointer Slice Range Exception Regex + Mutex StaticArray Array Hash Set Tuple Deque Box Process File + Dir Time Channel Concurrent Scheduler + abort at_exit caller delay exit fork future get_stack_top gets + lazy loop main p print printf puts + raise rand read_line sleep sprintf system with_color + '''.split(), prefix=r'(?<!\.)', suffix=r'\b'), Name.Builtin), + # normal heredocs + (r'(?<!\w)(<<-?)(["`\']?)([a-zA-Z_]\w*)(\2)(.*?\n)', + heredoc_callback), + # empty string heredocs + (r'(<<-?)("|\')()(\2)(.*?\n)', heredoc_callback), + (r'__END__', Comment.Preproc, 'end-part'), + # multiline regex (after keywords or assignments) + (r'(?:^|(?<=[=<>~!:])|' + r'(?<=(?:\s|;)when\s)|' + r'(?<=(?:\s|;)or\s)|' + r'(?<=(?:\s|;)and\s)|' + r'(?<=\.index\s)|' + r'(?<=\.scan\s)|' + r'(?<=\.sub\s)|' + r'(?<=\.sub!\s)|' + r'(?<=\.gsub\s)|' + r'(?<=\.gsub!\s)|' + r'(?<=\.match\s)|' + r'(?<=(?:\s|;)if\s)|' + r'(?<=(?:\s|;)elsif\s)|' + r'(?<=^when\s)|' + r'(?<=^index\s)|' + r'(?<=^scan\s)|' + r'(?<=^sub\s)|' + r'(?<=^gsub\s)|' + r'(?<=^sub!\s)|' + r'(?<=^gsub!\s)|' + r'(?<=^match\s)|' + r'(?<=^if\s)|' + r'(?<=^elsif\s)' + r')(\s*)(/)', bygroups(Text, String.Regex), 'multiline-regex'), + # multiline regex (in method calls or subscripts) + (r'(?<=\(|,|\[)/', String.Regex, 'multiline-regex'), + # multiline regex (this time the funny no whitespace rule) + (r'(\s+)(/)(?![\s=])', bygroups(Text, String.Regex), + 'multiline-regex'), + # lex numbers and ignore following regular expressions which + # are division operators in fact (grrrr. i hate that. any + # better ideas?) + # since pygments 0.7 we also eat a "?" operator after numbers + # so that the char operator does not work. Chars are not allowed + # there so that you can use the ternary operator. + # stupid example: + # x>=0?n[x]:"" + (r'(0o[0-7]+(?:_[0-7]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?', + bygroups(Number.Oct, Text, Operator)), + (r'(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?', + bygroups(Number.Hex, Text, Operator)), + (r'(0b[01]+(?:_[01]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?', + bygroups(Number.Bin, Text, Operator)), + # 3 separate expressions for floats because any of the 3 optional + # parts makes it a float + (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)(?:e[+-]?[0-9]+)?' + r'(?:_?f[0-9]+)?)(\s*)([/?])?', + bygroups(Number.Float, Text, Operator)), + (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)' + r'(?:_?f[0-9]+)?)(\s*)([/?])?', + bygroups(Number.Float, Text, Operator)), + (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)?' + r'(?:_?f[0-9]+))(\s*)([/?])?', + bygroups(Number.Float, Text, Operator)), + (r'(0\b|[1-9][\d]*(?:_\d+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?', + bygroups(Number.Integer, Text, Operator)), + # Names + (r'@@[a-zA-Z_]\w*', Name.Variable.Class), + (r'@[a-zA-Z_]\w*', Name.Variable.Instance), + (r'\$\w+', Name.Variable.Global), + (r'\$[!@&`\'+~=/\\,;.<>_*$?:"^-]', Name.Variable.Global), + (r'\$-[0adFiIlpvw]', Name.Variable.Global), + (r'::', Operator), + include('strings'), + # chars + (r'\?(\\[MC]-)*' # modifiers + r'(\\([\\befnrtv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)' + r'(?!\w)', + String.Char), + (r'[A-Z][A-Z_]+\b', Name.Constant), + # macro expansion + (r'\{%', String.Interpol, 'in-macro-control'), + (r'\{\{', String.Interpol, 'in-macro-expr'), + # attributes + (r'(@\[)(\s*)([A-Z]\w*)', + bygroups(Operator, Text, Name.Decorator), 'in-attr'), + # this is needed because Crystal attributes can look + # like keywords (class) or like this: ` ?!? + (words(CRYSTAL_OPERATORS, prefix=r'(\.|::)'), + bygroups(Operator, Name.Operator)), + (r'(\.|::)([a-zA-Z_]\w*[!?]?|[*%&^`~+\-/\[<>=])', + bygroups(Operator, Name)), + # Names can end with [!?] unless it's "!=" + (r'[a-zA-Z_]\w*(?:[!?](?!=))?', Name), + (r'(\[|\]\??|\*\*|<=>?|>=|<<?|>>?|=~|===|' + r'!~|&&?|\|\||\.{1,3})', Operator), + (r'[-+/*%=<>&!^|~]=?', Operator), + (r'[(){};,/?:\\]', Punctuation), + (r'\s+', Text) + ], + 'funcname': [ + (r'(?:([a-zA-Z_]\w*)(\.))?' + r'([a-zA-Z_]\w*[!?]?|\*\*?|[-+]@?|' + r'[/%&|^`~]|\[\]=?|<<|>>|<=?>|>=?|===?)', + bygroups(Name.Class, Operator, Name.Function), '#pop'), + default('#pop') + ], + 'classname': [ + (r'[A-Z_]\w*', Name.Class), + (r'(\()(\s*)([A-Z_]\w*)(\s*)(\))', + bygroups(Punctuation, Text, Name.Class, Text, Punctuation)), + default('#pop') + ], + 'in-intp': [ + (r'\{', String.Interpol, '#push'), + (r'\}', String.Interpol, '#pop'), + include('root'), + ], + 'string-intp': [ + (r'#\{', String.Interpol, 'in-intp'), + ], + 'string-escaped': [ + (r'\\([\\befnstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})', String.Escape) + ], + 'string-intp-escaped': [ + include('string-intp'), + include('string-escaped'), + ], + 'interpolated-regex': [ + include('string-intp'), + (r'[\\#]', String.Regex), + (r'[^\\#]+', String.Regex), + ], + 'interpolated-string': [ + include('string-intp'), + (r'[\\#]', String.Other), + (r'[^\\#]+', String.Other), + ], + 'multiline-regex': [ + include('string-intp'), + (r'\\\\', String.Regex), + (r'\\/', String.Regex), + (r'[\\#]', String.Regex), + (r'[^\\/#]+', String.Regex), + (r'/[imsx]*', String.Regex, '#pop'), + ], + 'end-part': [ + (r'.+', Comment.Preproc, '#pop') + ], + 'in-macro-control': [ + (r'\{%', String.Interpol, '#push'), + (r'%\}', String.Interpol, '#pop'), + (r'for\b|in\b', Keyword), + include('root'), + ], + 'in-macro-expr': [ + (r'\{\{', String.Interpol, '#push'), + (r'\}\}', String.Interpol, '#pop'), + include('root'), + ], + 'in-attr': [ + (r'\[', Operator, '#push'), + (r'\]', Operator, '#pop'), + include('root'), + ], + } + tokens.update(gen_crystalstrings_rules()) diff --git a/pygments/lexers/csound.py b/pygments/lexers/csound.py index 51414073..858aa348 100644 --- a/pygments/lexers/csound.py +++ b/pygments/lexers/csound.py @@ -5,11 +5,11 @@ Lexers for CSound languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -import copy, re +import re from pygments.lexer import RegexLexer, bygroups, default, include, using, words from pygments.token import Comment, Keyword, Name, Number, Operator, Punctuation, \ @@ -21,7 +21,7 @@ from pygments.lexers.scripting import LuaLexer __all__ = ['CsoundScoreLexer', 'CsoundOrchestraLexer', 'CsoundDocumentLexer'] -newline = (r'((?:;|//).*)*(\n)', bygroups(Comment.Single, Text)) +newline = (r'((?:(?:;|//).*)*)(\n)', bygroups(Comment.Single, Text)) class CsoundLexer(RegexLexer): @@ -177,7 +177,7 @@ class CsoundOrchestraLexer(CsoundLexer): (r'0[xX][a-fA-F0-9]+', Number.Hex), (r'\d+', Number.Integer), (r'"', String, 'single-line string'), - (r'{{', String, 'multi-line string'), + (r'\{\{', String, 'multi-line string'), (r'[+\-*/%^!=&|<>#~¬]', Operator), (r'[](),?:[]', Punctuation), (words(( @@ -273,40 +273,40 @@ class CsoundOrchestraLexer(CsoundLexer): (r'[\\"~$%\^\n]', String) ], 'multi-line string': [ - (r'}}', String, '#pop'), - (r'[^\}]+|\}(?!\})', String) + (r'\}\}', String, '#pop'), + (r'[^}]+|\}(?!\})', String) ], 'scoreline opcode': [ include('whitespace or macro call'), - (r'{{', String, 'scoreline'), + (r'\{\{', String, 'scoreline'), default('#pop') ], 'scoreline': [ - (r'}}', String, '#pop'), - (r'([^\}]+)|\}(?!\})', using(CsoundScoreLexer)) + (r'\}\}', String, '#pop'), + (r'([^}]+)|\}(?!\})', using(CsoundScoreLexer)) ], 'python opcode': [ include('whitespace or macro call'), - (r'{{', String, 'python'), + (r'\{\{', String, 'python'), default('#pop') ], 'python': [ - (r'}}', String, '#pop'), - (r'([^\}]+)|\}(?!\})', using(PythonLexer)) + (r'\}\}', String, '#pop'), + (r'([^}]+)|\}(?!\})', using(PythonLexer)) ], 'lua opcode': [ include('whitespace or macro call'), (r'"', String, 'single-line string'), - (r'{{', String, 'lua'), + (r'\{\{', String, 'lua'), (r',', Punctuation), default('#pop') ], 'lua': [ - (r'}}', String, '#pop'), - (r'([^\}]+)|\}(?!\})', using(LuaLexer)) + (r'\}\}', String, '#pop'), + (r'([^}]+)|\}(?!\})', using(LuaLexer)) ] } @@ -315,7 +315,7 @@ class CsoundDocumentLexer(RegexLexer): """ For `Csound <http://csound.github.io>`_ documents. - + .. versionadded:: 2.1 """ name = 'Csound Document' diff --git a/pygments/lexers/css.py b/pygments/lexers/css.py index b40201f4..29d83707 100644 --- a/pygments/lexers/css.py +++ b/pygments/lexers/css.py @@ -5,7 +5,7 @@ Lexers for CSS and related stylesheet formats. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -21,6 +21,251 @@ from pygments.util import iteritems __all__ = ['CssLexer', 'SassLexer', 'ScssLexer', 'LessCssLexer'] +# List of vendor prefixes obtained from: +# https://www.w3.org/TR/CSS21/syndata.html#vendor-keyword-history +_vendor_prefixes = ( + '-ms-', 'mso-', '-moz-', '-o-', '-xv-', '-atsc-', '-wap-', '-khtml-', + '-webkit-', 'prince-', '-ah-', '-hp-', '-ro-', '-rim-', '-tc-', +) + +# List of CSS properties obtained from: +# https://www.w3.org/Style/CSS/all-properties.en.html +# Note: handle --* separately +_css_properties = ( + 'align-content', 'align-items', 'align-self', 'alignment-baseline', 'all', + 'animation', 'animation-delay', 'animation-direction', + 'animation-duration', 'animation-fill-mode', 'animation-iteration-count', + 'animation-name', 'animation-play-state', 'animation-timing-function', + 'appearance', 'azimuth', 'backface-visibility', 'background', + 'background-attachment', 'background-blend-mode', 'background-clip', + 'background-color', 'background-image', 'background-origin', + 'background-position', 'background-repeat', 'background-size', + 'baseline-shift', 'bookmark-label', 'bookmark-level', 'bookmark-state', + 'border', 'border-bottom', 'border-bottom-color', + 'border-bottom-left-radius', 'border-bottom-right-radius', + 'border-bottom-style', 'border-bottom-width', 'border-boundary', + 'border-collapse', 'border-color', 'border-image', 'border-image-outset', + 'border-image-repeat', 'border-image-slice', 'border-image-source', + 'border-image-width', 'border-left', 'border-left-color', + 'border-left-style', 'border-left-width', 'border-radius', 'border-right', + 'border-right-color', 'border-right-style', 'border-right-width', + 'border-spacing', 'border-style', 'border-top', 'border-top-color', + 'border-top-left-radius', 'border-top-right-radius', 'border-top-style', + 'border-top-width', 'border-width', 'bottom', 'box-decoration-break', + 'box-shadow', 'box-sizing', 'box-snap', 'box-suppress', 'break-after', + 'break-before', 'break-inside', 'caption-side', 'caret', 'caret-animation', + 'caret-color', 'caret-shape', 'chains', 'clear', 'clip', 'clip-path', + 'clip-rule', 'color', 'color-interpolation-filters', 'column-count', + 'column-fill', 'column-gap', 'column-rule', 'column-rule-color', + 'column-rule-style', 'column-rule-width', 'column-span', 'column-width', + 'columns', 'content', 'counter-increment', 'counter-reset', 'counter-set', + 'crop', 'cue', 'cue-after', 'cue-before', 'cursor', 'direction', 'display', + 'dominant-baseline', 'elevation', 'empty-cells', 'filter', 'flex', + 'flex-basis', 'flex-direction', 'flex-flow', 'flex-grow', 'flex-shrink', + 'flex-wrap', 'float', 'float-defer', 'float-offset', 'float-reference', + 'flood-color', 'flood-opacity', 'flow', 'flow-from', 'flow-into', 'font', + 'font-family', 'font-feature-settings', 'font-kerning', + 'font-language-override', 'font-size', 'font-size-adjust', 'font-stretch', + 'font-style', 'font-synthesis', 'font-variant', 'font-variant-alternates', + 'font-variant-caps', 'font-variant-east-asian', 'font-variant-ligatures', + 'font-variant-numeric', 'font-variant-position', 'font-weight', + 'footnote-display', 'footnote-policy', 'glyph-orientation-vertical', + 'grid', 'grid-area', 'grid-auto-columns', 'grid-auto-flow', + 'grid-auto-rows', 'grid-column', 'grid-column-end', 'grid-column-gap', + 'grid-column-start', 'grid-gap', 'grid-row', 'grid-row-end', + 'grid-row-gap', 'grid-row-start', 'grid-template', 'grid-template-areas', + 'grid-template-columns', 'grid-template-rows', 'hanging-punctuation', + 'height', 'hyphenate-character', 'hyphenate-limit-chars', + 'hyphenate-limit-last', 'hyphenate-limit-lines', 'hyphenate-limit-zone', + 'hyphens', 'image-orientation', 'image-resolution', 'initial-letter', + 'initial-letter-align', 'initial-letter-wrap', 'isolation', + 'justify-content', 'justify-items', 'justify-self', 'left', + 'letter-spacing', 'lighting-color', 'line-break', 'line-grid', + 'line-height', 'line-snap', 'list-style', 'list-style-image', + 'list-style-position', 'list-style-type', 'margin', 'margin-bottom', + 'margin-left', 'margin-right', 'margin-top', 'marker-side', + 'marquee-direction', 'marquee-loop', 'marquee-speed', 'marquee-style', + 'mask', 'mask-border', 'mask-border-mode', 'mask-border-outset', + 'mask-border-repeat', 'mask-border-slice', 'mask-border-source', + 'mask-border-width', 'mask-clip', 'mask-composite', 'mask-image', + 'mask-mode', 'mask-origin', 'mask-position', 'mask-repeat', 'mask-size', + 'mask-type', 'max-height', 'max-lines', 'max-width', 'min-height', + 'min-width', 'mix-blend-mode', 'motion', 'motion-offset', 'motion-path', + 'motion-rotation', 'move-to', 'nav-down', 'nav-left', 'nav-right', + 'nav-up', 'object-fit', 'object-position', 'offset-after', 'offset-before', + 'offset-end', 'offset-start', 'opacity', 'order', 'orphans', 'outline', + 'outline-color', 'outline-offset', 'outline-style', 'outline-width', + 'overflow', 'overflow-style', 'overflow-wrap', 'overflow-x', 'overflow-y', + 'padding', 'padding-bottom', 'padding-left', 'padding-right', 'padding-top', + 'page', 'page-break-after', 'page-break-before', 'page-break-inside', + 'page-policy', 'pause', 'pause-after', 'pause-before', 'perspective', + 'perspective-origin', 'pitch', 'pitch-range', 'play-during', 'polar-angle', + 'polar-distance', 'position', 'presentation-level', 'quotes', + 'region-fragment', 'resize', 'rest', 'rest-after', 'rest-before', + 'richness', 'right', 'rotation', 'rotation-point', 'ruby-align', + 'ruby-merge', 'ruby-position', 'running', 'scroll-snap-coordinate', + 'scroll-snap-destination', 'scroll-snap-points-x', 'scroll-snap-points-y', + 'scroll-snap-type', 'shape-image-threshold', 'shape-inside', 'shape-margin', + 'shape-outside', 'size', 'speak', 'speak-as', 'speak-header', + 'speak-numeral', 'speak-punctuation', 'speech-rate', 'stress', 'string-set', + 'tab-size', 'table-layout', 'text-align', 'text-align-last', + 'text-combine-upright', 'text-decoration', 'text-decoration-color', + 'text-decoration-line', 'text-decoration-skip', 'text-decoration-style', + 'text-emphasis', 'text-emphasis-color', 'text-emphasis-position', + 'text-emphasis-style', 'text-indent', 'text-justify', 'text-orientation', + 'text-overflow', 'text-shadow', 'text-space-collapse', 'text-space-trim', + 'text-spacing', 'text-transform', 'text-underline-position', 'text-wrap', + 'top', 'transform', 'transform-origin', 'transform-style', 'transition', + 'transition-delay', 'transition-duration', 'transition-property', + 'transition-timing-function', 'unicode-bidi', 'user-select', + 'vertical-align', 'visibility', 'voice-balance', 'voice-duration', + 'voice-family', 'voice-pitch', 'voice-range', 'voice-rate', 'voice-stress', + 'voice-volume', 'volume', 'white-space', 'widows', 'width', 'will-change', + 'word-break', 'word-spacing', 'word-wrap', 'wrap-after', 'wrap-before', + 'wrap-flow', 'wrap-inside', 'wrap-through', 'writing-mode', 'z-index', +) + +# List of keyword values obtained from: +# http://cssvalues.com/ +_keyword_values = ( + 'absolute', 'alias', 'all', 'all-petite-caps', 'all-scroll', + 'all-small-caps', 'allow-end', 'alpha', 'alternate', 'alternate-reverse', + 'always', 'armenian', 'auto', 'avoid', 'avoid-column', 'avoid-page', + 'backwards', 'balance', 'baseline', 'below', 'blink', 'block', 'bold', + 'bolder', 'border-box', 'both', 'bottom', 'box-decoration', 'break-word', + 'capitalize', 'cell', 'center', 'circle', 'clip', 'clone', 'close-quote', + 'col-resize', 'collapse', 'color', 'color-burn', 'color-dodge', 'column', + 'column-reverse', 'compact', 'condensed', 'contain', 'container', + 'content-box', 'context-menu', 'copy', 'cover', 'crisp-edges', 'crosshair', + 'currentColor', 'cursive', 'darken', 'dashed', 'decimal', + 'decimal-leading-zero', 'default', 'descendants', 'difference', 'digits', + 'disc', 'distribute', 'dot', 'dotted', 'double', 'double-circle', 'e-resize', + 'each-line', 'ease', 'ease-in', 'ease-in-out', 'ease-out', 'edges', + 'ellipsis', 'end', 'ew-resize', 'exclusion', 'expanded', 'extra-condensed', + 'extra-expanded', 'fantasy', 'fill', 'fill-box', 'filled', 'first', 'fixed', + 'flat', 'flex', 'flex-end', 'flex-start', 'flip', 'force-end', 'forwards', + 'from-image', 'full-width', 'geometricPrecision', 'georgian', 'groove', + 'hanging', 'hard-light', 'help', 'hidden', 'hide', 'horizontal', 'hue', + 'icon', 'infinite', 'inherit', 'initial', 'ink', 'inline', 'inline-block', + 'inline-flex', 'inline-table', 'inset', 'inside', 'inter-word', 'invert', + 'isolate', 'italic', 'justify', 'large', 'larger', 'last', 'left', + 'lighten', 'lighter', 'line-through', 'linear', 'list-item', 'local', + 'loose', 'lower-alpha', 'lower-greek', 'lower-latin', 'lower-roman', + 'lowercase', 'ltr', 'luminance', 'luminosity', 'mandatory', 'manipulation', + 'manual', 'margin-box', 'match-parent', 'medium', 'mixed', 'monospace', + 'move', 'multiply', 'n-resize', 'ne-resize', 'nesw-resize', + 'no-close-quote', 'no-drop', 'no-open-quote', 'no-repeat', 'none', 'normal', + 'not-allowed', 'nowrap', 'ns-resize', 'nw-resize', 'nwse-resize', 'objects', + 'oblique', 'off', 'on', 'open', 'open-quote', 'optimizeLegibility', + 'optimizeSpeed', 'outset', 'outside', 'over', 'overlay', 'overline', + 'padding-box', 'page', 'pan-down', 'pan-left', 'pan-right', 'pan-up', + 'pan-x', 'pan-y', 'paused', 'petite-caps', 'pixelated', 'pointer', + 'preserve-3d', 'progress', 'proximity', 'relative', 'repeat', + 'repeat no-repeat', 'repeat-x', 'repeat-y', 'reverse', 'ridge', 'right', + 'round', 'row', 'row-resize', 'row-reverse', 'rtl', 'ruby', 'ruby-base', + 'ruby-base-container', 'ruby-text', 'ruby-text-container', 'run-in', + 'running', 's-resize', 'sans-serif', 'saturation', 'scale-down', 'screen', + 'scroll', 'se-resize', 'semi-condensed', 'semi-expanded', 'separate', + 'serif', 'sesame', 'show', 'sideways', 'sideways-left', 'sideways-right', + 'slice', 'small', 'small-caps', 'smaller', 'smooth', 'snap', 'soft-light', + 'solid', 'space', 'space-around', 'space-between', 'spaces', 'square', + 'start', 'static', 'step-end', 'step-start', 'sticky', 'stretch', 'strict', + 'stroke-box', 'style', 'sw-resize', 'table', 'table-caption', 'table-cell', + 'table-column', 'table-column-group', 'table-footer-group', + 'table-header-group', 'table-row', 'table-row-group', 'text', 'thick', + 'thin', 'titling-caps', 'to', 'top', 'triangle', 'ultra-condensed', + 'ultra-expanded', 'under', 'underline', 'unicase', 'unset', 'upper-alpha', + 'upper-latin', 'upper-roman', 'uppercase', 'upright', 'use-glyph-orientation', + 'vertical', 'vertical-text', 'view-box', 'visible', 'w-resize', 'wait', + 'wavy', 'weight', 'weight style', 'wrap', 'wrap-reverse', 'x-large', + 'x-small', 'xx-large', 'xx-small', 'zoom-in', 'zoom-out', +) + +# List of extended color keywords obtained from: +# https://drafts.csswg.org/css-color/#named-colors +_color_keywords = ( + 'aliceblue', 'antiquewhite', 'aqua', 'aquamarine', 'azure', 'beige', + 'bisque', 'black', 'blanchedalmond', 'blue', 'blueviolet', 'brown', + 'burlywood', 'cadetblue', 'chartreuse', 'chocolate', 'coral', + 'cornflowerblue', 'cornsilk', 'crimson', 'cyan', 'darkblue', 'darkcyan', + 'darkgoldenrod', 'darkgray', 'darkgreen', 'darkgrey', 'darkkhaki', + 'darkmagenta', 'darkolivegreen', 'darkorange', 'darkorchid', 'darkred', + 'darksalmon', 'darkseagreen', 'darkslateblue', 'darkslategray', + 'darkslategrey', 'darkturquoise', 'darkviolet', 'deeppink', 'deepskyblue', + 'dimgray', 'dimgrey', 'dodgerblue', 'firebrick', 'floralwhite', + 'forestgreen', 'fuchsia', 'gainsboro', 'ghostwhite', 'gold', 'goldenrod', + 'gray', 'green', 'greenyellow', 'grey', 'honeydew', 'hotpink', 'indianred', + 'indigo', 'ivory', 'khaki', 'lavender', 'lavenderblush', 'lawngreen', + 'lemonchiffon', 'lightblue', 'lightcoral', 'lightcyan', + 'lightgoldenrodyellow', 'lightgray', 'lightgreen', 'lightgrey', + 'lightpink', 'lightsalmon', 'lightseagreen', 'lightskyblue', + 'lightslategray', 'lightslategrey', 'lightsteelblue', 'lightyellow', + 'lime', 'limegreen', 'linen', 'magenta', 'maroon', 'mediumaquamarine', + 'mediumblue', 'mediumorchid', 'mediumpurple', 'mediumseagreen', + 'mediumslateblue', 'mediumspringgreen', 'mediumturquoise', + 'mediumvioletred', 'midnightblue', 'mintcream', 'mistyrose', 'moccasin', + 'navajowhite', 'navy', 'oldlace', 'olive', 'olivedrab', 'orange', + 'orangered', 'orchid', 'palegoldenrod', 'palegreen', 'paleturquoise', + 'palevioletred', 'papayawhip', 'peachpuff', 'peru', 'pink', 'plum', + 'powderblue', 'purple', 'rebeccapurple', 'red', 'rosybrown', 'royalblue', + 'saddlebrown', 'salmon', 'sandybrown', 'seagreen', 'seashell', 'sienna', + 'silver', 'skyblue', 'slateblue', 'slategray', 'slategrey', 'snow', + 'springgreen', 'steelblue', 'tan', 'teal', 'thistle', 'tomato', 'turquoise', + 'violet', 'wheat', 'white', 'whitesmoke', 'yellow', 'yellowgreen', +) + ('transparent',) + +# List of other keyword values from other sources: +_other_keyword_values = ( + 'above', 'aural', 'behind', 'bidi-override', 'center-left', 'center-right', + 'cjk-ideographic', 'continuous', 'crop', 'cross', 'embed', 'far-left', + 'far-right', 'fast', 'faster', 'hebrew', 'high', 'higher', 'hiragana', + 'hiragana-iroha', 'katakana', 'katakana-iroha', 'landscape', 'left-side', + 'leftwards', 'level', 'loud', 'low', 'lower', 'message-box', 'middle', + 'mix', 'narrower', 'once', 'portrait', 'right-side', 'rightwards', 'silent', + 'slow', 'slower', 'small-caption', 'soft', 'spell-out', 'status-bar', + 'super', 'text-bottom', 'text-top', 'wider', 'x-fast', 'x-high', 'x-loud', + 'x-low', 'x-soft', 'yes', 'pre', 'pre-wrap', 'pre-line', +) + +# List of functional notation and function keyword values: +_functional_notation_keyword_values = ( + 'attr', 'blackness', 'blend', 'blenda', 'blur', 'brightness', 'calc', + 'circle', 'color-mod', 'contrast', 'counter', 'cubic-bezier', 'device-cmyk', + 'drop-shadow', 'ellipse', 'gray', 'grayscale', 'hsl', 'hsla', 'hue', + 'hue-rotate', 'hwb', 'image', 'inset', 'invert', 'lightness', + 'linear-gradient', 'matrix', 'matrix3d', 'opacity', 'perspective', + 'polygon', 'radial-gradient', 'rect', 'repeating-linear-gradient', + 'repeating-radial-gradient', 'rgb', 'rgba', 'rotate', 'rotate3d', 'rotateX', + 'rotateY', 'rotateZ', 'saturate', 'saturation', 'scale', 'scale3d', + 'scaleX', 'scaleY', 'scaleZ', 'sepia', 'shade', 'skewX', 'skewY', 'steps', + 'tint', 'toggle', 'translate', 'translate3d', 'translateX', 'translateY', + 'translateZ', 'whiteness', +) +# Note! Handle url(...) separately. + +# List of units obtained from: +# https://www.w3.org/TR/css3-values/ +_angle_units = ( + 'deg', 'grad', 'rad', 'turn', +) +_frequency_units = ( + 'Hz', 'kHz', +) +_length_units = ( + 'em', 'ex', 'ch', 'rem', + 'vh', 'vw', 'vmin', 'vmax', + 'px', 'mm', 'cm', 'in', 'pt', 'pc', 'q', +) +_resolution_units = ( + 'dpi', 'dpcm', 'dppx', +) +_time_units = ( + 's', 'ms', +) +_all_units = _angle_units + _frequency_units + _length_units + \ + _resolution_units + _time_units + + class CssLexer(RegexLexer): """ For CSS (Cascading Style Sheets). @@ -39,10 +284,10 @@ class CssLexer(RegexLexer): (r'\s+', Text), (r'/\*(?:.|\n)*?\*/', Comment), (r'\{', Punctuation, 'content'), - (r'\:[\w-]+', Name.Decorator), - (r'\.[\w-]+', Name.Class), - (r'\#[\w-]+', Name.Namespace), - (r'@[\w-]+', Keyword, 'atrule'), + (r'(\:{1,2})([\w-]+)', bygroups(Punctuation, Name.Decorator)), + (r'(\.)([\w-]+)', bygroups(Punctuation, Name.Class)), + (r'(\#)([\w-]+)', bygroups(Punctuation, Name.Namespace)), + (r'(@)([\w-]+)', bygroups(Punctuation, Keyword), 'atrule'), (r'[\w-]+', Name.Tag), (r'[~^*!%&$\[\]()<>|+=@:;,./?-]', Operator), (r'"(\\\\|\\"|[^"])*"', String.Double), @@ -60,107 +305,81 @@ class CssLexer(RegexLexer): 'content': [ (r'\s+', Text), (r'\}', Punctuation, '#pop'), - (r'url\(.*?\)', String.Other), + (r';', Punctuation), (r'^@.*?$', Comment.Preproc), - (words(( - 'azimuth', 'background-attachment', 'background-color', - 'background-image', 'background-position', 'background-repeat', - 'background', 'border-bottom-color', 'border-bottom-style', - 'border-bottom-width', 'border-left-color', 'border-left-style', - 'border-left-width', 'border-right', 'border-right-color', - 'border-right-style', 'border-right-width', 'border-top-color', - 'border-top-style', 'border-top-width', 'border-bottom', - 'border-collapse', 'border-left', 'border-width', 'border-color', - 'border-spacing', 'border-style', 'border-top', 'border', 'caption-side', - 'clear', 'clip', 'color', 'content', 'counter-increment', 'counter-reset', - 'cue-after', 'cue-before', 'cue', 'cursor', 'direction', 'display', - 'elevation', 'empty-cells', 'float', 'font-family', 'font-size', - 'font-size-adjust', 'font-stretch', 'font-style', 'font-variant', - 'font-weight', 'font', 'height', 'letter-spacing', 'line-height', - 'list-style-type', 'list-style-image', 'list-style-position', - 'list-style', 'margin-bottom', 'margin-left', 'margin-right', - 'margin-top', 'margin', 'marker-offset', 'marks', 'max-height', 'max-width', - 'min-height', 'min-width', 'opacity', 'orphans', 'outline-color', - 'outline-style', 'outline-width', 'outline', 'overflow', 'overflow-x', - 'overflow-y', 'padding-bottom', 'padding-left', 'padding-right', 'padding-top', - 'padding', 'page', 'page-break-after', 'page-break-before', 'page-break-inside', - 'pause-after', 'pause-before', 'pause', 'pitch-range', 'pitch', - 'play-during', 'position', 'quotes', 'richness', 'right', 'size', - 'speak-header', 'speak-numeral', 'speak-punctuation', 'speak', - 'speech-rate', 'stress', 'table-layout', 'text-align', 'text-decoration', - 'text-indent', 'text-shadow', 'text-transform', 'top', 'unicode-bidi', - 'vertical-align', 'visibility', 'voice-family', 'volume', 'white-space', - 'widows', 'width', 'word-spacing', 'z-index', 'bottom', - 'above', 'absolute', 'always', 'armenian', 'aural', 'auto', 'avoid', 'baseline', - 'behind', 'below', 'bidi-override', 'blink', 'block', 'bolder', 'bold', 'both', - 'capitalize', 'center-left', 'center-right', 'center', 'circle', - 'cjk-ideographic', 'close-quote', 'collapse', 'condensed', 'continuous', - 'crop', 'crosshair', 'cross', 'cursive', 'dashed', 'decimal-leading-zero', - 'decimal', 'default', 'digits', 'disc', 'dotted', 'double', 'e-resize', 'embed', - 'extra-condensed', 'extra-expanded', 'expanded', 'fantasy', 'far-left', - 'far-right', 'faster', 'fast', 'fixed', 'georgian', 'groove', 'hebrew', 'help', - 'hidden', 'hide', 'higher', 'high', 'hiragana-iroha', 'hiragana', 'icon', - 'inherit', 'inline-table', 'inline', 'inset', 'inside', 'invert', 'italic', - 'justify', 'katakana-iroha', 'katakana', 'landscape', 'larger', 'large', - 'left-side', 'leftwards', 'left', 'level', 'lighter', 'line-through', 'list-item', - 'loud', 'lower-alpha', 'lower-greek', 'lower-roman', 'lowercase', 'ltr', - 'lower', 'low', 'medium', 'message-box', 'middle', 'mix', 'monospace', - 'n-resize', 'narrower', 'ne-resize', 'no-close-quote', 'no-open-quote', - 'no-repeat', 'none', 'normal', 'nowrap', 'nw-resize', 'oblique', 'once', - 'open-quote', 'outset', 'outside', 'overline', 'pointer', 'portrait', 'px', - 'relative', 'repeat-x', 'repeat-y', 'repeat', 'rgb', 'ridge', 'right-side', - 'rightwards', 's-resize', 'sans-serif', 'scroll', 'se-resize', - 'semi-condensed', 'semi-expanded', 'separate', 'serif', 'show', 'silent', - 'slower', 'slow', 'small-caps', 'small-caption', 'smaller', 'soft', 'solid', - 'spell-out', 'square', 'static', 'status-bar', 'super', 'sw-resize', - 'table-caption', 'table-cell', 'table-column', 'table-column-group', - 'table-footer-group', 'table-header-group', 'table-row', - 'table-row-group', 'text-bottom', 'text-top', 'text', 'thick', 'thin', - 'transparent', 'ultra-condensed', 'ultra-expanded', 'underline', - 'upper-alpha', 'upper-latin', 'upper-roman', 'uppercase', 'url', - 'visible', 'w-resize', 'wait', 'wider', 'x-fast', 'x-high', 'x-large', 'x-loud', - 'x-low', 'x-small', 'x-soft', 'xx-large', 'xx-small', 'yes'), suffix=r'\b'), - Name.Builtin), - (words(( - 'indigo', 'gold', 'firebrick', 'indianred', 'yellow', 'darkolivegreen', - 'darkseagreen', 'mediumvioletred', 'mediumorchid', 'chartreuse', - 'mediumslateblue', 'black', 'springgreen', 'crimson', 'lightsalmon', 'brown', - 'turquoise', 'olivedrab', 'cyan', 'silver', 'skyblue', 'gray', 'darkturquoise', - 'goldenrod', 'darkgreen', 'darkviolet', 'darkgray', 'lightpink', 'teal', - 'darkmagenta', 'lightgoldenrodyellow', 'lavender', 'yellowgreen', 'thistle', - 'violet', 'navy', 'orchid', 'blue', 'ghostwhite', 'honeydew', 'cornflowerblue', - 'darkblue', 'darkkhaki', 'mediumpurple', 'cornsilk', 'red', 'bisque', 'slategray', - 'darkcyan', 'khaki', 'wheat', 'deepskyblue', 'darkred', 'steelblue', 'aliceblue', - 'gainsboro', 'mediumturquoise', 'floralwhite', 'coral', 'purple', 'lightgrey', - 'lightcyan', 'darksalmon', 'beige', 'azure', 'lightsteelblue', 'oldlace', - 'greenyellow', 'royalblue', 'lightseagreen', 'mistyrose', 'sienna', - 'lightcoral', 'orangered', 'navajowhite', 'lime', 'palegreen', 'burlywood', - 'seashell', 'mediumspringgreen', 'fuchsia', 'papayawhip', 'blanchedalmond', - 'peru', 'aquamarine', 'white', 'darkslategray', 'ivory', 'dodgerblue', - 'lemonchiffon', 'chocolate', 'orange', 'forestgreen', 'slateblue', 'olive', - 'mintcream', 'antiquewhite', 'darkorange', 'cadetblue', 'moccasin', - 'limegreen', 'saddlebrown', 'darkslateblue', 'lightskyblue', 'deeppink', - 'plum', 'aqua', 'darkgoldenrod', 'maroon', 'sandybrown', 'magenta', 'tan', - 'rosybrown', 'pink', 'lightblue', 'palevioletred', 'mediumseagreen', - 'dimgray', 'powderblue', 'seagreen', 'snow', 'mediumblue', 'midnightblue', - 'paleturquoise', 'palegoldenrod', 'whitesmoke', 'darkorchid', 'salmon', - 'lightslategray', 'lawngreen', 'lightgreen', 'tomato', 'hotpink', - 'lightyellow', 'lavenderblush', 'linen', 'mediumaquamarine', 'green', - 'blueviolet', 'peachpuff'), suffix=r'\b'), - Name.Builtin), + + (words(_vendor_prefixes,), Keyword.Pseudo), + (r'('+r'|'.join(_css_properties)+r')(\s*)(\:)', + bygroups(Keyword, Text, Punctuation), 'value-start'), + (r'([a-zA-Z_][\w-]*)(\s*)(\:)', bygroups(Name, Text, Punctuation), + 'value-start'), + + (r'/\*(?:.|\n)*?\*/', Comment), + ], + 'value-start': [ + (r'\s+', Text), + (words(_vendor_prefixes,), Name.Builtin.Pseudo), + include('urls'), + (r'('+r'|'.join(_functional_notation_keyword_values)+r')(\()', + bygroups(Name.Builtin, Punctuation), 'function-start'), + (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'), + (words(_keyword_values, suffix=r'\b'), Keyword.Constant), + (words(_other_keyword_values, suffix=r'\b'), Keyword.Constant), + (words(_color_keywords, suffix=r'\b'), Keyword.Constant), + (words(_css_properties, suffix=r'\b'), Keyword), # for transition-property etc. (r'\!important', Comment.Preproc), (r'/\*(?:.|\n)*?\*/', Comment), - (r'\#[a-zA-Z0-9]{1,6}', Number), - (r'[.-]?[0-9]*[.]?[0-9]+(em|px|pt|pc|in|mm|cm|ex|s)\b', Number), - # Separate regex for percentages, as can't do word boundaries with % - (r'[.-]?[0-9]*[.]?[0-9]+%', Number), - (r'-?[0-9]+', Number), - (r'[~^*!%&<>|+=@:,./?-]+', Operator), - (r'[\[\]();]+', Punctuation), + + include('numeric-values'), + + (r'[~^*!%&<>|+=@:./?-]+', Operator), + (r'[\[\](),]+', Punctuation), (r'"(\\\\|\\"|[^"])*"', String.Double), (r"'(\\\\|\\'|[^'])*'", String.Single), - (r'[a-zA-Z_]\w*', Name) - ] + (r'[a-zA-Z_][\w-]*', Name), + (r';', Punctuation, '#pop'), + (r'\}', Punctuation, '#pop:2'), + ], + 'function-start': [ + (r'\s+', Text), + include('urls'), + (words(_vendor_prefixes,), Keyword.Pseudo), + (words(_keyword_values, suffix=r'\b'), Keyword.Constant), + (words(_other_keyword_values, suffix=r'\b'), Keyword.Constant), + (words(_color_keywords, suffix=r'\b'), Keyword.Constant), + + # function-start may be entered recursively + (r'(' + r'|'.join(_functional_notation_keyword_values) + r')(\()', + bygroups(Name.Builtin, Punctuation), 'function-start'), + (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'), + + (r'/\*(?:.|\n)*?\*/', Comment), + include('numeric-values'), + (r'[*+/-]', Operator), + (r'[,]', Punctuation), + (r'"(\\\\|\\"|[^"])*"', String.Double), + (r"'(\\\\|\\'|[^'])*'", String.Single), + (r'[a-zA-Z_-]\w*', Name), + (r'\)', Punctuation, '#pop'), + ], + 'urls': [ + (r'(url)(\()(".*?")(\))', bygroups(Name.Builtin, Punctuation, + String.Double, Punctuation)), + (r"(url)(\()('.*?')(\))", bygroups(Name.Builtin, Punctuation, + String.Single, Punctuation)), + (r'(url)(\()(.*?)(\))', bygroups(Name.Builtin, Punctuation, + String.Other, Punctuation)), + ], + 'numeric-values': [ + (r'\#[a-zA-Z0-9]{1,6}', Number.Hex), + (r'[+\-]?[0-9]*[.][0-9]+', Number.Float, 'numeric-end'), + (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'), + ], + 'numeric-end': [ + (words(_all_units, suffix=r'\b'), Keyword.Type), + (r'%', Keyword.Type), + default('#pop'), + ], } @@ -170,35 +389,7 @@ common_sass_tokens = { (r'[!$][\w-]+', Name.Variable), (r'url\(', String.Other, 'string-url'), (r'[a-z_-][\w-]*(?=\()', Name.Function), - (words(( - 'azimuth', 'background-attachment', 'background-color', - 'background-image', 'background-position', 'background-repeat', - 'background', 'border-bottom-color', 'border-bottom-style', - 'border-bottom-width', 'border-left-color', 'border-left-style', - 'border-left-width', 'border-right', 'border-right-color', - 'border-right-style', 'border-right-width', 'border-top-color', - 'border-top-style', 'border-top-width', 'border-bottom', - 'border-collapse', 'border-left', 'border-width', 'border-color', - 'border-spacing', 'border-style', 'border-top', 'border', 'caption-side', - 'clear', 'clip', 'color', 'content', 'counter-increment', 'counter-reset', - 'cue-after', 'cue-before', 'cue', 'cursor', 'direction', 'display', - 'elevation', 'empty-cells', 'float', 'font-family', 'font-size', - 'font-size-adjust', 'font-stretch', 'font-style', 'font-variant', - 'font-weight', 'font', 'height', 'letter-spacing', 'line-height', - 'list-style-type', 'list-style-image', 'list-style-position', - 'list-style', 'margin-bottom', 'margin-left', 'margin-right', - 'margin-top', 'margin', 'marker-offset', 'marks', 'max-height', 'max-width', - 'min-height', 'min-width', 'opacity', 'orphans', 'outline', 'outline-color', - 'outline-style', 'outline-width', 'overflow', 'padding-bottom', - 'padding-left', 'padding-right', 'padding-top', 'padding', 'page', - 'page-break-after', 'page-break-before', 'page-break-inside', - 'pause-after', 'pause-before', 'pause', 'pitch', 'pitch-range', - 'play-during', 'position', 'quotes', 'richness', 'right', 'size', - 'speak-header', 'speak-numeral', 'speak-punctuation', 'speak', - 'speech-rate', 'stress', 'table-layout', 'text-align', 'text-decoration', - 'text-indent', 'text-shadow', 'text-transform', 'top', 'unicode-bidi', - 'vertical-align', 'visibility', 'voice-family', 'volume', 'white-space', - 'widows', 'width', 'word-spacing', 'z-index', 'bottom', 'left', + (words(_css_properties + ( 'above', 'absolute', 'always', 'armenian', 'aural', 'auto', 'avoid', 'baseline', 'behind', 'below', 'bidi-override', 'blink', 'block', 'bold', 'bolder', 'both', 'capitalize', 'center-left', 'center-right', 'center', 'circle', @@ -229,33 +420,7 @@ common_sass_tokens = { 'visible', 'w-resize', 'wait', 'wider', 'x-fast', 'x-high', 'x-large', 'x-loud', 'x-low', 'x-small', 'x-soft', 'xx-large', 'xx-small', 'yes'), suffix=r'\b'), Name.Constant), - (words(( - 'indigo', 'gold', 'firebrick', 'indianred', 'darkolivegreen', - 'darkseagreen', 'mediumvioletred', 'mediumorchid', 'chartreuse', - 'mediumslateblue', 'springgreen', 'crimson', 'lightsalmon', 'brown', - 'turquoise', 'olivedrab', 'cyan', 'skyblue', 'darkturquoise', - 'goldenrod', 'darkgreen', 'darkviolet', 'darkgray', 'lightpink', - 'darkmagenta', 'lightgoldenrodyellow', 'lavender', 'yellowgreen', 'thistle', - 'violet', 'orchid', 'ghostwhite', 'honeydew', 'cornflowerblue', - 'darkblue', 'darkkhaki', 'mediumpurple', 'cornsilk', 'bisque', 'slategray', - 'darkcyan', 'khaki', 'wheat', 'deepskyblue', 'darkred', 'steelblue', 'aliceblue', - 'gainsboro', 'mediumturquoise', 'floralwhite', 'coral', 'lightgrey', - 'lightcyan', 'darksalmon', 'beige', 'azure', 'lightsteelblue', 'oldlace', - 'greenyellow', 'royalblue', 'lightseagreen', 'mistyrose', 'sienna', - 'lightcoral', 'orangered', 'navajowhite', 'palegreen', 'burlywood', - 'seashell', 'mediumspringgreen', 'papayawhip', 'blanchedalmond', - 'peru', 'aquamarine', 'darkslategray', 'ivory', 'dodgerblue', - 'lemonchiffon', 'chocolate', 'orange', 'forestgreen', 'slateblue', - 'mintcream', 'antiquewhite', 'darkorange', 'cadetblue', 'moccasin', - 'limegreen', 'saddlebrown', 'darkslateblue', 'lightskyblue', 'deeppink', - 'plum', 'darkgoldenrod', 'sandybrown', 'magenta', 'tan', - 'rosybrown', 'pink', 'lightblue', 'palevioletred', 'mediumseagreen', - 'dimgray', 'powderblue', 'seagreen', 'snow', 'mediumblue', 'midnightblue', - 'paleturquoise', 'palegoldenrod', 'whitesmoke', 'darkorchid', 'salmon', - 'lightslategray', 'lawngreen', 'lightgreen', 'tomato', 'hotpink', - 'lightyellow', 'lavenderblush', 'linen', 'mediumaquamarine', - 'blueviolet', 'peachpuff'), suffix=r'\b'), - Name.Entity), + (words(_color_keywords, suffix=r'\b'), Name.Entity), (words(( 'black', 'silver', 'gray', 'white', 'maroon', 'red', 'purple', 'fuchsia', 'green', 'lime', 'olive', 'yellow', 'navy', 'blue', 'teal', 'aqua'), suffix=r'\b'), @@ -476,8 +641,8 @@ class ScssLexer(RegexLexer): (r'@[\w-]+', Keyword, 'selector'), (r'(\$[\w-]*\w)([ \t]*:)', bygroups(Name.Variable, Operator), 'value'), # TODO: broken, and prone to infinite loops. - #(r'(?=[^;{}][;}])', Name.Attribute, 'attr'), - #(r'(?=[^;{}:]+:[^a-z])', Name.Attribute, 'attr'), + # (r'(?=[^;{}][;}])', Name.Attribute, 'attr'), + # (r'(?=[^;{}:]+:[^a-z])', Name.Attribute, 'attr'), default('selector'), ], @@ -518,7 +683,7 @@ class LessCssLexer(CssLexer): inherit, ], 'content': [ - (r'{', Punctuation, '#push'), + (r'\{', Punctuation, '#push'), inherit, ], } diff --git a/pygments/lexers/d.py b/pygments/lexers/d.py index 98e01dcf..09e6fe87 100644 --- a/pygments/lexers/d.py +++ b/pygments/lexers/d.py @@ -5,7 +5,7 @@ Lexers for D languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/dalvik.py b/pygments/lexers/dalvik.py index 2f26fa04..c211f13e 100644 --- a/pygments/lexers/dalvik.py +++ b/pygments/lexers/dalvik.py @@ -5,7 +5,7 @@ Pygments lexers for Dalvik VM-related languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/data.py b/pygments/lexers/data.py index c88375d5..296366c2 100644 --- a/pygments/lexers/data.py +++ b/pygments/lexers/data.py @@ -5,7 +5,7 @@ Lexers for data file format. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -14,9 +14,9 @@ import re from pygments.lexer import RegexLexer, ExtendedRegexLexer, LexerContext, \ include, bygroups, inherit from pygments.token import Text, Comment, Keyword, Name, String, Number, \ - Punctuation, Literal + Punctuation, Literal, Error -__all__ = ['YamlLexer', 'JsonLexer', 'JsonLdLexer'] +__all__ = ['YamlLexer', 'JsonLexer', 'JsonBareObjectLexer', 'JsonLdLexer'] class YamlLexerContext(LexerContext): @@ -247,10 +247,10 @@ class YamlLexer(ExtendedRegexLexer): # tags, anchors, aliases 'descriptors': [ # a full-form tag - (r'!<[\w;/?:@&=+$,.!~*\'()\[\]%-]+>', Keyword.Type), + (r'!<[\w#;/?:@&=+$,.!~*\'()\[\]%-]+>', Keyword.Type), # a tag in the form '!', '!suffix' or '!handle!suffix' - (r'!(?:[\w-]+)?' - r'(?:![\w;/?:@&=+$,.!~*\'()\[\]%-]+)?', Keyword.Type), + (r'!(?:[\w-]+!)?' + r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]+', Keyword.Type), # an anchor (r'&[\w-]+', Name.Label), # an alias @@ -476,7 +476,7 @@ class JsonLexer(RegexLexer): # comma terminates the attribute but expects more (r',', Punctuation, '#pop'), # a closing bracket terminates the entire object, so pop twice - (r'\}', Punctuation, ('#pop', '#pop')), + (r'\}', Punctuation, '#pop:2'), ], # a json object - { attr, attr, ... } @@ -508,6 +508,31 @@ class JsonLexer(RegexLexer): ], } + +class JsonBareObjectLexer(JsonLexer): + """ + For JSON data structures (with missing object curly braces). + + .. versionadded:: 2.2 + """ + + name = 'JSONBareObject' + aliases = ['json-object'] + filenames = [] + mimetypes = ['application/json-object'] + + tokens = { + 'root': [ + (r'\}', Error), + include('objectvalue'), + ], + 'objectattribute': [ + (r'\}', Error), + inherit, + ], + } + + class JsonLdLexer(JsonLexer): """ For `JSON-LD <http://json-ld.org/>`_ linked data. diff --git a/pygments/lexers/diff.py b/pygments/lexers/diff.py index d3b1589d..f7019440 100644 --- a/pygments/lexers/diff.py +++ b/pygments/lexers/diff.py @@ -5,15 +5,17 @@ Lexers for diff/patch formats. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ +import re + from pygments.lexer import RegexLexer, include, bygroups from pygments.token import Text, Comment, Operator, Keyword, Name, Generic, \ Literal -__all__ = ['DiffLexer', 'DarcsPatchLexer'] +__all__ = ['DiffLexer', 'DarcsPatchLexer', 'WDiffLexer'] class DiffLexer(RegexLexer): @@ -104,3 +106,60 @@ class DarcsPatchLexer(RegexLexer): (r'[^\n\[]+', Generic.Deleted), ], } + + +class WDiffLexer(RegexLexer): + """ + A `wdiff <https://www.gnu.org/software/wdiff/>`_ lexer. + + Note that: + + * only to normal output (without option like -l). + * if target files of wdiff contain "[-", "-]", "{+", "+}", + especially they are unbalanced, this lexer will get confusing. + + .. versionadded:: 2.2 + """ + + name = 'WDiff' + aliases = ['wdiff'] + filenames = ['*.wdiff'] + mimetypes = [] + + flags = re.MULTILINE | re.DOTALL + + # We can only assume "[-" after "[-" before "-]" is `nested`, + # for instance wdiff to wdiff outputs. We have no way to + # distinct these marker is of wdiff output from original text. + + ins_op = r"\{\+" + ins_cl = r"\+\}" + del_op = r"\[\-" + del_cl = r"\-\]" + normal = r'[^{}[\]+-]+' # for performance + tokens = { + 'root': [ + (ins_op, Generic.Inserted, 'inserted'), + (del_op, Generic.Deleted, 'deleted'), + (normal, Text), + (r'.', Text), + ], + 'inserted': [ + (ins_op, Generic.Inserted, '#push'), + (del_op, Generic.Inserted, '#push'), + (del_cl, Generic.Inserted, '#pop'), + + (ins_cl, Generic.Inserted, '#pop'), + (normal, Generic.Inserted), + (r'.', Generic.Inserted), + ], + 'deleted': [ + (del_op, Generic.Deleted, '#push'), + (ins_op, Generic.Deleted, '#push'), + (ins_cl, Generic.Deleted, '#pop'), + + (del_cl, Generic.Deleted, '#pop'), + (normal, Generic.Deleted), + (r'.', Generic.Deleted), + ], + } diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py index eac4b5e5..4e2bc8ab 100644 --- a/pygments/lexers/dotnet.py +++ b/pygments/lexers/dotnet.py @@ -5,13 +5,13 @@ Lexers for .net languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re from pygments.lexer import RegexLexer, DelegatingLexer, bygroups, include, \ - using, this, default + using, this, default, words from pygments.token import Punctuation, \ Text, Comment, Operator, Keyword, Name, String, Number, Literal, Other from pygments.util import get_choice_opt, iteritems @@ -375,8 +375,8 @@ class VbNetLexer(RegexLexer): filenames = ['*.vb', '*.bas'] mimetypes = ['text/x-vbnet', 'text/x-vba'] # (?) - uni_name = '[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' + \ - '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc', + uni_name = '[_' + uni.combine('Ll', 'Lt', 'Lm', 'Nl') + ']' + \ + '[' + uni.combine('Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc', 'Cf', 'Mn', 'Mc') + ']*' flags = re.MULTILINE | re.IGNORECASE @@ -394,25 +394,26 @@ class VbNetLexer(RegexLexer): (r'[(){}!#,.:]', Punctuation), (r'Option\s+(Strict|Explicit|Compare)\s+' r'(On|Off|Binary|Text)', Keyword.Declaration), - (r'(?<!\.)(AddHandler|Alias|' - r'ByRef|ByVal|Call|Case|Catch|CBool|CByte|CChar|CDate|' - r'CDec|CDbl|CInt|CLng|CObj|Continue|CSByte|CShort|' - r'CSng|CStr|CType|CUInt|CULng|CUShort|Declare|' - r'Default|Delegate|DirectCast|Do|Each|Else|ElseIf|' - r'EndIf|Erase|Error|Event|Exit|False|Finally|For|' - r'Friend|Get|Global|GoSub|GoTo|Handles|If|' - r'Implements|Inherits|Interface|' - r'Let|Lib|Loop|Me|MustInherit|' - r'MustOverride|MyBase|MyClass|Narrowing|New|Next|' - r'Not|Nothing|NotInheritable|NotOverridable|Of|On|' - r'Operator|Option|Optional|Overloads|Overridable|' - r'Overrides|ParamArray|Partial|Private|Protected|' - r'Public|RaiseEvent|ReadOnly|ReDim|RemoveHandler|Resume|' - r'Return|Select|Set|Shadows|Shared|Single|' - r'Static|Step|Stop|SyncLock|Then|' - r'Throw|To|True|Try|TryCast|Wend|' - r'Using|When|While|Widening|With|WithEvents|' - r'WriteOnly)\b', Keyword), + (words(( + 'AddHandler', 'Alias', 'ByRef', 'ByVal', 'Call', 'Case', + 'Catch', 'CBool', 'CByte', 'CChar', 'CDate', 'CDec', 'CDbl', + 'CInt', 'CLng', 'CObj', 'Continue', 'CSByte', 'CShort', 'CSng', + 'CStr', 'CType', 'CUInt', 'CULng', 'CUShort', 'Declare', + 'Default', 'Delegate', 'DirectCast', 'Do', 'Each', 'Else', + 'ElseIf', 'EndIf', 'Erase', 'Error', 'Event', 'Exit', 'False', + 'Finally', 'For', 'Friend', 'Get', 'Global', 'GoSub', 'GoTo', + 'Handles', 'If', 'Implements', 'Inherits', 'Interface', 'Let', + 'Lib', 'Loop', 'Me', 'MustInherit', 'MustOverride', 'MyBase', + 'MyClass', 'Narrowing', 'New', 'Next', 'Not', 'Nothing', + 'NotInheritable', 'NotOverridable', 'Of', 'On', 'Operator', + 'Option', 'Optional', 'Overloads', 'Overridable', 'Overrides', + 'ParamArray', 'Partial', 'Private', 'Protected', 'Public', + 'RaiseEvent', 'ReadOnly', 'ReDim', 'RemoveHandler', 'Resume', + 'Return', 'Select', 'Set', 'Shadows', 'Shared', 'Single', + 'Static', 'Step', 'Stop', 'SyncLock', 'Then', 'Throw', 'To', + 'True', 'Try', 'TryCast', 'Wend', 'Using', 'When', 'While', + 'Widening', 'With', 'WithEvents', 'WriteOnly'), + prefix='(?<!\.)', suffix=r'\b'), Keyword), (r'(?<!\.)End\b', Keyword, 'end'), (r'(?<!\.)(Dim|Const)\b', Keyword, 'dim'), (r'(?<!\.)(Function|Sub|Property)(\s+)', diff --git a/pygments/lexers/dsls.py b/pygments/lexers/dsls.py index 24fda2a2..a1426bd6 100644 --- a/pygments/lexers/dsls.py +++ b/pygments/lexers/dsls.py @@ -5,20 +5,20 @@ Lexers for various domain-specific languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re -from pygments.lexer import RegexLexer, bygroups, words, include, default, \ - this, using, combined +from pygments.lexer import ExtendedRegexLexer, RegexLexer, bygroups, words, \ + include, default, this, using, combined from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Literal, Whitespace __all__ = ['ProtoBufLexer', 'BroLexer', 'PuppetLexer', 'RslLexer', 'MscgenLexer', 'VGLLexer', 'AlloyLexer', 'PanLexer', - 'CrmshLexer', 'ThriftLexer'] + 'CrmshLexer', 'ThriftLexer', 'FlatlineLexer', 'SnowballLexer'] class ProtoBufLexer(RegexLexer): @@ -36,7 +36,7 @@ class ProtoBufLexer(RegexLexer): tokens = { 'root': [ (r'[ \t]+', Text), - (r'[,;{}\[\]()]', Punctuation), + (r'[,;{}\[\]()<>]', Punctuation), (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single), (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment.Multiline), (words(( @@ -111,8 +111,8 @@ class ThriftLexer(RegexLexer): include('keywords'), include('numbers'), (r'[&=]', Operator), - (r'[:;\,\{\}\(\)\<>\[\]]', Punctuation), - (r'[a-zA-Z_](\.[a-zA-Z_0-9]|[a-zA-Z_0-9])*', Name), + (r'[:;,{}()<>\[\]]', Punctuation), + (r'[a-zA-Z_](\.\w|\w)*', Name), ], 'whitespace': [ (r'\n', Text.Whitespace), @@ -135,7 +135,7 @@ class ThriftLexer(RegexLexer): (r'[^\\\'\n]+', String.Single), ], 'namespace': [ - (r'[a-z\*](\.[a-zA-Z_0-9]|[a-zA-Z_0-9])*', Name.Namespace, '#pop'), + (r'[a-z*](\.\w|\w)*', Name.Namespace, '#pop'), default('#pop'), ], 'class': [ @@ -156,7 +156,7 @@ class ThriftLexer(RegexLexer): Keyword.Namespace), (words(( 'void', 'bool', 'byte', 'i16', 'i32', 'i64', 'double', - 'string', 'binary', 'void', 'map', 'list', 'set', 'slist', + 'string', 'binary', 'map', 'list', 'set', 'slist', 'senum'), suffix=r'\b'), Keyword.Type), (words(( @@ -581,7 +581,7 @@ class PanLexer(RegexLexer): 'if', 'for', 'with', 'else', 'type', 'bind', 'while', 'valid', 'final', 'prefix', 'unique', 'object', 'foreach', 'include', 'template', 'function', 'variable', 'structure', 'extensible', 'declaration'), - prefix=r'\b', suffix=r'\s*\b'), + prefix=r'\b', suffix=r'\s*\b'), Keyword), (words(( 'file_contents', 'format', 'index', 'length', 'match', 'matches', @@ -593,7 +593,7 @@ class PanLexer(RegexLexer): 'is_number', 'is_property', 'is_resource', 'is_string', 'to_boolean', 'to_double', 'to_long', 'to_string', 'clone', 'delete', 'exists', 'path_exists', 'if_exists', 'return', 'value'), - prefix=r'\b', suffix=r'\s*\b'), + prefix=r'\b', suffix=r'\s*\b'), Name.Builtin), (r'#.*', Comment), (r'\\[\w\W]', String.Escape), @@ -692,3 +692,187 @@ class CrmshLexer(RegexLexer): (r'\s+|\n', Whitespace), ], } + + +class FlatlineLexer(RegexLexer): + """ + Lexer for `Flatline <https://github.com/bigmlcom/flatline>`_ expressions. + + .. versionadded:: 2.2 + """ + name = 'Flatline' + aliases = ['flatline'] + filenames = [] + mimetypes = ['text/x-flatline'] + + special_forms = ('let',) + + builtins = ( + "!=", "*", "+", "-", "<", "<=", "=", ">", ">=", "abs", "acos", "all", + "all-but", "all-with-defaults", "all-with-numeric-default", "and", + "asin", "atan", "avg", "avg-window", "bin-center", "bin-count", "call", + "category-count", "ceil", "cond", "cond-window", "cons", "cos", "cosh", + "count", "diff-window", "div", "ensure-value", "ensure-weighted-value", + "epoch", "epoch-day", "epoch-fields", "epoch-hour", "epoch-millisecond", + "epoch-minute", "epoch-month", "epoch-second", "epoch-weekday", + "epoch-year", "exp", "f", "field", "field-prop", "fields", "filter", + "first", "floor", "head", "if", "in", "integer", "language", "length", + "levenshtein", "linear-regression", "list", "ln", "log", "log10", "map", + "matches", "matches?", "max", "maximum", "md5", "mean", "median", "min", + "minimum", "missing", "missing-count", "missing?", "missing_count", + "mod", "mode", "normalize", "not", "nth", "occurrences", "or", + "percentile", "percentile-label", "population", "population-fraction", + "pow", "preferred", "preferred?", "quantile-label", "rand", "rand-int", + "random-value", "re-quote", "real", "replace", "replace-first", "rest", + "round", "row-number", "segment-label", "sha1", "sha256", "sin", "sinh", + "sqrt", "square", "standard-deviation", "standard_deviation", "str", + "subs", "sum", "sum-squares", "sum-window", "sum_squares", "summary", + "summary-no", "summary-str", "tail", "tan", "tanh", "to-degrees", + "to-radians", "variance", "vectorize", "weighted-random-value", "window", + "winnow", "within-percentiles?", "z-score", + ) + + valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+' + + tokens = { + 'root': [ + # whitespaces - usually not relevant + (r'[,\s]+', Text), + + # numbers + (r'-?\d+\.\d+', Number.Float), + (r'-?\d+', Number.Integer), + (r'0x-?[a-f\d]+', Number.Hex), + + # strings, symbols and characters + (r'"(\\\\|\\"|[^"])*"', String), + (r"\\(.|[a-z]+)", String.Char), + + # expression template placeholder + (r'_', String.Symbol), + + # highlight the special forms + (words(special_forms, suffix=' '), Keyword), + + # highlight the builtins + (words(builtins, suffix=' '), Name.Builtin), + + # the remaining functions + (r'(?<=\()' + valid_name, Name.Function), + + # find the remaining variables + (valid_name, Name.Variable), + + # parentheses + (r'(\(|\))', Punctuation), + ], + } + + +class SnowballLexer(ExtendedRegexLexer): + """ + Lexer for `Snowball <http://snowballstem.org/>`_ source code. + + .. versionadded:: 2.2 + """ + + name = 'Snowball' + aliases = ['snowball'] + filenames = ['*.sbl'] + + _ws = r'\n\r\t ' + + def __init__(self, **options): + self._reset_stringescapes() + ExtendedRegexLexer.__init__(self, **options) + + def _reset_stringescapes(self): + self._start = "'" + self._end = "'" + + def _string(do_string_first): + def callback(lexer, match, ctx): + s = match.start() + text = match.group() + string = re.compile(r'([^%s]*)(.)' % re.escape(lexer._start)).match + escape = re.compile(r'([^%s]*)(.)' % re.escape(lexer._end)).match + pos = 0 + do_string = do_string_first + while pos < len(text): + if do_string: + match = string(text, pos) + yield s + match.start(1), String.Single, match.group(1) + if match.group(2) == "'": + yield s + match.start(2), String.Single, match.group(2) + ctx.stack.pop() + break + yield s + match.start(2), String.Escape, match.group(2) + pos = match.end() + match = escape(text, pos) + yield s + match.start(), String.Escape, match.group() + if match.group(2) != lexer._end: + ctx.stack[-1] = 'escape' + break + pos = match.end() + do_string = True + ctx.pos = s + match.end() + return callback + + def _stringescapes(lexer, match, ctx): + lexer._start = match.group(3) + lexer._end = match.group(5) + return bygroups(Keyword.Reserved, Text, String.Escape, Text, + String.Escape)(lexer, match, ctx) + + tokens = { + 'root': [ + (words(('len', 'lenof'), suffix=r'\b'), Operator.Word), + include('root1'), + ], + 'root1': [ + (r'[%s]+' % _ws, Text), + (r'\d+', Number.Integer), + (r"'", String.Single, 'string'), + (r'[()]', Punctuation), + (r'/\*[\w\W]*?\*/', Comment.Multiline), + (r'//.*', Comment.Single), + (r'[!*+\-/<=>]=|[-=]>|<[+-]|[$*+\-/<=>?\[\]]', Operator), + (words(('as', 'get', 'hex', 'among', 'define', 'decimal', + 'backwardmode'), suffix=r'\b'), + Keyword.Reserved), + (words(('strings', 'booleans', 'integers', 'routines', 'externals', + 'groupings'), suffix=r'\b'), + Keyword.Reserved, 'declaration'), + (words(('do', 'or', 'and', 'for', 'hop', 'non', 'not', 'set', 'try', + 'fail', 'goto', 'loop', 'next', 'test', 'true', + 'false', 'unset', 'atmark', 'attach', 'delete', 'gopast', + 'insert', 'repeat', 'sizeof', 'tomark', 'atleast', + 'atlimit', 'reverse', 'setmark', 'tolimit', 'setlimit', + 'backwards', 'substring'), suffix=r'\b'), + Operator.Word), + (words(('size', 'limit', 'cursor', 'maxint', 'minint'), + suffix=r'\b'), + Name.Builtin), + (r'(stringdef\b)([%s]*)([^%s]+)' % (_ws, _ws), + bygroups(Keyword.Reserved, Text, String.Escape)), + (r'(stringescapes\b)([%s]*)(.)([%s]*)(.)' % (_ws, _ws), + _stringescapes), + (r'[A-Za-z]\w*', Name), + ], + 'declaration': [ + (r'\)', Punctuation, '#pop'), + (words(('len', 'lenof'), suffix=r'\b'), Name, + ('root1', 'declaration')), + include('root1'), + ], + 'string': [ + (r"[^']*'", _string(True)), + ], + 'escape': [ + (r"[^']*'", _string(False)), + ], + } + + def get_tokens_unprocessed(self, text=None, context=None): + self._reset_stringescapes() + return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context) diff --git a/pygments/lexers/dylan.py b/pygments/lexers/dylan.py index 600a78e5..f61bb60d 100644 --- a/pygments/lexers/dylan.py +++ b/pygments/lexers/dylan.py @@ -5,7 +5,7 @@ Lexers for the Dylan language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/ecl.py b/pygments/lexers/ecl.py index 95572ba7..bd80ad19 100644 --- a/pygments/lexers/ecl.py +++ b/pygments/lexers/ecl.py @@ -5,7 +5,7 @@ Lexers for the ECL language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/eiffel.py b/pygments/lexers/eiffel.py index 8a244613..a90ab0a5 100644 --- a/pygments/lexers/eiffel.py +++ b/pygments/lexers/eiffel.py @@ -5,7 +5,7 @@ Lexer for the Eiffel language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/elm.py b/pygments/lexers/elm.py index 7df6346a..0fa36367 100644 --- a/pygments/lexers/elm.py +++ b/pygments/lexers/elm.py @@ -5,7 +5,7 @@ Lexer for the Elm programming language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -46,7 +46,7 @@ class ElmLexer(RegexLexer): 'root': [ # Comments - (r'{-', Comment.Multiline, 'comment'), + (r'\{-', Comment.Multiline, 'comment'), (r'--.*', Comment.Single), # Whitespace @@ -86,20 +86,20 @@ class ElmLexer(RegexLexer): (validName, Name.Variable), # Parens - (r'[,\(\)\[\]{}]', Punctuation), + (r'[,()\[\]{}]', Punctuation), ], 'comment': [ - (r'-(?!})', Comment.Multiline), - (r'{-', Comment.Multiline, 'comment'), + (r'-(?!\})', Comment.Multiline), + (r'\{-', Comment.Multiline, 'comment'), (r'[^-}]', Comment.Multiline), - (r'-}', Comment.Multiline, '#pop'), + (r'-\}', Comment.Multiline, '#pop'), ], 'doublequote': [ - (r'\\u[0-9a-fA-F]\{4}', String.Escape), - (r'\\[nrfvb\\\"]', String.Escape), + (r'\\u[0-9a-fA-F]{4}', String.Escape), + (r'\\[nrfvb\\"]', String.Escape), (r'[^"]', String), (r'"', String, '#pop'), ], diff --git a/pygments/lexers/erlang.py b/pygments/lexers/erlang.py index c353a4dc..9e7f85c1 100644 --- a/pygments/lexers/erlang.py +++ b/pygments/lexers/erlang.py @@ -5,7 +5,7 @@ Lexers for Erlang. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -82,7 +82,11 @@ class ErlangLexer(RegexLexer): variable_re = r'(?:[A-Z_]\w*)' - escape_re = r'(?:\\(?:[bdefnrstv\'"\\/]|[0-7][0-7]?[0-7]?|\^[a-zA-Z]))' + esc_char_re = r'[bdefnrstv\'"\\]' + esc_octal_re = r'[0-7][0-7]?[0-7]?' + esc_hex_re = r'(?:x[0-9a-fA-F]{2}|x\{[0-9a-fA-F]+\})' + esc_ctrl_re = r'\^[a-zA-Z]' + escape_re = r'(?:\\(?:'+esc_char_re+r'|'+esc_octal_re+r'|'+esc_hex_re+r'|'+esc_ctrl_re+r'))' macro_re = r'(?:'+variable_re+r'|'+atom_re+r')' @@ -112,11 +116,18 @@ class ErlangLexer(RegexLexer): (r'\?'+macro_re, Name.Constant), (r'\$(?:'+escape_re+r'|\\[ %]|[^\\])', String.Char), (r'#'+atom_re+r'(:?\.'+atom_re+r')?', Name.Label), + + # Erlang script shebang + (r'\A#!.+\n', Comment.Hashbang), + + # EEP 43: Maps + # http://www.erlang.org/eeps/eep-0043.html + (r'#\{', Punctuation, 'map_key'), ], 'string': [ (escape_re, String.Escape), (r'"', String, '#pop'), - (r'~[0-9.*]*[~#+bBcdefginpPswWxX]', String.Interpol), + (r'~[0-9.*]*[~#+BPWXb-ginpswx]', String.Interpol), (r'[^"\\~]+', String), (r'~', String), ], @@ -127,6 +138,17 @@ class ErlangLexer(RegexLexer): bygroups(Name.Entity, Text, Punctuation, Name.Label), '#pop'), (atom_re, Name.Entity, '#pop'), ], + 'map_key': [ + include('root'), + (r'=>', Punctuation, 'map_val'), + (r':=', Punctuation, 'map_val'), + (r'\}', Punctuation, '#pop'), + ], + 'map_val': [ + include('root'), + (r',', Punctuation, '#pop'), + (r'(?=\})', Punctuation, '#pop'), + ], } @@ -218,11 +240,11 @@ class ElixirLexer(RegexLexer): KEYWORD_OPERATOR = ('not', 'and', 'or', 'when', 'in') BUILTIN = ( 'case', 'cond', 'for', 'if', 'unless', 'try', 'receive', 'raise', - 'quote', 'unquote', 'unquote_splicing', 'throw', 'super' + 'quote', 'unquote', 'unquote_splicing', 'throw', 'super', ) BUILTIN_DECLARATION = ( 'def', 'defp', 'defmodule', 'defprotocol', 'defmacro', 'defmacrop', - 'defdelegate', 'defexception', 'defstruct', 'defimpl', 'defcallback' + 'defdelegate', 'defexception', 'defstruct', 'defimpl', 'defcallback', ) BUILTIN_NAMESPACE = ('import', 'require', 'use', 'alias') @@ -241,7 +263,7 @@ class ElixirLexer(RegexLexer): OPERATORS1 = ('<', '>', '+', '-', '*', '/', '!', '^', '&') PUNCTUATION = ( - '\\\\', '<<', '>>', '=>', '(', ')', ':', ';', ',', '[', ']' + '\\\\', '<<', '>>', '=>', '(', ')', ':', ';', ',', '[', ']', ) def get_tokens_unprocessed(self, text): diff --git a/pygments/lexers/esoteric.py b/pygments/lexers/esoteric.py index 73ea4a4a..793c28be 100644 --- a/pygments/lexers/esoteric.py +++ b/pygments/lexers/esoteric.py @@ -5,15 +5,16 @@ Lexers for esoteric languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ from pygments.lexer import RegexLexer, include, words from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ - Number, Punctuation, Error, Whitespace + Number, Punctuation, Error -__all__ = ['BrainfuckLexer', 'BefungeLexer', 'BoogieLexer', 'RedcodeLexer', 'CAmkESLexer'] +__all__ = ['BrainfuckLexer', 'BefungeLexer', 'RedcodeLexer', 'CAmkESLexer', + 'CapDLLexer', 'AheuiLexer'] class BrainfuckLexer(RegexLexer): @@ -90,7 +91,7 @@ class CAmkESLexer(RegexLexer): filenames = ['*.camkes', '*.idl4'] tokens = { - 'root':[ + 'root': [ # C pre-processor directive (r'^\s*#.*\n', Comment.Preproc), @@ -99,21 +100,25 @@ class CAmkESLexer(RegexLexer): (r'/\*(.|\n)*?\*/', Comment), (r'//.*\n', Comment), - (r'[\[\(\){},\.;=\]]', Punctuation), + (r'[\[(){},.;\]]', Punctuation), + (r'[~!%^&*+=|?:<>/-]', Operator), (words(('assembly', 'attribute', 'component', 'composition', 'configuration', 'connection', 'connector', 'consumes', - 'control', 'dataport', 'Dataport', 'emits', 'event', - 'Event', 'from', 'group', 'hardware', 'has', 'interface', - 'Interface', 'maybe', 'procedure', 'Procedure', 'provides', - 'template', 'to', 'uses'), suffix=r'\b'), Keyword), + 'control', 'dataport', 'Dataport', 'Dataports', 'emits', + 'event', 'Event', 'Events', 'export', 'from', 'group', + 'hardware', 'has', 'interface', 'Interface', 'maybe', + 'procedure', 'Procedure', 'Procedures', 'provides', + 'template', 'thread', 'threads', 'to', 'uses', 'with'), + suffix=r'\b'), Keyword), (words(('bool', 'boolean', 'Buf', 'char', 'character', 'double', 'float', 'in', 'inout', 'int', 'int16_6', 'int32_t', 'int64_t', 'int8_t', 'integer', 'mutex', 'out', 'real', - 'refin', 'semaphore', 'signed', 'string', 'uint16_t', - 'uint32_t', 'uint64_t', 'uint8_t', 'uintptr_t', 'unsigned', - 'void'), suffix=r'\b'), Keyword.Type), + 'refin', 'semaphore', 'signed', 'string', 'struct', + 'uint16_t', 'uint32_t', 'uint64_t', 'uint8_t', 'uintptr_t', + 'unsigned', 'void'), + suffix=r'\b'), Keyword.Type), # Recognised attributes (r'[a-zA-Z_]\w*_(priority|domain|buffer)', Keyword.Reserved), @@ -131,6 +136,7 @@ class CAmkESLexer(RegexLexer): (r'-?[\d]+', Number), (r'-?[\d]+\.[\d]+', Number.Float), (r'"[^"]*"', String), + (r'[Tt]rue|[Ff]alse', Name.Builtin), # Identifiers (r'[a-zA-Z_]\w*', Name), @@ -138,6 +144,65 @@ class CAmkESLexer(RegexLexer): } +class CapDLLexer(RegexLexer): + """ + Basic lexer for + `CapDL <https://ssrg.nicta.com.au/publications/nictaabstracts/Kuz_KLW_10.abstract.pml>`_. + + The source of the primary tool that reads such specifications is available + at https://github.com/seL4/capdl/tree/master/capDL-tool. Note that this + lexer only supports a subset of the grammar. For example, identifiers can + shadow type names, but these instances are currently incorrectly + highlighted as types. Supporting this would need a stateful lexer that is + considered unnecessarily complex for now. + + .. versionadded:: 2.2 + """ + name = 'CapDL' + aliases = ['capdl'] + filenames = ['*.cdl'] + + tokens = { + 'root': [ + # C pre-processor directive + (r'^\s*#.*\n', Comment.Preproc), + + # Whitespace, comments + (r'\s+', Text), + (r'/\*(.|\n)*?\*/', Comment), + (r'(//|--).*\n', Comment), + + (r'[<>\[(){},:;=\]]', Punctuation), + (r'\.\.', Punctuation), + + (words(('arch', 'arm11', 'caps', 'child_of', 'ia32', 'irq', 'maps', + 'objects'), suffix=r'\b'), Keyword), + + (words(('aep', 'asid_pool', 'cnode', 'ep', 'frame', 'io_device', + 'io_ports', 'io_pt', 'notification', 'pd', 'pt', 'tcb', + 'ut', 'vcpu'), suffix=r'\b'), Keyword.Type), + + # Properties + (words(('asid', 'addr', 'badge', 'cached', 'dom', 'domainID', 'elf', + 'fault_ep', 'G', 'guard', 'guard_size', 'init', 'ip', + 'prio', 'sp', 'R', 'RG', 'RX', 'RW', 'RWG', 'RWX', 'W', + 'WG', 'WX', 'level', 'masked', 'master_reply', 'paddr', + 'ports', 'reply', 'uncached'), suffix=r'\b'), + Keyword.Reserved), + + # Literals + (r'0[xX][\da-fA-F]+', Number.Hex), + (r'\d+(\.\d+)?(k|M)?', Number), + (words(('bits',), suffix=r'\b'), Number), + (words(('cspace', 'vspace', 'reply_slot', 'caller_slot', + 'ipc_buffer_slot'), suffix=r'\b'), Number), + + # Identifiers + (r'[a-zA-Z_][-@\.\w]*', Name), + ], + } + + class RedcodeLexer(RegexLexer): """ A simple Redcode lexer based on ICWS'94. @@ -174,46 +239,39 @@ class RedcodeLexer(RegexLexer): } -class BoogieLexer(RegexLexer): +class AheuiLexer(RegexLexer): """ - For `Boogie <https://boogie.codeplex.com/>`_ source code. + Aheui_ Lexer. + + Aheui_ is esoteric language based on Korean alphabets. + + .. _Aheui:: http://aheui.github.io/ - .. versionadded:: 2.1 """ - name = 'Boogie' - aliases = ['boogie'] - filenames = ['*.bpl'] + + name = 'Aheui' + aliases = ['aheui'] + filenames = ['*.aheui'] tokens = { 'root': [ - # Whitespace and Comments - (r'\n', Whitespace), - (r'\s+', Whitespace), - (r'//[/!](.*?)\n', Comment.Doc), - (r'//(.*?)\n', Comment.Single), - (r'/\*', Comment.Multiline, 'comment'), - - (words(( - 'axiom', 'break', 'call', 'ensures', 'else', 'exists', 'function', - 'forall', 'if', 'invariant', 'modifies', 'procedure', 'requires', - 'then', 'var', 'while'), - suffix=r'\b'), Keyword), - (words(('const',), suffix=r'\b'), Keyword.Reserved), - - (words(('bool', 'int', 'ref'), suffix=r'\b'), Keyword.Type), - include('numbers'), - (r"(>=|<=|:=|!=|==>|&&|\|\||[+/\-=>*<\[\]])", Operator), - (r"([{}():;,.])", Punctuation), - # Identifier - (r'[a-zA-Z_]\w*', Name), - ], - 'comment': [ - (r'[^*/]+', Comment.Multiline), - (r'/\*', Comment.Multiline, '#push'), - (r'\*/', Comment.Multiline, '#pop'), - (r'[*/]', Comment.Multiline), - ], - 'numbers': [ - (r'[0-9]+', Number.Integer), + (u'[' + u'나-낳냐-냫너-넣녀-녛노-놓뇨-눟뉴-닇' + u'다-닿댜-댷더-덯뎌-뎧도-돟됴-둫듀-딓' + u'따-땋땨-떃떠-떻뗘-뗳또-똫뚀-뚷뜌-띟' + u'라-랗랴-럏러-렇려-렿로-롷료-뤃류-릫' + u'마-맣먀-먛머-멓며-몋모-뫃묘-뭏뮤-믷' + u'바-밯뱌-뱧버-벟벼-볗보-봏뵤-붛뷰-빃' + u'빠-빻뺘-뺳뻐-뻫뼈-뼣뽀-뽛뾰-뿧쀼-삏' + u'사-샇샤-샿서-섷셔-셯소-솧쇼-숳슈-싛' + u'싸-쌓쌰-썋써-쎃쎠-쎻쏘-쏳쑈-쑿쓔-씧' + u'자-잫쟈-쟣저-젛져-졓조-좋죠-줗쥬-즿' + u'차-챃챠-챻처-첳쳐-쳫초-촣쵸-춯츄-칗' + u'카-캏캬-컇커-컿켜-켷코-콯쿄-쿻큐-킣' + u'타-탛탸-턓터-텋텨-톃토-톻툐-퉇튜-틯' + u'파-팧퍄-퍟퍼-펗펴-폏포-퐇표-풓퓨-픻' + u'하-핳햐-햫허-헣혀-혛호-홓효-훟휴-힇' + u']', Operator), + ('.', Comment), ], } diff --git a/pygments/lexers/ezhil.py b/pygments/lexers/ezhil.py index a5468a0f..ce1cdb2d 100644 --- a/pygments/lexers/ezhil.py +++ b/pygments/lexers/ezhil.py @@ -4,8 +4,8 @@ ~~~~~~~~~~~~~~~~~~~~~ Pygments lexers for Ezhil language. - - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -16,6 +16,7 @@ from pygments.token import String, Number, Punctuation, Operator __all__ = ['EzhilLexer'] + class EzhilLexer(RegexLexer): """ Lexer for `Ezhil, a Tamil script-based programming language <http://ezhillang.org>`_ @@ -36,13 +37,13 @@ class EzhilLexer(RegexLexer): (r'#.*\n', Comment.Single), (r'[@+/*,^\-%]|[!<>=]=?|&&?|\|\|?', Operator), (u'இல்', Operator.Word), - (words(('assert', 'max', 'min', - 'நீளம்','சரம்_இடமாற்று','சரம்_கண்டுபிடி', - 'பட்டியல்','பின்இணை','வரிசைப்படுத்து', - 'எடு','தலைகீழ்','நீட்டிக்க','நுழைக்க','வை', - 'கோப்பை_திற','கோப்பை_எழுது','கோப்பை_மூடு', - 'pi','sin','cos','tan','sqrt','hypot','pow','exp','log','log10' - 'min','max','exit', + (words((u'assert', u'max', u'min', + u'நீளம்', u'சரம்_இடமாற்று', u'சரம்_கண்டுபிடி', + u'பட்டியல்', u'பின்இணை', u'வரிசைப்படுத்து', + u'எடு', u'தலைகீழ்', u'நீட்டிக்க', u'நுழைக்க', u'வை', + u'கோப்பை_திற', u'கோப்பை_எழுது', u'கோப்பை_மூடு', + u'pi', u'sin', u'cos', u'tan', u'sqrt', u'hypot', u'pow', + u'exp', u'log', u'log10', u'exit', ), suffix=r'\b'), Name.Builtin), (r'(True|False)\b', Keyword.Constant), (r'[^\S\n]+', Text), @@ -62,7 +63,7 @@ class EzhilLexer(RegexLexer): (r'(?u)\d+', Number.Integer), ] } - + def __init__(self, **options): super(EzhilLexer, self).__init__(**options) self.encoding = options.get('encoding', 'utf-8') diff --git a/pygments/lexers/factor.py b/pygments/lexers/factor.py index 6a39a1d4..09d85c27 100644 --- a/pygments/lexers/factor.py +++ b/pygments/lexers/factor.py @@ -5,7 +5,7 @@ Lexers for the Factor language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/fantom.py b/pygments/lexers/fantom.py index c20a3f38..3ea2177c 100644 --- a/pygments/lexers/fantom.py +++ b/pygments/lexers/fantom.py @@ -5,7 +5,7 @@ Lexer for the Fantom language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/felix.py b/pygments/lexers/felix.py index b7659769..8f0695b5 100644 --- a/pygments/lexers/felix.py +++ b/pygments/lexers/felix.py @@ -5,7 +5,7 @@ Lexer for the Felix language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -237,7 +237,7 @@ class FelixLexer(RegexLexer): ], 'strings': [ (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' - '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol), + '[hlL]?[E-GXc-giorsux%]', String.Interpol), (r'[^\\\'"%\n]+', String), # quotes, percents and backslashes must be parsed one at a time (r'[\'"\\]', String), diff --git a/pygments/lexers/forth.py b/pygments/lexers/forth.py new file mode 100644 index 00000000..a51f1b57 --- /dev/null +++ b/pygments/lexers/forth.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.forth + ~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, bygroups +from pygments.token import Error, Punctuation, Literal, Token, \ + Text, Comment, Operator, Keyword, Name, String, Number, Generic + + +__all__ = ['ForthLexer'] + + +class ForthLexer(RegexLexer): + """ + Lexer for Forth files. + + .. versionadded:: 2.2 + """ + name = 'Forth' + aliases = ['forth'] + filenames = ['*.frt', '*.fs'] + mimetypes = ['application/x-forth'] + + delimiter = r'\s' + delimiter_end = r'(?=[%s])' % delimiter + + valid_name_chars = r'[^%s]' % delimiter + valid_name = r"%s+%s" % (valid_name_chars, delimiter_end) + + flags = re.IGNORECASE | re.MULTILINE + + tokens = { + 'root': [ + (r'\s+', Text), + # All comment types + (r'\\.*?\n', Comment.Single), + (r'\([\s].*?\)', Comment.Single), + # defining words. The next word is a new command name + (r'(:|variable|constant|value|buffer:)(\s+)', + bygroups(Keyword.Namespace, Text), 'worddef'), + # strings are rather simple + (r'([.sc]")(\s+?)', bygroups(String, Text), 'stringdef'), + # keywords from the various wordsets + # *** Wordset BLOCK + (r'(blk|block|buffer|evaluate|flush|load|save-buffers|update|' + # *** Wordset BLOCK-EXT + r'empty-buffers|list|refill|scr|thru|' + # *** Wordset CORE + r'\#s|\*\/mod|\+loop|\/mod|0<|0=|1\+|1-|2!|' + r'2\*|2\/|2@|2drop|2dup|2over|2swap|>body|' + r'>in|>number|>r|\?dup|abort|abort\"|abs|' + r'accept|align|aligned|allot|and|base|begin|' + r'bl|c!|c,|c@|cell\+|cells|char|char\+|' + r'chars|constant|count|cr|create|decimal|' + r'depth|do|does>|drop|dup|else|emit|environment\?|' + r'evaluate|execute|exit|fill|find|fm\/mod|' + r'here|hold|i|if|immediate|invert|j|key|' + r'leave|literal|loop|lshift|m\*|max|min|' + r'mod|move|negate|or|over|postpone|quit|' + r'r>|r@|recurse|repeat|rot|rshift|s\"|s>d|' + r'sign|sm\/rem|source|space|spaces|state|swap|' + r'then|type|u\.|u\<|um\*|um\/mod|unloop|until|' + r'variable|while|word|xor|\[char\]|\[\'\]|' + r'@|!|\#|<\#|\#>|:|;|\+|-|\*|\/|,|<|>|\|1\+|1-|\.|' + # *** Wordset CORE-EXT + r'\.r|0<>|' + r'0>|2>r|2r>|2r@|:noname|\?do|again|c\"|' + r'case|compile,|endcase|endof|erase|false|' + r'hex|marker|nip|of|pad|parse|pick|refill|' + r'restore-input|roll|save-input|source-id|to|' + r'true|tuck|u\.r|u>|unused|value|within|' + r'\[compile\]|' + # *** Wordset CORE-EXT-obsolescent + r'\#tib|convert|expect|query|span|' + r'tib|' + # *** Wordset DOUBLE + r'2constant|2literal|2variable|d\+|d-|' + r'd\.|d\.r|d0<|d0=|d2\*|d2\/|d<|d=|d>s|' + r'dabs|dmax|dmin|dnegate|m\*\/|m\+|' + # *** Wordset DOUBLE-EXT + r'2rot|du<|' + # *** Wordset EXCEPTION + r'catch|throw|' + # *** Wordset EXCEPTION-EXT + r'abort|abort\"|' + # *** Wordset FACILITY + r'at-xy|key\?|page|' + # *** Wordset FACILITY-EXT + r'ekey|ekey>char|ekey\?|emit\?|ms|time&date|' + # *** Wordset FILE + r'BIN|CLOSE-FILE|CREATE-FILE|DELETE-FILE|FILE-POSITION|' + r'FILE-SIZE|INCLUDE-FILE|INCLUDED|OPEN-FILE|R\/O|' + r'R\/W|READ-FILE|READ-LINE|REPOSITION-FILE|RESIZE-FILE|' + r'S\"|SOURCE-ID|W/O|WRITE-FILE|WRITE-LINE|' + # *** Wordset FILE-EXT + r'FILE-STATUS|FLUSH-FILE|REFILL|RENAME-FILE|' + # *** Wordset FLOAT + r'>float|d>f|' + r'f!|f\*|f\+|f-|f\/|f0<|f0=|f<|f>d|f@|' + r'falign|faligned|fconstant|fdepth|fdrop|fdup|' + r'fliteral|float\+|floats|floor|fmax|fmin|' + r'fnegate|fover|frot|fround|fswap|fvariable|' + r'represent|' + # *** Wordset FLOAT-EXT + r'df!|df@|dfalign|dfaligned|dfloat\+|' + r'dfloats|f\*\*|f\.|fabs|facos|facosh|falog|' + r'fasin|fasinh|fatan|fatan2|fatanh|fcos|fcosh|' + r'fe\.|fexp|fexpm1|fln|flnp1|flog|fs\.|fsin|' + r'fsincos|fsinh|fsqrt|ftan|ftanh|f~|precision|' + r'set-precision|sf!|sf@|sfalign|sfaligned|sfloat\+|' + r'sfloats|' + # *** Wordset LOCAL + r'\(local\)|to|' + # *** Wordset LOCAL-EXT + r'locals\||' + # *** Wordset MEMORY + r'allocate|free|resize|' + # *** Wordset SEARCH + r'definitions|find|forth-wordlist|get-current|' + r'get-order|search-wordlist|set-current|set-order|' + r'wordlist|' + # *** Wordset SEARCH-EXT + r'also|forth|only|order|previous|' + # *** Wordset STRING + r'-trailing|\/string|blank|cmove|cmove>|compare|' + r'search|sliteral|' + # *** Wordset TOOLS + r'.s|dump|see|words|' + # *** Wordset TOOLS-EXT + r';code|' + r'ahead|assembler|bye|code|cs-pick|cs-roll|' + r'editor|state|\[else\]|\[if\]|\[then\]|' + # *** Wordset TOOLS-EXT-obsolescent + r'forget|' + # Forth 2012 + r'defer|defer@|defer!|action-of|begin-structure|field:|buffer:|' + r'parse-name|buffer:|traverse-wordlist|n>r|nr>|2value|fvalue|' + r'name>interpret|name>compile|name>string|' + r'cfield:|end-structure)'+delimiter, Keyword), + + # Numbers + (r'(\$[0-9A-F]+)', Number.Hex), + (r'(\#|%|&|\-|\+)?[0-9]+', Number.Integer), + (r'(\#|%|&|\-|\+)?[0-9.]+', Keyword.Type), + # amforth specific + (r'(@i|!i|@e|!e|pause|noop|turnkey|sleep|' + r'itype|icompare|sp@|sp!|rp@|rp!|up@|up!|' + r'>a|a>|a@|a!|a@+|a@-|>b|b>|b@|b!|b@+|b@-|' + r'find-name|1ms|' + r'sp0|rp0|\(evaluate\)|int-trap|int!)' + delimiter, + Name.Constant), + # a proposal + (r'(do-recognizer|r:fail|recognizer:|get-recognizers|' + r'set-recognizers|r:float|r>comp|r>int|r>post|' + r'r:name|r:word|r:dnum|r:num|recognizer|forth-recognizer|' + r'rec:num|rec:float|rec:word)' + delimiter, Name.Decorator), + # defining words. The next word is a new command name + (r'(Evalue|Rvalue|Uvalue|Edefer|Rdefer|Udefer)(\s+)', + bygroups(Keyword.Namespace, Text), 'worddef'), + + (valid_name, Name.Function), # Anything else is executed + + ], + 'worddef': [ + (r'\S+', Name.Class, '#pop'), + ], + 'stringdef': [ + (r'[^"]+', String, '#pop'), + ], + } diff --git a/pygments/lexers/fortran.py b/pygments/lexers/fortran.py index 4c22139d..1a611c9d 100644 --- a/pygments/lexers/fortran.py +++ b/pygments/lexers/fortran.py @@ -5,13 +5,13 @@ Lexers for Fortran languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re -from pygments.lexer import RegexLexer, bygroups, include, words, using +from pygments.lexer import RegexLexer, bygroups, include, words, using, default from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Generic @@ -156,8 +156,8 @@ class FortranLexer(RegexLexer): 'nums': [ (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer), - (r'[+-]?\d*\.\d+(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float), - (r'[+-]?\d+\.\d*(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float), + (r'[+-]?\d*\.\d+([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), + (r'[+-]?\d+\.\d*([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float), ], } @@ -191,16 +191,15 @@ class FortranFixedLexer(RegexLexer): (r'(.{5})', Name.Label, 'cont-char'), (r'.*\n', using(FortranLexer)), ], - 'cont-char': [ (' ', Text, 'code'), ('0', Comment, 'code'), - ('.', Generic.Strong, 'code') + ('.', Generic.Strong, 'code'), ], - 'code': [ (r'(.{66})(.*)(\n)', bygroups(_lex_fortran, Comment, Text), 'root'), (r'(.*)(\n)', bygroups(_lex_fortran, Text), 'root'), - (r'', Text, 'root')] + default('root'), + ] } diff --git a/pygments/lexers/foxpro.py b/pygments/lexers/foxpro.py index c7f368c7..7c0d2621 100644 --- a/pygments/lexers/foxpro.py +++ b/pygments/lexers/foxpro.py @@ -5,7 +5,7 @@ Simple lexer for Microsoft Visual FoxPro source code. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py index 13c72b1e..254df795 100644 --- a/pygments/lexers/functional.py +++ b/pygments/lexers/functional.py @@ -5,7 +5,7 @@ Just export lexer classes previously contained in this module. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/go.py b/pygments/lexers/go.py index 8bd6c7fb..cc2a6d63 100644 --- a/pygments/lexers/go.py +++ b/pygments/lexers/go.py @@ -5,7 +5,7 @@ Lexers for the Google Go language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/grammar_notation.py b/pygments/lexers/grammar_notation.py index 460914f4..076249d3 100644 --- a/pygments/lexers/grammar_notation.py +++ b/pygments/lexers/grammar_notation.py @@ -5,15 +5,17 @@ Lexers for grammer notations like BNF. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -from pygments.lexer import RegexLexer, bygroups, words -from pygments.token import Punctuation, Text, Comment, Operator, \ - Keyword, Name, Literal +import re -__all__ = ['BnfLexer', 'AbnfLexer'] +from pygments.lexer import RegexLexer, bygroups, include, this, using, words +from pygments.token import Comment, Keyword, Literal, Name, Number, \ + Operator, Punctuation, String, Text + +__all__ = ['BnfLexer', 'AbnfLexer', 'JsgfLexer'] class BnfLexer(RegexLexer): @@ -129,3 +131,83 @@ class AbnfLexer(RegexLexer): (r'.', Text), ], } + + +class JsgfLexer(RegexLexer): + """ + For `JSpeech Grammar Format <https://www.w3.org/TR/jsgf/>`_ + grammars. + + .. versionadded:: 2.2 + """ + name = 'JSGF' + aliases = ['jsgf'] + filenames = ['*.jsgf'] + mimetypes = ['application/jsgf', 'application/x-jsgf', 'text/jsgf'] + + flags = re.MULTILINE | re.UNICODE + + tokens = { + 'root': [ + include('comments'), + include('non-comments'), + ], + 'comments': [ + (r'/\*\*(?!/)', Comment.Multiline, 'documentation comment'), + (r'/\*[\w\W]*?\*/', Comment.Multiline), + (r'//.*', Comment.Single), + ], + 'non-comments': [ + ('\A#JSGF[^;]*', Comment.Preproc), + (r'\s+', Text), + (r';', Punctuation), + (r'[=|()\[\]*+]', Operator), + (r'/[^/]+/', Number.Float), + (r'"', String.Double, 'string'), + (r'\{', String.Other, 'tag'), + (words(('import', 'public'), suffix=r'\b'), Keyword.Reserved), + (r'grammar\b', Keyword.Reserved, 'grammar name'), + (r'(<)(NULL|VOID)(>)', + bygroups(Punctuation, Name.Builtin, Punctuation)), + (r'<', Punctuation, 'rulename'), + (r'\w+|[^\s;=|()\[\]*+/"{<\w]+', Text), + ], + 'string': [ + (r'"', String.Double, '#pop'), + (r'\\.', String.Escape), + (r'[^\\"]+', String.Double), + ], + 'tag': [ + (r'\}', String.Other, '#pop'), + (r'\\.', String.Escape), + (r'[^\\}]+', String.Other), + ], + 'grammar name': [ + (r';', Punctuation, '#pop'), + (r'\s+', Text), + (r'\.', Punctuation), + (r'[^;\s.]+', Name.Namespace), + ], + 'rulename': [ + (r'>', Punctuation, '#pop'), + (r'\*', Punctuation), + (r'\s+', Text), + (r'([^.>]+)(\s*)(\.)', bygroups(Name.Namespace, Text, Punctuation)), + (r'[^.>]+', Name.Constant), + ], + 'documentation comment': [ + (r'\*/', Comment.Multiline, '#pop'), + (r'(^\s*\*?\s*)(@(?:example|see)\s+)' + r'([\w\W]*?(?=(?:^\s*\*?\s*@|\*/)))', + bygroups(Comment.Multiline, Comment.Special, + using(this, state='example'))), + (r'(^\s*\*?\s*)(@\S*)', + bygroups(Comment.Multiline, Comment.Special)), + (r'[^*\n@]+|\w|\W', Comment.Multiline), + ], + 'example': [ + (r'\n\s*\*', Comment.Multiline), + include('non-comments'), + (r'.', Comment.Multiline), + ], + } diff --git a/pygments/lexers/graph.py b/pygments/lexers/graph.py index 8315898c..1a338246 100644 --- a/pygments/lexers/graph.py +++ b/pygments/lexers/graph.py @@ -5,7 +5,7 @@ Lexers for graph query languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/graphics.py b/pygments/lexers/graphics.py index b40e0286..c8af9f99 100644 --- a/pygments/lexers/graphics.py +++ b/pygments/lexers/graphics.py @@ -5,7 +5,7 @@ Lexers for computer graphics and plotting related languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/haskell.py b/pygments/lexers/haskell.py index 95e68a33..1a2f2217 100644 --- a/pygments/lexers/haskell.py +++ b/pygments/lexers/haskell.py @@ -5,7 +5,7 @@ Lexers for Haskell and related languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -39,7 +39,7 @@ class HaskellLexer(RegexLexer): flags = re.MULTILINE | re.UNICODE reserved = ('case', 'class', 'data', 'default', 'deriving', 'do', 'else', - 'if', 'in', 'infix[lr]?', 'instance', + 'family', 'if', 'in', 'infix[lr]?', 'instance', 'let', 'newtype', 'of', 'then', 'type', 'where', '_') ascii = ('NUL', 'SOH', '[SE]TX', 'EOT', 'ENQ', 'ACK', 'BEL', 'BS', 'HT', 'LF', 'VT', 'FF', 'CR', 'S[OI]', 'DLE', @@ -63,6 +63,9 @@ class HaskellLexer(RegexLexer): (r'^[_' + uni.Ll + r'][\w\']*', Name.Function), (r"'?[_" + uni.Ll + r"][\w']*", Name), (r"('')?[" + uni.Lu + r"][\w\']*", Keyword.Type), + (r"(')[" + uni.Lu + r"][\w\']*", Keyword.Type), + (r"(')\[[^\]]*\]", Keyword.Type), # tuples and lists get special treatment in GHC + (r"(')\([^)]*\)", Keyword.Type), # .. # Operators (r'\\(?![:!#$%&*+.\\/<=>?@^|~-]+)', Name.Function), # lambda operator (r'(<-|::|->|=>|=)(?![:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials @@ -321,7 +324,7 @@ class AgdaLexer(RegexLexer): 'module': [ (r'\{-', Comment.Multiline, 'comment'), (r'[a-zA-Z][\w.]*', Name, '#pop'), - (r'[^a-zA-Z]+', Text) + (r'[\W0-9_]+', Text) ], 'comment': HaskellLexer.tokens['comment'], 'character': HaskellLexer.tokens['character'], diff --git a/pygments/lexers/haxe.py b/pygments/lexers/haxe.py index e0e15c11..6f5c3599 100644 --- a/pygments/lexers/haxe.py +++ b/pygments/lexers/haxe.py @@ -5,7 +5,7 @@ Lexers for Haxe and related stuff. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/hdl.py b/pygments/lexers/hdl.py index 04cef14e..57fb7ac9 100644 --- a/pygments/lexers/hdl.py +++ b/pygments/lexers/hdl.py @@ -5,7 +5,7 @@ Lexers for hardware descriptor languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/hexdump.py b/pygments/lexers/hexdump.py index efe16fa7..cba49be7 100644 --- a/pygments/lexers/hexdump.py +++ b/pygments/lexers/hexdump.py @@ -5,12 +5,10 @@ Lexers for hexadecimal dumps. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -import re - from pygments.lexer import RegexLexer, bygroups, include from pygments.token import Text, Name, Number, String, Punctuation @@ -36,7 +34,7 @@ class HexdumpLexer(RegexLexer): * ``od -t x1z FILE`` * ``xxd FILE`` * ``DEBUG.EXE FILE.COM`` and entering ``d`` to the prompt. - + .. versionadded:: 2.1 """ name = 'Hexdump' @@ -48,12 +46,17 @@ class HexdumpLexer(RegexLexer): 'root': [ (r'\n', Text), include('offset'), - (r'('+hd+r'{2})(\-)('+hd+r'{2})', bygroups(Number.Hex, Punctuation, Number.Hex)), + (r'('+hd+r'{2})(\-)('+hd+r'{2})', + bygroups(Number.Hex, Punctuation, Number.Hex)), (hd+r'{2}', Number.Hex), - (r'(\s{2,3})(\>)(.{16})(\<)$', bygroups(Text, Punctuation, String, Punctuation), 'bracket-strings'), - (r'(\s{2,3})(\|)(.{16})(\|)$', bygroups(Text, Punctuation, String, Punctuation), 'piped-strings'), - (r'(\s{2,3})(\>)(.{1,15})(\<)$', bygroups(Text, Punctuation, String, Punctuation)), - (r'(\s{2,3})(\|)(.{1,15})(\|)$', bygroups(Text, Punctuation, String, Punctuation)), + (r'(\s{2,3})(\>)(.{16})(\<)$', + bygroups(Text, Punctuation, String, Punctuation), 'bracket-strings'), + (r'(\s{2,3})(\|)(.{16})(\|)$', + bygroups(Text, Punctuation, String, Punctuation), 'piped-strings'), + (r'(\s{2,3})(\>)(.{1,15})(\<)$', + bygroups(Text, Punctuation, String, Punctuation)), + (r'(\s{2,3})(\|)(.{1,15})(\|)$', + bygroups(Text, Punctuation, String, Punctuation)), (r'(\s{2,3})(.{1,15})$', bygroups(Text, String)), (r'(\s{2,3})(.{16}|.{20})$', bygroups(Text, String), 'nonpiped-strings'), (r'\s', Text), @@ -72,7 +75,8 @@ class HexdumpLexer(RegexLexer): (r'\n', Text), include('offset'), (hd+r'{2}', Number.Hex), - (r'(\s{2,3})(\|)(.{1,16})(\|)$', bygroups(Text, Punctuation, String, Punctuation)), + (r'(\s{2,3})(\|)(.{1,16})(\|)$', + bygroups(Text, Punctuation, String, Punctuation)), (r'\s', Text), (r'^\*', Punctuation), ], @@ -80,14 +84,16 @@ class HexdumpLexer(RegexLexer): (r'\n', Text), include('offset'), (hd+r'{2}', Number.Hex), - (r'(\s{2,3})(\>)(.{1,16})(\<)$', bygroups(Text, Punctuation, String, Punctuation)), + (r'(\s{2,3})(\>)(.{1,16})(\<)$', + bygroups(Text, Punctuation, String, Punctuation)), (r'\s', Text), (r'^\*', Punctuation), ], 'nonpiped-strings': [ (r'\n', Text), include('offset'), - (r'('+hd+r'{2})(\-)('+hd+r'{2})', bygroups(Number.Hex, Punctuation, Number.Hex)), + (r'('+hd+r'{2})(\-)('+hd+r'{2})', + bygroups(Number.Hex, Punctuation, Number.Hex)), (hd+r'{2}', Number.Hex), (r'(\s{19,})(.{1,20}?)$', bygroups(Text, String)), (r'(\s{2,3})(.{1,20})$', bygroups(Text, String)), diff --git a/pygments/lexers/html.py b/pygments/lexers/html.py index 7893952f..73f020fa 100644 --- a/pygments/lexers/html.py +++ b/pygments/lexers/html.py @@ -5,7 +5,7 @@ Lexers for HTML, XML and related markup. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -23,7 +23,7 @@ from pygments.lexers.css import CssLexer, _indentation, _starts_block from pygments.lexers.ruby import RubyLexer __all__ = ['HtmlLexer', 'DtdLexer', 'XmlLexer', 'XsltLexer', 'HamlLexer', - 'ScamlLexer', 'JadeLexer'] + 'ScamlLexer', 'PugLexer'] class HtmlLexer(RegexLexer): @@ -492,19 +492,19 @@ class ScamlLexer(ExtendedRegexLexer): } -class JadeLexer(ExtendedRegexLexer): +class PugLexer(ExtendedRegexLexer): """ - For Jade markup. - Jade is a variant of Scaml, see: + For Pug markup. + Pug is a variant of Scaml, see: http://scalate.fusesource.org/documentation/scaml-reference.html .. versionadded:: 1.4 """ - name = 'Jade' - aliases = ['jade'] - filenames = ['*.jade'] - mimetypes = ['text/x-jade'] + name = 'Pug' + aliases = ['pug', 'jade'] + filenames = ['*.pug', '*.jade'] + mimetypes = ['text/x-pug', 'text/x-jade'] flags = re.IGNORECASE _dot = r'.' @@ -599,3 +599,4 @@ class JadeLexer(ExtendedRegexLexer): (r'\n', Text, 'root'), ], } +JadeLexer = PugLexer # compat diff --git a/pygments/lexers/idl.py b/pygments/lexers/idl.py index d745bcfd..99078970 100644 --- a/pygments/lexers/idl.py +++ b/pygments/lexers/idl.py @@ -5,7 +5,7 @@ Lexers for IDL. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -258,12 +258,13 @@ class IDLLexer(RegexLexer): (r'\b(mod|lt|le|eq|ne|ge|gt|not|and|or|xor)\b', Operator), (r'"[^\"]*"', String.Double), (r"'[^\']*'", String.Single), - (r'\b[\+\-]?([0-9]*\.[0-9]+|[0-9]+\.[0-9]*)(D|E)?([\+\-]?[0-9]+)?\b', Number.Float), - (r'\b\'[\+\-]?[0-9A-F]+\'X(U?(S?|L{1,2})|B)\b', Number.Hex), - (r'\b\'[\+\-]?[0-7]+\'O(U?(S?|L{1,2})|B)\b', Number.Oct), - (r'\b[\+\-]?[0-9]+U?L{1,2}\b', Number.Integer.Long), - (r'\b[\+\-]?[0-9]+U?S?\b', Number.Integer), - (r'\b[\+\-]?[0-9]+B\b', Number), + (r'\b[+\-]?([0-9]*\.[0-9]+|[0-9]+\.[0-9]*)(D|E)?([+\-]?[0-9]+)?\b', + Number.Float), + (r'\b\'[+\-]?[0-9A-F]+\'X(U?(S?|L{1,2})|B)\b', Number.Hex), + (r'\b\'[+\-]?[0-7]+\'O(U?(S?|L{1,2})|B)\b', Number.Oct), + (r'\b[+\-]?[0-9]+U?L{1,2}\b', Number.Integer.Long), + (r'\b[+\-]?[0-9]+U?S?\b', Number.Integer), + (r'\b[+\-]?[0-9]+B\b', Number), (r'.', Text), ] } diff --git a/pygments/lexers/igor.py b/pygments/lexers/igor.py index b0eaf6aa..1a21fe87 100644 --- a/pygments/lexers/igor.py +++ b/pygments/lexers/igor.py @@ -5,7 +5,7 @@ Lexers for Igor Pro. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -40,7 +40,7 @@ class IgorLexer(RegexLexer): types = ( 'variable', 'string', 'constant', 'strconstant', 'NVAR', 'SVAR', 'WAVE', 'STRUCT', 'dfref', 'funcref', 'char', 'uchar', 'int16', 'uint16', 'int32', - 'uint32', 'float', 'double' + 'uint32', 'int64', 'uint64', 'float', 'double' ) keywords = ( 'override', 'ThreadSafe', 'MultiThread', 'static', 'Proc', @@ -48,213 +48,221 @@ class IgorLexer(RegexLexer): 'Structure', 'EndStructure', 'EndMacro', 'Menu', 'SubMenu' ) operations = ( - 'Abort', 'AddFIFOData', 'AddFIFOVectData', 'AddMovieAudio', - 'AddMovieFrame', 'APMath', 'Append', 'AppendImage', - 'AppendLayoutObject', 'AppendMatrixContour', 'AppendText', - 'AppendToGraph', 'AppendToLayout', 'AppendToTable', 'AppendXYZContour', - 'AutoPositionWindow', 'BackgroundInfo', 'Beep', 'BoundingBall', - 'BrowseURL', 'BuildMenu', 'Button', 'cd', 'Chart', 'CheckBox', - 'CheckDisplayed', 'ChooseColor', 'Close', 'CloseMovie', 'CloseProc', - 'ColorScale', 'ColorTab2Wave', 'Concatenate', 'ControlBar', - 'ControlInfo', 'ControlUpdate', 'ConvexHull', 'Convolve', 'CopyFile', - 'CopyFolder', 'CopyScales', 'Correlate', 'CreateAliasShortcut', 'Cross', - 'CtrlBackground', 'CtrlFIFO', 'CtrlNamedBackground', 'Cursor', - 'CurveFit', 'CustomControl', 'CWT', 'Debugger', 'DebuggerOptions', - 'DefaultFont', 'DefaultGuiControls', 'DefaultGuiFont', 'DefineGuide', - 'DelayUpdate', 'DeleteFile', 'DeleteFolder', 'DeletePoints', - 'Differentiate', 'dir', 'Display', 'DisplayHelpTopic', - 'DisplayProcedure', 'DoAlert', 'DoIgorMenu', 'DoUpdate', 'DoWindow', - 'DoXOPIdle', 'DrawAction', 'DrawArc', 'DrawBezier', 'DrawLine', - 'DrawOval', 'DrawPICT', 'DrawPoly', 'DrawRect', 'DrawRRect', 'DrawText', - 'DSPDetrend', 'DSPPeriodogram', 'Duplicate', 'DuplicateDataFolder', - 'DWT', 'EdgeStats', 'Edit', 'ErrorBars', 'Execute', 'ExecuteScriptText', - 'ExperimentModified', 'Extract', 'FastGaussTransform', 'FastOp', - 'FBinRead', 'FBinWrite', 'FFT', 'FIFO2Wave', 'FIFOStatus', 'FilterFIR', - 'FilterIIR', 'FindLevel', 'FindLevels', 'FindPeak', 'FindPointsInPoly', - 'FindRoots', 'FindSequence', 'FindValue', 'FPClustering', 'fprintf', - 'FReadLine', 'FSetPos', 'FStatus', 'FTPDelete', 'FTPDownload', - 'FTPUpload', 'FuncFit', 'FuncFitMD', 'GetAxis', 'GetFileFolderInfo', - 'GetLastUserMenuInfo', 'GetMarquee', 'GetSelection', 'GetWindow', - 'GraphNormal', 'GraphWaveDraw', 'GraphWaveEdit', 'Grep', 'GroupBox', - 'Hanning', 'HideIgorMenus', 'HideInfo', 'HideProcedures', 'HideTools', - 'HilbertTransform', 'Histogram', 'IFFT', 'ImageAnalyzeParticles', - 'ImageBlend', 'ImageBoundaryToMask', 'ImageEdgeDetection', - 'ImageFileInfo', 'ImageFilter', 'ImageFocus', 'ImageGenerateROIMask', - 'ImageHistModification', 'ImageHistogram', 'ImageInterpolate', - 'ImageLineProfile', 'ImageLoad', 'ImageMorphology', 'ImageRegistration', - 'ImageRemoveBackground', 'ImageRestore', 'ImageRotate', 'ImageSave', - 'ImageSeedFill', 'ImageSnake', 'ImageStats', 'ImageThreshold', - 'ImageTransform', 'ImageUnwrapPhase', 'ImageWindow', 'IndexSort', - 'InsertPoints', 'Integrate', 'IntegrateODE', 'Interp3DPath', - 'Interpolate3D', 'KillBackground', 'KillControl', 'KillDataFolder', - 'KillFIFO', 'KillFreeAxis', 'KillPath', 'KillPICTs', 'KillStrings', - 'KillVariables', 'KillWaves', 'KillWindow', 'KMeans', 'Label', 'Layout', - 'Legend', 'LinearFeedbackShiftRegister', 'ListBox', 'LoadData', - 'LoadPackagePreferences', 'LoadPICT', 'LoadWave', 'Loess', - 'LombPeriodogram', 'Make', 'MakeIndex', 'MarkPerfTestTime', - 'MatrixConvolve', 'MatrixCorr', 'MatrixEigenV', 'MatrixFilter', - 'MatrixGaussJ', 'MatrixInverse', 'MatrixLinearSolve', - 'MatrixLinearSolveTD', 'MatrixLLS', 'MatrixLUBkSub', 'MatrixLUD', - 'MatrixMultiply', 'MatrixOP', 'MatrixSchur', 'MatrixSolve', - 'MatrixSVBkSub', 'MatrixSVD', 'MatrixTranspose', 'MeasureStyledText', - 'Modify', 'ModifyContour', 'ModifyControl', 'ModifyControlList', - 'ModifyFreeAxis', 'ModifyGraph', 'ModifyImage', 'ModifyLayout', - 'ModifyPanel', 'ModifyTable', 'ModifyWaterfall', 'MoveDataFolder', - 'MoveFile', 'MoveFolder', 'MoveString', 'MoveSubwindow', 'MoveVariable', - 'MoveWave', 'MoveWindow', 'NeuralNetworkRun', 'NeuralNetworkTrain', - 'NewDataFolder', 'NewFIFO', 'NewFIFOChan', 'NewFreeAxis', 'NewImage', - 'NewLayout', 'NewMovie', 'NewNotebook', 'NewPanel', 'NewPath', - 'NewWaterfall', 'Note', 'Notebook', 'NotebookAction', 'Open', - 'OpenNotebook', 'Optimize', 'ParseOperationTemplate', 'PathInfo', - 'PauseForUser', 'PauseUpdate', 'PCA', 'PlayMovie', 'PlayMovieAction', - 'PlaySnd', 'PlaySound', 'PopupContextualMenu', 'PopupMenu', - 'Preferences', 'PrimeFactors', 'Print', 'printf', 'PrintGraphs', - 'PrintLayout', 'PrintNotebook', 'PrintSettings', 'PrintTable', - 'Project', 'PulseStats', 'PutScrapText', 'pwd', 'Quit', - 'RatioFromNumber', 'Redimension', 'Remove', 'RemoveContour', + 'Abort', 'AddFIFOData', 'AddFIFOVectData', 'AddMovieAudio', 'AddMovieFrame', + 'AdoptFiles', 'APMath', 'Append', 'AppendImage', 'AppendLayoutObject', + 'AppendMatrixContour', 'AppendText', 'AppendToGizmo', 'AppendToGraph', + 'AppendToLayout', 'AppendToTable', 'AppendXYZContour', 'AutoPositionWindow', + 'BackgroundInfo', 'Beep', 'BoundingBall', 'BoxSmooth', 'BrowseURL', 'BuildMenu', + 'Button', 'cd', 'Chart', 'CheckBox', 'CheckDisplayed', 'ChooseColor', 'Close', + 'CloseHelp', 'CloseMovie', 'CloseProc', 'ColorScale', 'ColorTab2Wave', + 'Concatenate', 'ControlBar', 'ControlInfo', 'ControlUpdate', + 'ConvertGlobalStringTextEncoding', 'ConvexHull', 'Convolve', 'CopyFile', + 'CopyFolder', 'CopyScales', 'Correlate', 'CreateAliasShortcut', 'CreateBrowser', + 'Cross', 'CtrlBackground', 'CtrlFIFO', 'CtrlNamedBackground', 'Cursor', + 'CurveFit', 'CustomControl', 'CWT', 'Debugger', 'DebuggerOptions', 'DefaultFont', + 'DefaultGuiControls', 'DefaultGuiFont', 'DefaultTextEncoding', 'DefineGuide', + 'DelayUpdate', 'DeleteAnnotations', 'DeleteFile', 'DeleteFolder', 'DeletePoints', + 'Differentiate', 'dir', 'Display', 'DisplayHelpTopic', 'DisplayProcedure', + 'DoAlert', 'DoIgorMenu', 'DoUpdate', 'DoWindow', 'DoXOPIdle', 'DPSS', + 'DrawAction', 'DrawArc', 'DrawBezier', 'DrawLine', 'DrawOval', 'DrawPICT', + 'DrawPoly', 'DrawRect', 'DrawRRect', 'DrawText', 'DrawUserShape', 'DSPDetrend', + 'DSPPeriodogram', 'Duplicate', 'DuplicateDataFolder', 'DWT', 'EdgeStats', 'Edit', + 'ErrorBars', 'EstimatePeakSizes', 'Execute', 'ExecuteScriptText', + 'ExperimentModified', 'ExportGizmo', 'Extract', 'FastGaussTransform', 'FastOp', + 'FBinRead', 'FBinWrite', 'FFT', 'FIFOStatus', 'FIFO2Wave', 'FilterFIR', + 'FilterIIR', 'FindAPeak', 'FindContour', 'FindDuplicates', 'FindLevel', + 'FindLevels', 'FindPeak', 'FindPointsInPoly', 'FindRoots', 'FindSequence', + 'FindValue', 'FPClustering', 'fprintf', 'FReadLine', 'FSetPos', 'FStatus', + 'FTPCreateDirectory', 'FTPDelete', 'FTPDownload', 'FTPUpload', 'FuncFit', + 'FuncFitMD', 'GBLoadWave', 'GetAxis', 'GetCamera', 'GetFileFolderInfo', + 'GetGizmo', 'GetLastUserMenuInfo', 'GetMarquee', 'GetMouse', 'GetSelection', + 'GetWindow', 'GPIBReadBinaryWave2', 'GPIBReadBinary2', 'GPIBReadWave2', + 'GPIBRead2', 'GPIBWriteBinaryWave2', 'GPIBWriteBinary2', 'GPIBWriteWave2', + 'GPIBWrite2', 'GPIB2', 'GraphNormal', 'GraphWaveDraw', 'GraphWaveEdit', 'Grep', + 'GroupBox', 'Hanning', 'HDF5CloseFile', 'HDF5CloseGroup', 'HDF5ConvertColors', + 'HDF5CreateFile', 'HDF5CreateGroup', 'HDF5CreateLink', 'HDF5Dump', + 'HDF5DumpErrors', 'HDF5DumpState', 'HDF5ListAttributes', 'HDF5ListGroup', + 'HDF5LoadData', 'HDF5LoadGroup', 'HDF5LoadImage', 'HDF5OpenFile', 'HDF5OpenGroup', + 'HDF5SaveData', 'HDF5SaveGroup', 'HDF5SaveImage', 'HDF5TestOperation', + 'HDF5UnlinkObject', 'HideIgorMenus', 'HideInfo', 'HideProcedures', 'HideTools', + 'HilbertTransform', 'Histogram', 'ICA', 'IFFT', 'ImageAnalyzeParticles', + 'ImageBlend', 'ImageBoundaryToMask', 'ImageEdgeDetection', 'ImageFileInfo', + 'ImageFilter', 'ImageFocus', 'ImageFromXYZ', 'ImageGenerateROIMask', 'ImageGLCM', + 'ImageHistModification', 'ImageHistogram', 'ImageInterpolate', 'ImageLineProfile', + 'ImageLoad', 'ImageMorphology', 'ImageRegistration', 'ImageRemoveBackground', + 'ImageRestore', 'ImageRotate', 'ImageSave', 'ImageSeedFill', 'ImageSkeleton3d', + 'ImageSnake', 'ImageStats', 'ImageThreshold', 'ImageTransform', + 'ImageUnwrapPhase', 'ImageWindow', 'IndexSort', 'InsertPoints', 'Integrate', + 'IntegrateODE', 'Integrate2D', 'Interpolate2', 'Interpolate3D', 'Interp3DPath', + 'JCAMPLoadWave', 'JointHistogram', 'KillBackground', 'KillControl', + 'KillDataFolder', 'KillFIFO', 'KillFreeAxis', 'KillPath', 'KillPICTs', + 'KillStrings', 'KillVariables', 'KillWaves', 'KillWindow', 'KMeans', 'Label', + 'Layout', 'LayoutPageAction', 'LayoutSlideShow', 'Legend', + 'LinearFeedbackShiftRegister', 'ListBox', 'LoadData', 'LoadPackagePreferences', + 'LoadPICT', 'LoadWave', 'Loess', 'LombPeriodogram', 'Make', 'MakeIndex', + 'MarkPerfTestTime', 'MatrixConvolve', 'MatrixCorr', 'MatrixEigenV', + 'MatrixFilter', 'MatrixGaussJ', 'MatrixGLM', 'MatrixInverse', 'MatrixLinearSolve', + 'MatrixLinearSolveTD', 'MatrixLLS', 'MatrixLUBkSub', 'MatrixLUD', 'MatrixLUDTD', + 'MatrixMultiply', 'MatrixOP', 'MatrixSchur', 'MatrixSolve', 'MatrixSVBkSub', + 'MatrixSVD', 'MatrixTranspose', 'MeasureStyledText', 'MLLoadWave', 'Modify', + 'ModifyBrowser', 'ModifyCamera', 'ModifyContour', 'ModifyControl', + 'ModifyControlList', 'ModifyFreeAxis', 'ModifyGizmo', 'ModifyGraph', + 'ModifyImage', 'ModifyLayout', 'ModifyPanel', 'ModifyTable', 'ModifyWaterfall', + 'MoveDataFolder', 'MoveFile', 'MoveFolder', 'MoveString', 'MoveSubwindow', + 'MoveVariable', 'MoveWave', 'MoveWindow', 'MultiTaperPSD', + 'MultiThreadingControl', 'NeuralNetworkRun', 'NeuralNetworkTrain', 'NewCamera', + 'NewDataFolder', 'NewFIFO', 'NewFIFOChan', 'NewFreeAxis', 'NewGizmo', 'NewImage', + 'NewLayout', 'NewMovie', 'NewNotebook', 'NewPanel', 'NewPath', 'NewWaterfall', + 'NI4882', 'Note', 'Notebook', 'NotebookAction', 'Open', 'OpenHelp', + 'OpenNotebook', 'Optimize', 'ParseOperationTemplate', 'PathInfo', 'PauseForUser', + 'PauseUpdate', 'PCA', 'PlayMovie', 'PlayMovieAction', 'PlaySound', + 'PopupContextualMenu', 'PopupMenu', 'Preferences', 'PrimeFactors', 'Print', + 'printf', 'PrintGraphs', 'PrintLayout', 'PrintNotebook', 'PrintSettings', + 'PrintTable', 'Project', 'PulseStats', 'PutScrapText', 'pwd', 'Quit', + 'RatioFromNumber', 'Redimension', 'Remove', 'RemoveContour', 'RemoveFromGizmo', 'RemoveFromGraph', 'RemoveFromLayout', 'RemoveFromTable', 'RemoveImage', - 'RemoveLayoutObjects', 'RemovePath', 'Rename', 'RenameDataFolder', - 'RenamePath', 'RenamePICT', 'RenameWindow', 'ReorderImages', - 'ReorderTraces', 'ReplaceText', 'ReplaceWave', 'Resample', - 'ResumeUpdate', 'Reverse', 'Rotate', 'Save', 'SaveData', - 'SaveExperiment', 'SaveGraphCopy', 'SaveNotebook', - 'SavePackagePreferences', 'SavePICT', 'SaveTableCopy', - 'SetActiveSubwindow', 'SetAxis', 'SetBackground', 'SetDashPattern', - 'SetDataFolder', 'SetDimLabel', 'SetDrawEnv', 'SetDrawLayer', - 'SetFileFolderInfo', 'SetFormula', 'SetIgorHook', 'SetIgorMenuMode', - 'SetIgorOption', 'SetMarquee', 'SetProcessSleep', 'SetRandomSeed', - 'SetScale', 'SetVariable', 'SetWaveLock', 'SetWindow', 'ShowIgorMenus', - 'ShowInfo', 'ShowTools', 'Silent', 'Sleep', 'Slider', 'Smooth', - 'SmoothCustom', 'Sort', 'SoundInRecord', 'SoundInSet', - 'SoundInStartChart', 'SoundInStatus', 'SoundInStopChart', - 'SphericalInterpolate', 'SphericalTriangulate', 'SplitString', - 'sprintf', 'sscanf', 'Stack', 'StackWindows', + 'RemoveLayoutObjects', 'RemovePath', 'Rename', 'RenameDataFolder', 'RenamePath', + 'RenamePICT', 'RenameWindow', 'ReorderImages', 'ReorderTraces', 'ReplaceText', + 'ReplaceWave', 'Resample', 'ResumeUpdate', 'Reverse', 'Rotate', 'Save', + 'SaveData', 'SaveExperiment', 'SaveGraphCopy', 'SaveNotebook', + 'SavePackagePreferences', 'SavePICT', 'SaveTableCopy', 'SetActiveSubwindow', + 'SetAxis', 'SetBackground', 'SetDashPattern', 'SetDataFolder', 'SetDimLabel', + 'SetDrawEnv', 'SetDrawLayer', 'SetFileFolderInfo', 'SetFormula', 'SetIgorHook', + 'SetIgorMenuMode', 'SetIgorOption', 'SetMarquee', 'SetProcessSleep', + 'SetRandomSeed', 'SetScale', 'SetVariable', 'SetWaveLock', 'SetWaveTextEncoding', + 'SetWindow', 'ShowIgorMenus', 'ShowInfo', 'ShowTools', 'Silent', 'Sleep', + 'Slider', 'Smooth', 'SmoothCustom', 'Sort', 'SortColumns', 'SoundInRecord', + 'SoundInSet', 'SoundInStartChart', 'SoundInStatus', 'SoundInStopChart', + 'SoundLoadWave', 'SoundSaveWave', 'SphericalInterpolate', 'SphericalTriangulate', + 'SplitString', 'SplitWave', 'sprintf', 'sscanf', 'Stack', 'StackWindows', 'StatsAngularDistanceTest', 'StatsANOVA1Test', 'StatsANOVA2NRTest', 'StatsANOVA2RMTest', 'StatsANOVA2Test', 'StatsChiTest', - 'StatsCircularCorrelationTest', 'StatsCircularMeans', - 'StatsCircularMoments', 'StatsCircularTwoSampleTest', - 'StatsCochranTest', 'StatsContingencyTable', 'StatsDIPTest', - 'StatsDunnettTest', 'StatsFriedmanTest', 'StatsFTest', - 'StatsHodgesAjneTest', 'StatsJBTest', 'StatsKendallTauTest', + 'StatsCircularCorrelationTest', 'StatsCircularMeans', 'StatsCircularMoments', + 'StatsCircularTwoSampleTest', 'StatsCochranTest', 'StatsContingencyTable', + 'StatsDIPTest', 'StatsDunnettTest', 'StatsFriedmanTest', 'StatsFTest', + 'StatsHodgesAjneTest', 'StatsJBTest', 'StatsKDE', 'StatsKendallTauTest', 'StatsKSTest', 'StatsKWTest', 'StatsLinearCorrelationTest', - 'StatsLinearRegression', 'StatsMultiCorrelationTest', - 'StatsNPMCTest', 'StatsNPNominalSRTest', 'StatsQuantiles', - 'StatsRankCorrelationTest', 'StatsResample', 'StatsSample', - 'StatsScheffeTest', 'StatsSignTest', 'StatsSRTest', 'StatsTTest', - 'StatsTukeyTest', 'StatsVariancesTest', 'StatsWatsonUSquaredTest', - 'StatsWatsonWilliamsTest', 'StatsWheelerWatsonTest', - 'StatsWilcoxonRankTest', 'StatsWRCorrelationTest', 'String', - 'StructGet', 'StructPut', 'TabControl', 'Tag', 'TextBox', 'Tile', - 'TileWindows', 'TitleBox', 'ToCommandLine', 'ToolsGrid', - 'Triangulate3d', 'Unwrap', 'ValDisplay', 'Variable', 'WaveMeanStdv', - 'WaveStats', 'WaveTransform', 'wfprintf', 'WignerTransform', - 'WindowFunction', + 'StatsLinearRegression', 'StatsMultiCorrelationTest', 'StatsNPMCTest', + 'StatsNPNominalSRTest', 'StatsQuantiles', 'StatsRankCorrelationTest', + 'StatsResample', 'StatsSample', 'StatsScheffeTest', 'StatsShapiroWilkTest', + 'StatsSignTest', 'StatsSRTest', 'StatsTTest', 'StatsTukeyTest', + 'StatsVariancesTest', 'StatsWatsonUSquaredTest', 'StatsWatsonWilliamsTest', + 'StatsWheelerWatsonTest', 'StatsWilcoxonRankTest', 'StatsWRCorrelationTest', + 'String', 'StructGet', 'StructPut', 'SumDimension', 'SumSeries', 'TabControl', + 'Tag', 'TextBox', 'ThreadGroupPutDF', 'ThreadStart', 'Tile', 'TileWindows', + 'TitleBox', 'ToCommandLine', 'ToolsGrid', 'Triangulate3d', 'Unwrap', 'URLRequest', + 'ValDisplay', 'Variable', 'VDTClosePort2', 'VDTGetPortList2', 'VDTGetStatus2', + 'VDTOpenPort2', 'VDTOperationsPort2', 'VDTReadBinaryWave2', 'VDTReadBinary2', + 'VDTReadHexWave2', 'VDTReadHex2', 'VDTReadWave2', 'VDTRead2', 'VDTTerminalPort2', + 'VDTWriteBinaryWave2', 'VDTWriteBinary2', 'VDTWriteHexWave2', 'VDTWriteHex2', + 'VDTWriteWave2', 'VDTWrite2', 'VDT2', 'WaveMeanStdv', 'WaveStats', + 'WaveTransform', 'wfprintf', 'WignerTransform', 'WindowFunction', 'XLLoadWave' ) functions = ( - 'abs', 'acos', 'acosh', 'AiryA', 'AiryAD', 'AiryB', 'AiryBD', 'alog', - 'area', 'areaXY', 'asin', 'asinh', 'atan', 'atan2', 'atanh', - 'AxisValFromPixel', 'Besseli', 'Besselj', 'Besselk', 'Bessely', 'bessi', - 'bessj', 'bessk', 'bessy', 'beta', 'betai', 'BinarySearch', + 'abs', 'acos', 'acosh', 'AddListItem', 'AiryA', 'AiryAD', 'AiryB', 'AiryBD', + 'alog', 'AnnotationInfo', 'AnnotationList', 'area', 'areaXY', 'asin', 'asinh', + 'atan', 'atanh', 'atan2', 'AxisInfo', 'AxisList', 'AxisValFromPixel', 'Besseli', + 'Besselj', 'Besselk', 'Bessely', 'beta', 'betai', 'BinarySearch', 'BinarySearchInterp', 'binomial', 'binomialln', 'binomialNoise', 'cabs', - 'CaptureHistoryStart', 'ceil', 'cequal', 'char2num', 'chebyshev', - 'chebyshevU', 'CheckName', 'cmplx', 'cmpstr', 'conj', 'ContourZ', 'cos', - 'cosh', 'cot', 'CountObjects', 'CountObjectsDFR', 'cpowi', - 'CreationDate', 'csc', 'DataFolderExists', 'DataFolderRefsEqual', - 'DataFolderRefStatus', 'date2secs', 'datetime', 'DateToJulian', - 'Dawson', 'DDEExecute', 'DDEInitiate', 'DDEPokeString', 'DDEPokeWave', - 'DDERequestWave', 'DDEStatus', 'DDETerminate', 'defined', 'deltax', 'digamma', - 'DimDelta', 'DimOffset', 'DimSize', 'ei', 'enoise', 'equalWaves', 'erf', - 'erfc', 'exists', 'exp', 'expInt', 'expNoise', 'factorial', 'fakedata', - 'faverage', 'faverageXY', 'FindDimLabel', 'FindListItem', 'floor', + 'CaptureHistory', 'CaptureHistoryStart', 'ceil', 'cequal', 'char2num', + 'chebyshev', 'chebyshevU', 'CheckName', 'ChildWindowList', 'CleanupName', 'cmplx', + 'cmpstr', 'conj', 'ContourInfo', 'ContourNameList', 'ContourNameToWaveRef', + 'ContourZ', 'ControlNameList', 'ConvertTextEncoding', 'cos', 'cosh', + 'cosIntegral', 'cot', 'coth', 'CountObjects', 'CountObjectsDFR', 'cpowi', + 'CreationDate', 'csc', 'csch', 'CsrInfo', 'CsrWave', 'CsrWaveRef', 'CsrXWave', + 'CsrXWaveRef', 'CTabList', 'DataFolderDir', 'DataFolderExists', + 'DataFolderRefsEqual', 'DataFolderRefStatus', 'date', 'datetime', 'DateToJulian', + 'date2secs', 'Dawson', 'DDERequestString', 'defined', 'deltax', 'digamma', + 'dilogarithm', 'DimDelta', 'DimOffset', 'DimSize', 'ei', 'enoise', 'equalWaves', + 'erf', 'erfc', 'erfcw', 'exists', 'exp', 'ExpConvExp', 'ExpConvExpFit', + 'ExpConvExpFitBL', 'ExpConvExpFit1Shape', 'ExpConvExpFit1ShapeBL', 'ExpGauss', + 'ExpGaussFit', 'ExpGaussFitBL', 'ExpGaussFit1Shape', 'ExpGaussFit1ShapeBL', + 'expInt', 'expIntegralE1', 'expNoise', 'factorial', 'fakedata', 'faverage', + 'faverageXY', 'FetchURL', 'FindDimLabel', 'FindListItem', 'floor', 'FontList', 'FontSizeHeight', 'FontSizeStringWidth', 'FresnelCos', 'FresnelSin', - 'gamma', 'gammaInc', 'gammaNoise', 'gammln', 'gammp', 'gammq', 'Gauss', - 'Gauss1D', 'Gauss2D', 'gcd', 'GetDefaultFontSize', - 'GetDefaultFontStyle', 'GetKeyState', 'GetRTError', 'gnoise', - 'GrepString', 'hcsr', 'hermite', 'hermiteGauss', 'HyperG0F1', - 'HyperG1F1', 'HyperG2F1', 'HyperGNoise', 'HyperGPFQ', 'IgorVersion', - 'ilim', 'imag', 'Inf', 'Integrate1D', 'interp', 'Interp2D', 'Interp3D', - 'inverseERF', 'inverseERFC', 'ItemsInList', 'jlim', 'Laguerre', - 'LaguerreA', 'LaguerreGauss', 'leftx', 'LegendreA', 'limit', 'ln', - 'log', 'logNormalNoise', 'lorentzianNoise', 'magsqr', 'MandelbrotPoint', - 'MarcumQ', 'MatrixDet', 'MatrixDot', 'MatrixRank', 'MatrixTrace', 'max', - 'mean', 'min', 'mod', 'ModDate', 'NaN', 'norm', 'NumberByKey', - 'numpnts', 'numtype', 'NumVarOrDefault', 'NVAR_Exists', 'p2rect', - 'ParamIsDefault', 'pcsr', 'Pi', 'PixelFromAxisVal', 'pnt2x', - 'poissonNoise', 'poly', 'poly2D', 'PolygonArea', 'qcsr', 'r2polar', - 'real', 'rightx', 'round', 'sawtooth', 'ScreenResolution', 'sec', - 'SelectNumber', 'sign', 'sin', 'sinc', 'sinh', 'SphericalBessJ', - 'SphericalBessJD', 'SphericalBessY', 'SphericalBessYD', - 'SphericalHarmonics', 'sqrt', 'StartMSTimer', 'StatsBetaCDF', - 'StatsBetaPDF', 'StatsBinomialCDF', 'StatsBinomialPDF', - 'StatsCauchyCDF', 'StatsCauchyPDF', 'StatsChiCDF', 'StatsChiPDF', - 'StatsCMSSDCDF', 'StatsCorrelation', 'StatsDExpCDF', 'StatsDExpPDF', - 'StatsErlangCDF', 'StatsErlangPDF', 'StatsErrorPDF', 'StatsEValueCDF', - 'StatsEValuePDF', 'StatsExpCDF', 'StatsExpPDF', 'StatsFCDF', - 'StatsFPDF', 'StatsFriedmanCDF', 'StatsGammaCDF', 'StatsGammaPDF', - 'StatsGeometricCDF', 'StatsGeometricPDF', 'StatsHyperGCDF', - 'StatsHyperGPDF', 'StatsInvBetaCDF', 'StatsInvBinomialCDF', - 'StatsInvCauchyCDF', 'StatsInvChiCDF', 'StatsInvCMSSDCDF', - 'StatsInvDExpCDF', 'StatsInvEValueCDF', 'StatsInvExpCDF', - 'StatsInvFCDF', 'StatsInvFriedmanCDF', 'StatsInvGammaCDF', - 'StatsInvGeometricCDF', 'StatsInvKuiperCDF', 'StatsInvLogisticCDF', - 'StatsInvLogNormalCDF', 'StatsInvMaxwellCDF', 'StatsInvMooreCDF', - 'StatsInvNBinomialCDF', 'StatsInvNCChiCDF', 'StatsInvNCFCDF', - 'StatsInvNormalCDF', 'StatsInvParetoCDF', 'StatsInvPoissonCDF', - 'StatsInvPowerCDF', 'StatsInvQCDF', 'StatsInvQpCDF', + 'FuncRefInfo', 'FunctionInfo', 'FunctionList', 'FunctionPath', 'gamma', + 'gammaEuler', 'gammaInc', 'gammaNoise', 'gammln', 'gammp', 'gammq', 'Gauss', + 'GaussFit', 'GaussFitBL', 'GaussFit1Width', 'GaussFit1WidthBL', 'Gauss1D', + 'Gauss2D', 'gcd', 'GetBrowserLine', 'GetBrowserSelection', 'GetDataFolder', + 'GetDataFolderDFR', 'GetDefaultFont', 'GetDefaultFontSize', 'GetDefaultFontStyle', + 'GetDimLabel', 'GetEnvironmentVariable', 'GetErrMessage', 'GetFormula', + 'GetIndependentModuleName', 'GetIndexedObjName', 'GetIndexedObjNameDFR', + 'GetKeyState', 'GetRTErrMessage', 'GetRTError', 'GetRTLocation', 'GetRTLocInfo', + 'GetRTStackInfo', 'GetScrapText', 'GetUserData', 'GetWavesDataFolder', + 'GetWavesDataFolderDFR', 'GizmoInfo', 'GizmoScale', 'gnoise', 'GrepList', + 'GrepString', 'GuideInfo', 'GuideNameList', 'Hash', 'hcsr', 'HDF5AttributeInfo', + 'HDF5DatasetInfo', 'HDF5LibraryInfo', 'HDF5TypeInfo', 'hermite', 'hermiteGauss', + 'HyperGNoise', 'HyperGPFQ', 'HyperG0F1', 'HyperG1F1', 'HyperG2F1', 'IgorInfo', + 'IgorVersion', 'imag', 'ImageInfo', 'ImageNameList', 'ImageNameToWaveRef', + 'IndependentModuleList', 'IndexedDir', 'IndexedFile', 'Inf', 'Integrate1D', + 'interp', 'Interp2D', 'Interp3D', 'inverseERF', 'inverseERFC', 'ItemsInList', + 'JacobiCn', 'JacobiSn', 'JulianToDate', 'Laguerre', 'LaguerreA', 'LaguerreGauss', + 'LambertW', 'LayoutInfo', 'leftx', 'LegendreA', 'limit', 'ListMatch', + 'ListToTextWave', 'ListToWaveRefWave', 'ln', 'log', 'logNormalNoise', + 'LorentzianFit', 'LorentzianFitBL', 'LorentzianFit1Width', + 'LorentzianFit1WidthBL', 'lorentzianNoise', 'LowerStr', 'MacroList', 'magsqr', + 'MandelbrotPoint', 'MarcumQ', 'MatrixCondition', 'MatrixDet', 'MatrixDot', + 'MatrixRank', 'MatrixTrace', 'max', 'mean', 'median', 'min', 'mod', 'ModDate', + 'MPFXEMGPeak', 'MPFXExpConvExpPeak', 'MPFXGaussPeak', 'MPFXLorenzianPeak', + 'MPFXVoigtPeak', 'NameOfWave', 'NaN', 'NewFreeDataFolder', 'NewFreeWave', 'norm', + 'NormalizeUnicode', 'note', 'NumberByKey', 'numpnts', 'numtype', + 'NumVarOrDefault', 'num2char', 'num2istr', 'num2str', 'NVAR_Exists', + 'OperationList', 'PadString', 'PanelResolution', 'ParamIsDefault', + 'ParseFilePath', 'PathList', 'pcsr', 'Pi', 'PICTInfo', 'PICTList', + 'PixelFromAxisVal', 'pnt2x', 'poissonNoise', 'poly', 'PolygonArea', 'poly2D', + 'PossiblyQuoteName', 'ProcedureText', 'p2rect', 'qcsr', 'real', 'RemoveByKey', + 'RemoveEnding', 'RemoveFromList', 'RemoveListItem', 'ReplaceNumberByKey', + 'ReplaceString', 'ReplaceStringByKey', 'rightx', 'round', 'r2polar', 'sawtooth', + 'scaleToIndex', 'ScreenResolution', 'sec', 'sech', 'Secs2Date', 'Secs2Time', + 'SelectNumber', 'SelectString', 'SetEnvironmentVariable', 'sign', 'sin', 'sinc', + 'sinh', 'sinIntegral', 'SortList', 'SpecialCharacterInfo', 'SpecialCharacterList', + 'SpecialDirPath', 'SphericalBessJ', 'SphericalBessJD', 'SphericalBessY', + 'SphericalBessYD', 'SphericalHarmonics', 'sqrt', 'StartMSTimer', 'StatsBetaCDF', + 'StatsBetaPDF', 'StatsBinomialCDF', 'StatsBinomialPDF', 'StatsCauchyCDF', + 'StatsCauchyPDF', 'StatsChiCDF', 'StatsChiPDF', 'StatsCMSSDCDF', + 'StatsCorrelation', 'StatsDExpCDF', 'StatsDExpPDF', 'StatsErlangCDF', + 'StatsErlangPDF', 'StatsErrorPDF', 'StatsEValueCDF', 'StatsEValuePDF', + 'StatsExpCDF', 'StatsExpPDF', 'StatsFCDF', 'StatsFPDF', 'StatsFriedmanCDF', + 'StatsGammaCDF', 'StatsGammaPDF', 'StatsGeometricCDF', 'StatsGeometricPDF', + 'StatsGEVCDF', 'StatsGEVPDF', 'StatsHyperGCDF', 'StatsHyperGPDF', + 'StatsInvBetaCDF', 'StatsInvBinomialCDF', 'StatsInvCauchyCDF', 'StatsInvChiCDF', + 'StatsInvCMSSDCDF', 'StatsInvDExpCDF', 'StatsInvEValueCDF', 'StatsInvExpCDF', + 'StatsInvFCDF', 'StatsInvFriedmanCDF', 'StatsInvGammaCDF', 'StatsInvGeometricCDF', + 'StatsInvKuiperCDF', 'StatsInvLogisticCDF', 'StatsInvLogNormalCDF', + 'StatsInvMaxwellCDF', 'StatsInvMooreCDF', 'StatsInvNBinomialCDF', + 'StatsInvNCChiCDF', 'StatsInvNCFCDF', 'StatsInvNormalCDF', 'StatsInvParetoCDF', + 'StatsInvPoissonCDF', 'StatsInvPowerCDF', 'StatsInvQCDF', 'StatsInvQpCDF', 'StatsInvRayleighCDF', 'StatsInvRectangularCDF', 'StatsInvSpearmanCDF', 'StatsInvStudentCDF', 'StatsInvTopDownCDF', 'StatsInvTriangularCDF', 'StatsInvUsquaredCDF', 'StatsInvVonMisesCDF', 'StatsInvWeibullCDF', - 'StatsKuiperCDF', 'StatsLogisticCDF', 'StatsLogisticPDF', - 'StatsLogNormalCDF', 'StatsLogNormalPDF', 'StatsMaxwellCDF', - 'StatsMaxwellPDF', 'StatsMedian', 'StatsMooreCDF', 'StatsNBinomialCDF', - 'StatsNBinomialPDF', 'StatsNCChiCDF', 'StatsNCChiPDF', 'StatsNCFCDF', - 'StatsNCFPDF', 'StatsNCTCDF', 'StatsNCTPDF', 'StatsNormalCDF', - 'StatsNormalPDF', 'StatsParetoCDF', 'StatsParetoPDF', 'StatsPermute', - 'StatsPoissonCDF', 'StatsPoissonPDF', 'StatsPowerCDF', - 'StatsPowerNoise', 'StatsPowerPDF', 'StatsQCDF', 'StatsQpCDF', - 'StatsRayleighCDF', 'StatsRayleighPDF', 'StatsRectangularCDF', - 'StatsRectangularPDF', 'StatsRunsCDF', 'StatsSpearmanRhoCDF', - 'StatsStudentCDF', 'StatsStudentPDF', 'StatsTopDownCDF', + 'StatsKuiperCDF', 'StatsLogisticCDF', 'StatsLogisticPDF', 'StatsLogNormalCDF', + 'StatsLogNormalPDF', 'StatsMaxwellCDF', 'StatsMaxwellPDF', 'StatsMedian', + 'StatsMooreCDF', 'StatsNBinomialCDF', 'StatsNBinomialPDF', 'StatsNCChiCDF', + 'StatsNCChiPDF', 'StatsNCFCDF', 'StatsNCFPDF', 'StatsNCTCDF', 'StatsNCTPDF', + 'StatsNormalCDF', 'StatsNormalPDF', 'StatsParetoCDF', 'StatsParetoPDF', + 'StatsPermute', 'StatsPoissonCDF', 'StatsPoissonPDF', 'StatsPowerCDF', + 'StatsPowerNoise', 'StatsPowerPDF', 'StatsQCDF', 'StatsQpCDF', 'StatsRayleighCDF', + 'StatsRayleighPDF', 'StatsRectangularCDF', 'StatsRectangularPDF', 'StatsRunsCDF', + 'StatsSpearmanRhoCDF', 'StatsStudentCDF', 'StatsStudentPDF', 'StatsTopDownCDF', 'StatsTriangularCDF', 'StatsTriangularPDF', 'StatsTrimmedMean', - 'StatsUSquaredCDF', 'StatsVonMisesCDF', 'StatsVonMisesNoise', - 'StatsVonMisesPDF', 'StatsWaldCDF', 'StatsWaldPDF', 'StatsWeibullCDF', - 'StatsWeibullPDF', 'StopMSTimer', 'str2num', 'stringCRC', 'stringmatch', - 'strlen', 'strsearch', 'StudentA', 'StudentT', 'sum', 'SVAR_Exists', - 'TagVal', 'tan', 'tanh', 'ThreadGroupCreate', 'ThreadGroupRelease', - 'ThreadGroupWait', 'ThreadProcessorCount', 'ThreadReturnValue', 'ticks', - 'trunc', 'Variance', 'vcsr', 'WaveCRC', 'WaveDims', 'WaveExists', - 'WaveMax', 'WaveMin', 'WaveRefsEqual', 'WaveType', 'WhichListItem', - 'WinType', 'WNoise', 'x2pnt', 'xcsr', 'zcsr', 'ZernikeR', - ) - functions += ( - 'AddListItem', 'AnnotationInfo', 'AnnotationList', 'AxisInfo', - 'AxisList', 'CaptureHistory', 'ChildWindowList', 'CleanupName', - 'ContourInfo', 'ContourNameList', 'ControlNameList', 'CsrInfo', - 'CsrWave', 'CsrXWave', 'CTabList', 'DataFolderDir', 'date', - 'DDERequestString', 'FontList', 'FuncRefInfo', 'FunctionInfo', - 'FunctionList', 'FunctionPath', 'GetDataFolder', 'GetDefaultFont', - 'GetDimLabel', 'GetErrMessage', 'GetFormula', - 'GetIndependentModuleName', 'GetIndexedObjName', 'GetIndexedObjNameDFR', - 'GetRTErrMessage', 'GetRTStackInfo', 'GetScrapText', 'GetUserData', - 'GetWavesDataFolder', 'GrepList', 'GuideInfo', 'GuideNameList', 'Hash', - 'IgorInfo', 'ImageInfo', 'ImageNameList', 'IndexedDir', 'IndexedFile', - 'JulianToDate', 'LayoutInfo', 'ListMatch', 'LowerStr', 'MacroList', - 'NameOfWave', 'note', 'num2char', 'num2istr', 'num2str', - 'OperationList', 'PadString', 'ParseFilePath', 'PathList', 'PICTInfo', - 'PICTList', 'PossiblyQuoteName', 'ProcedureText', 'RemoveByKey', - 'RemoveEnding', 'RemoveFromList', 'RemoveListItem', - 'ReplaceNumberByKey', 'ReplaceString', 'ReplaceStringByKey', - 'Secs2Date', 'Secs2Time', 'SelectString', 'SortList', - 'SpecialCharacterInfo', 'SpecialCharacterList', 'SpecialDirPath', - 'StringByKey', 'StringFromList', 'StringList', 'StrVarOrDefault', - 'TableInfo', 'TextFile', 'ThreadGroupGetDF', 'time', 'TraceFromPixel', - 'TraceInfo', 'TraceNameList', 'UniqueName', 'UnPadString', 'UpperStr', - 'VariableList', 'WaveInfo', 'WaveList', 'WaveName', 'WaveUnits', - 'WinList', 'WinName', 'WinRecreation', 'XWaveName', - 'ContourNameToWaveRef', 'CsrWaveRef', 'CsrXWaveRef', - 'ImageNameToWaveRef', 'NewFreeWave', 'TagWaveRef', 'TraceNameToWaveRef', - 'WaveRefIndexed', 'XWaveRefFromTrace', 'GetDataFolderDFR', - 'GetWavesDataFolderDFR', 'NewFreeDataFolder', 'ThreadGroupGetDFR', + 'StatsUSquaredCDF', 'StatsVonMisesCDF', 'StatsVonMisesNoise', 'StatsVonMisesPDF', + 'StatsWaldCDF', 'StatsWaldPDF', 'StatsWeibullCDF', 'StatsWeibullPDF', + 'StopMSTimer', 'StringByKey', 'stringCRC', 'StringFromList', 'StringList', + 'stringmatch', 'strlen', 'strsearch', 'StrVarOrDefault', 'str2num', 'StudentA', + 'StudentT', 'sum', 'SVAR_Exists', 'TableInfo', 'TagVal', 'TagWaveRef', 'tan', + 'tanh', 'TextEncodingCode', 'TextEncodingName', 'TextFile', 'ThreadGroupCreate', + 'ThreadGroupGetDF', 'ThreadGroupGetDFR', 'ThreadGroupRelease', 'ThreadGroupWait', + 'ThreadProcessorCount', 'ThreadReturnValue', 'ticks', 'time', 'TraceFromPixel', + 'TraceInfo', 'TraceNameList', 'TraceNameToWaveRef', 'trunc', 'UniqueName', + 'UnPadString', 'UnsetEnvironmentVariable', 'UpperStr', 'URLDecode', 'URLEncode', + 'VariableList', 'Variance', 'vcsr', 'Voigt', 'VoigtFit', 'VoigtFitBL', + 'VoigtFit1Shape', 'VoigtFit1ShapeBL', 'VoigtFit1Shape1Width', + 'VoigtFit1Shape1WidthBL', 'VoigtFunc', 'WaveCRC', 'WaveDims', 'WaveExists', + 'WaveInfo', 'WaveList', 'WaveMax', 'WaveMin', 'WaveName', 'WaveRefIndexed', + 'WaveRefIndexedDFR', 'WaveRefsEqual', 'WaveRefWaveToList', 'WaveTextEncoding', + 'WaveType', 'WaveUnits', 'WhichListItem', 'WinList', 'WinName', 'WinRecreation', + 'WinType', 'WMFindWholeWord', 'WNoise', 'xcsr', 'XWaveName', 'XWaveRefFromTrace', + 'x2pnt', 'zcsr', 'ZernikeR', 'zeta' ) tokens = { @@ -272,7 +280,7 @@ class IgorLexer(RegexLexer): # Built-in functions. (words(functions, prefix=r'\b', suffix=r'\b'), Name.Function), # Compiler directives. - (r'^#(include|pragma|define|ifdef|ifndef|endif)', + (r'^#(include|pragma|define|undef|ifdef|ifndef|if|elif|else|endif)', Name.Decorator), (r'[^a-z"/]+$', Text), (r'.', Text), diff --git a/pygments/lexers/inferno.py b/pygments/lexers/inferno.py index bfbea571..5fc5a0ba 100644 --- a/pygments/lexers/inferno.py +++ b/pygments/lexers/inferno.py @@ -5,7 +5,7 @@ Lexers for Inferno os and all the related stuff. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/installers.py b/pygments/lexers/installers.py index c436afed..0323d140 100644 --- a/pygments/lexers/installers.py +++ b/pygments/lexers/installers.py @@ -5,7 +5,7 @@ Lexers for installer/packager DSLs and formats. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/int_fiction.py b/pygments/lexers/int_fiction.py index 724f9b27..f280a56d 100644 --- a/pygments/lexers/int_fiction.py +++ b/pygments/lexers/int_fiction.py @@ -5,7 +5,7 @@ Lexers for interactive fiction languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/iolang.py b/pygments/lexers/iolang.py index e62dd434..bbc17faf 100644 --- a/pygments/lexers/iolang.py +++ b/pygments/lexers/iolang.py @@ -5,7 +5,7 @@ Lexers for the Io language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/j.py b/pygments/lexers/j.py index 278374e5..434964fe 100644 --- a/pygments/lexers/j.py +++ b/pygments/lexers/j.py @@ -5,7 +5,7 @@ Lexer for the J programming language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -48,7 +48,7 @@ class JLexer(RegexLexer): # Definitions (r'0\s+:\s*0|noun\s+define\s*$', Name.Entity, 'nounDefinition'), - (r'\b(([1-4]|13)\s+:\s*0)|((adverb|conjunction|dyad|monad|verb)\s+define)\b', + (r'(([1-4]|13)\s+:\s*0|(adverb|conjunction|dyad|monad|verb)\s+define)\b', Name.Function, 'explicitDefinition'), # Flow Control @@ -75,8 +75,8 @@ class JLexer(RegexLexer): 'fetch', 'file2url', 'fixdotdot', 'fliprgb', 'getargs', 'getenv', 'hfd', 'inv', 'inverse', 'iospath', 'isatty', 'isutf8', 'items', 'leaf', 'list', - 'nameclass', 'namelist', 'namelist', 'names', 'nc', - 'nl', 'on', 'pick', 'pick', 'rows', + 'nameclass', 'namelist', 'names', 'nc', + 'nl', 'on', 'pick', 'rows', 'script', 'scriptd', 'sign', 'sminfo', 'smoutput', 'sort', 'split', 'stderr', 'stdin', 'stdout', 'table', 'take', 'timespacex', 'timex', 'tmoutput', diff --git a/pygments/lexers/javascript.py b/pygments/lexers/javascript.py index 2a01cd42..862535c9 100644 --- a/pygments/lexers/javascript.py +++ b/pygments/lexers/javascript.py @@ -5,7 +5,7 @@ Lexers for JavaScript and related languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -20,7 +20,7 @@ import pygments.unistring as uni __all__ = ['JavascriptLexer', 'KalLexer', 'LiveScriptLexer', 'DartLexer', 'TypeScriptLexer', 'LassoLexer', 'ObjectiveJLexer', - 'CoffeeScriptLexer', 'MaskLexer', 'EarlGreyLexer'] + 'CoffeeScriptLexer', 'MaskLexer', 'EarlGreyLexer', 'JuttleLexer'] JS_IDENT_START = ('(?:[$_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl') + ']|\\\\u[a-fA-F0-9]{4})') @@ -53,7 +53,7 @@ class JavascriptLexer(RegexLexer): 'slashstartsregex': [ include('commentsandwhitespace'), (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' - r'([gim]+\b|\B)', String.Regex, '#pop'), + r'([gimuy]+\b|\B)', String.Regex, '#pop'), (r'(?=/)', Text, ('#pop', 'badregex')), default('#pop') ], @@ -64,9 +64,14 @@ class JavascriptLexer(RegexLexer): (r'\A#! ?/.*?\n', Comment.Hashbang), # recognized by node.js (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'), include('commentsandwhitespace'), + (r'(\.\d+|[0-9]+\.[0-9]*)([eE][-+]?[0-9]+)?', Number.Float), + (r'0[bB][01]+', Number.Bin), + (r'0[oO][0-7]+', Number.Oct), + (r'0[xX][0-9a-fA-F]+', Number.Hex), + (r'[0-9]+', Number.Integer), + (r'\.\.\.|=>', Punctuation), (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|' - r'(<<|>>>?|=>|==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'), - (r'\.\.\.', Punctuation), + r'(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'), (r'[{(\[;,]', Punctuation, 'slashstartsregex'), (r'[})\].]', Punctuation), (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|' @@ -84,11 +89,6 @@ class JavascriptLexer(RegexLexer): r'Error|eval|isFinite|isNaN|isSafeInteger|parseFloat|parseInt|' r'document|this|window)\b', Name.Builtin), (JS_IDENT, Name.Other), - (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), - (r'0b[01]+', Number.Bin), - (r'0o[0-7]+', Number.Oct), - (r'0x[0-9a-fA-F]+', Number.Hex), - (r'[0-9]+', Number.Integer), (r'"(\\\\|\\"|[^"])*"', String.Double), (r"'(\\\\|\\'|[^'])*'", String.Single), (r'`', String.Backtick, 'interp'), @@ -97,13 +97,13 @@ class JavascriptLexer(RegexLexer): (r'`', String.Backtick, '#pop'), (r'\\\\', String.Backtick), (r'\\`', String.Backtick), - (r'\${', String.Interpol, 'interp-inside'), + (r'\$\{', String.Interpol, 'interp-inside'), (r'\$', String.Backtick), (r'[^`\\$]+', String.Backtick), ], 'interp-inside': [ # TODO: should this include single-line comments and allow nesting strings? - (r'}', String.Interpol, '#pop'), + (r'\}', String.Interpol, '#pop'), include('root'), ], # (\\\\|\\`|[^`])*`', String.Backtick), @@ -366,9 +366,10 @@ class DartLexer(RegexLexer): (r'\b(assert|break|case|catch|continue|default|do|else|finally|for|' r'if|in|is|new|return|super|switch|this|throw|try|while)\b', Keyword), - (r'\b(abstract|const|extends|factory|final|get|implements|' - r'native|operator|set|static|typedef|var)\b', Keyword.Declaration), - (r'\b(bool|double|Dynamic|int|num|Object|String|void)\b', Keyword.Type), + (r'\b(abstract|async|await|const|extends|factory|final|get|' + r'implements|native|operator|set|static|sync|typedef|var|with|' + r'yield)\b', Keyword.Declaration), + (r'\b(bool|double|dynamic|int|num|Object|String|void)\b', Keyword.Type), (r'\b(false|null|true)\b', Keyword.Constant), (r'[~!%^&*+=|?:<>/-]|as\b', Operator), (r'[a-zA-Z_$]\w*:', Name.Label), @@ -447,7 +448,7 @@ class TypeScriptLexer(RegexLexer): name = 'TypeScript' aliases = ['ts', 'typescript'] - filenames = ['*.ts'] + filenames = ['*.ts', '*.tsx'] mimetypes = ['text/x-typescript'] flags = re.DOTALL | re.MULTILINE @@ -511,11 +512,34 @@ class TypeScriptLexer(RegexLexer): (r'[0-9]+', Number.Integer), (r'"(\\\\|\\"|[^"])*"', String.Double), (r"'(\\\\|\\'|[^'])*'", String.Single), + (r'`', String.Backtick, 'interp'), # Match stuff like: Decorators (r'@\w+', Keyword.Declaration), - ] + ], + + # The 'interp*' rules match those in JavascriptLexer. Changes made + # there should be reflected here as well. + 'interp': [ + (r'`', String.Backtick, '#pop'), + (r'\\\\', String.Backtick), + (r'\\`', String.Backtick), + (r'\$\{', String.Interpol, 'interp-inside'), + (r'\$', String.Backtick), + (r'[^`\\$]+', String.Backtick), + ], + 'interp-inside': [ + # TODO: should this include single-line comments and allow nesting strings? + (r'\}', String.Interpol, '#pop'), + include('root'), + ], } + def analyse_text(text): + if re.search('^(import.+(from\s+)?["\']|' + '(export\s*)?(interface|class|function)\s+)', + text, re.MULTILINE): + return 1.0 + class LassoLexer(RegexLexer): """ @@ -545,12 +569,7 @@ class LassoLexer(RegexLexer): tokens = { 'root': [ (r'^#![ \S]+lasso9\b', Comment.Preproc, 'lasso'), - (r'\[no_square_brackets\]', Comment.Preproc, 'nosquarebrackets'), - (r'\[noprocess\]', Comment.Preproc, ('delimiters', 'noprocess')), - (r'\[', Comment.Preproc, ('delimiters', 'squarebrackets')), - (r'<\?(LassoScript|lasso|=)', Comment.Preproc, - ('delimiters', 'anglebrackets')), - (r'<(!--.*?-->)?', Other, 'delimiters'), + (r'(?=\[|<)', Other, 'delimiters'), (r'\s+', Other), default(('delimiters', 'lassofile')), ], @@ -558,14 +577,14 @@ class LassoLexer(RegexLexer): (r'\[no_square_brackets\]', Comment.Preproc, 'nosquarebrackets'), (r'\[noprocess\]', Comment.Preproc, 'noprocess'), (r'\[', Comment.Preproc, 'squarebrackets'), - (r'<\?(LassoScript|lasso|=)', Comment.Preproc, 'anglebrackets'), + (r'<\?(lasso(script)?|=)', Comment.Preproc, 'anglebrackets'), (r'<(!--.*?-->)?', Other), (r'[^[<]+', Other), ], 'nosquarebrackets': [ (r'\[noprocess\]', Comment.Preproc, 'noprocess'), (r'\[', Other), - (r'<\?(LassoScript|lasso|=)', Comment.Preproc, 'anglebrackets'), + (r'<\?(lasso(script)?|=)', Comment.Preproc, 'anglebrackets'), (r'<(!--.*?-->)?', Other), (r'[^[<]+', Other), ], @@ -607,7 +626,7 @@ class LassoLexer(RegexLexer): # names (r'\$[a-z_][\w.]*', Name.Variable), - (r'#([a-z_][\w.]*|\d+)', Name.Variable.Instance), + (r'#([a-z_][\w.]*|\d+\b)', Name.Variable.Instance), (r"(\.\s*)('[a-z_][\w.]*')", bygroups(Name.Builtin.Pseudo, Name.Variable.Class)), (r"(self)(\s*->\s*)('[a-z_][\w.]*')", @@ -658,20 +677,20 @@ class LassoLexer(RegexLexer): r'Database_TableNames|Define_Tag|Define_Type|Email_Batch|' r'Encode_Set|HTML_Comment|Handle|Handle_Error|Header|If|Inline|' r'Iterate|LJAX_Target|Link|Link_CurrentAction|Link_CurrentGroup|' - r'Link_CurrentRecord|Link_Detail|Link_FirstGroup|' - r'Link_FirstRecord|Link_LastGroup|Link_LastRecord|Link_NextGroup|' - r'Link_NextRecord|Link_PrevGroup|Link_PrevRecord|Log|Loop|' - r'NoProcess|Output_None|Portal|Private|Protect|Records|Referer|' - r'Referrer|Repeating|ResultSet|Rows|Search_Args|Search_Arguments|' - r'Select|Sort_Args|Sort_Arguments|Thread_Atomic|Value_List|While|' - r'Abort|Case|Else|If_Empty|If_False|If_Null|If_True|Loop_Abort|' - r'Loop_Continue|Loop_Count|Params|Params_Up|Return|Return_Value|' - r'Run_Children|SOAP_DefineTag|SOAP_LastRequest|SOAP_LastResponse|' - r'Tag_Name|ascending|average|by|define|descending|do|equals|' - r'frozen|group|handle_failure|import|in|into|join|let|match|max|' - r'min|on|order|parent|protected|provide|public|require|returnhome|' - r'skip|split_thread|sum|take|thread|to|trait|type|where|with|' - r'yield|yieldhome)\b', + r'Link_CurrentRecord|Link_Detail|Link_FirstGroup|Link_FirstRecord|' + r'Link_LastGroup|Link_LastRecord|Link_NextGroup|Link_NextRecord|' + r'Link_PrevGroup|Link_PrevRecord|Log|Loop|Output_None|Portal|' + r'Private|Protect|Records|Referer|Referrer|Repeating|ResultSet|' + r'Rows|Search_Args|Search_Arguments|Select|Sort_Args|' + r'Sort_Arguments|Thread_Atomic|Value_List|While|Abort|Case|Else|' + r'Fail_If|Fail_IfNot|Fail|If_Empty|If_False|If_Null|If_True|' + r'Loop_Abort|Loop_Continue|Loop_Count|Params|Params_Up|Return|' + r'Return_Value|Run_Children|SOAP_DefineTag|SOAP_LastRequest|' + r'SOAP_LastResponse|Tag_Name|ascending|average|by|define|' + r'descending|do|equals|frozen|group|handle_failure|import|in|into|' + r'join|let|match|max|min|on|order|parent|protected|provide|public|' + r'require|returnhome|skip|split_thread|sum|take|thread|to|trait|' + r'type|where|with|yield|yieldhome)\b', bygroups(Punctuation, Keyword)), # other @@ -1016,6 +1035,12 @@ class CoffeeScriptLexer(RegexLexer): filenames = ['*.coffee'] mimetypes = ['text/coffeescript'] + + _operator_re = ( + r'\+\+|~|&&|\band\b|\bor\b|\bis\b|\bisnt\b|\bnot\b|\?|:|' + r'\|\||\\(?=\n)|' + r'(<<|>>>?|==?(?!>)|!=?|=(?!>)|-(?!>)|[<>+*`%&\|\^/])=?') + flags = re.DOTALL tokens = { 'commentsandwhitespace': [ @@ -1034,17 +1059,17 @@ class CoffeeScriptLexer(RegexLexer): (r'///', String.Regex, ('#pop', 'multilineregex')), (r'/(?! )(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' r'([gim]+\b|\B)', String.Regex, '#pop'), + # This isn't really guarding against mishighlighting well-formed + # code, just the ability to infinite-loop between root and + # slashstartsregex. + (r'/', Operator), default('#pop'), ], 'root': [ - # this next expr leads to infinite loops root -> slashstartsregex - # (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'), include('commentsandwhitespace'), - (r'\+\+|~|&&|\band\b|\bor\b|\bis\b|\bisnt\b|\bnot\b|\?|:|' - r'\|\||\\(?=\n)|' - r'(<<|>>>?|==?(?!>)|!=?|=(?!>)|-(?!>)|[<>+*`%&|^/])=?', - Operator, 'slashstartsregex'), - (r'(?:\([^()]*\))?\s*[=-]>', Name.Function), + (r'^(?=\s|/)', Text, 'slashstartsregex'), + (_operator_re, Operator, 'slashstartsregex'), + (r'(?:\([^()]*\))?\s*[=-]>', Name.Function, 'slashstartsregex'), (r'[{(\[;,]', Punctuation, 'slashstartsregex'), (r'[})\].]', Punctuation), (r'(?<![.$])(for|own|in|of|while|until|' @@ -1065,7 +1090,7 @@ class CoffeeScriptLexer(RegexLexer): (r'@[$a-zA-Z_][\w.:$]*\s*[:=]\s', Name.Variable.Instance, 'slashstartsregex'), (r'@', Name.Other, 'slashstartsregex'), - (r'@?[$a-zA-Z_][\w$]*', Name.Other, 'slashstartsregex'), + (r'@?[$a-zA-Z_][\w$]*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -1245,32 +1270,32 @@ class EarlGreyLexer(RegexLexer): include('control'), (r'[^\S\n]+', Text), (r';;.*\n', Comment), - (r'[\[\]\{\}\:\(\)\,\;]', Punctuation), + (r'[\[\]{}:(),;]', Punctuation), (r'\\\n', Text), (r'\\', Text), include('errors'), (words(( 'with', 'where', 'when', 'and', 'not', 'or', 'in', 'as', 'of', 'is'), - prefix=r'(?<=\s|\[)', suffix=r'(?![\w\$\-])'), + prefix=r'(?<=\s|\[)', suffix=r'(?![\w$\-])'), Operator.Word), - (r'[\*@]?->', Name.Function), + (r'[*@]?->', Name.Function), (r'[+\-*/~^<>%&|?!@#.]*=', Operator.Word), (r'\.{2,3}', Operator.Word), # Range Operator (r'([+*/~^<>&|?!]+)|([#\-](?=\s))|@@+(?=\s)|=+', Operator), - (r'(?<![\w\$\-])(var|let)(?:[^\w\$])', Keyword.Declaration), + (r'(?<![\w$\-])(var|let)(?:[^\w$])', Keyword.Declaration), include('keywords'), include('builtins'), include('assignment'), (r'''(?x) - (?:()([a-zA-Z$_](?:[a-zA-Z$0-9_\-]*[a-zA-Z$0-9_])?)| - (?<=[\s\{\[\(])(\.)([a-zA-Z$_](?:[a-zA-Z$0-9_\-]*[a-zA-Z$0-9_])?)) + (?:()([a-zA-Z$_](?:[\w$\-]*[\w$])?)| + (?<=[\s{\[(])(\.)([a-zA-Z$_](?:[\w$\-]*[\w$])?)) (?=.*%)''', bygroups(Punctuation, Name.Tag, Punctuation, Name.Class.Start), 'dbs'), (r'[rR]?`', String.Backtick, 'bt'), (r'[rR]?```', String.Backtick, 'tbt'), - (r'(?<=[\s\[\{\(,;])\.([a-zA-Z$_](?:[a-zA-Z$0-9_-]*[a-zA-Z$0-9_])?)' - r'(?=[\s\]\}\),;])', String.Symbol), + (r'(?<=[\s\[{(,;])\.([a-zA-Z$_](?:[\w$\-]*[\w$])?)' + r'(?=[\s\]}),;])', String.Symbol), include('nested'), (r'(?:[rR]|[rR]\.[gmi]{1,3})?"', String, combined('stringescape', 'dqs')), (r'(?:[rR]|[rR]\.[gmi]{1,3})?\'', String, combined('stringescape', 'sqs')), @@ -1281,9 +1306,9 @@ class EarlGreyLexer(RegexLexer): include('numbers'), ], 'dbs': [ - (r'(\.)([a-zA-Z$_](?:[a-zA-Z$0-9_\-]*[a-zA-Z$0-9_])?)(?=[\[\.\s])', + (r'(\.)([a-zA-Z$_](?:[\w$\-]*[\w$])?)(?=[.\[\s])', bygroups(Punctuation, Name.Class.DBS)), - (r'(\[)([\^#][a-zA-Z$_](?:[a-zA-Z$0-9_\-]*[a-zA-Z$0-9_])?)(\])', + (r'(\[)([\^#][a-zA-Z$_](?:[\w$\-]*[\w$])?)(\])', bygroups(Punctuation, Name.Entity.DBS, Punctuation)), (r'\s+', Text), (r'%', Operator.DBS, '#pop'), @@ -1293,29 +1318,29 @@ class EarlGreyLexer(RegexLexer): bygroups(Text.Whitespace, Text)), ], 'assignment': [ - (r'(\.)?([a-zA-Z$_](?:[a-zA-Z$0-9_-]*[a-zA-Z$0-9_])?)' + (r'(\.)?([a-zA-Z$_](?:[\w$\-]*[\w$])?)' r'(?=\s+[+\-*/~^<>%&|?!@#.]*\=\s)', bygroups(Punctuation, Name.Variable)) ], 'errors': [ (words(('Error', 'TypeError', 'ReferenceError'), - prefix=r'(?<![\w\$\-\.])', suffix=r'(?![\w\$\-\.])'), + prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$.])'), Name.Exception), (r'''(?x) - (?<![\w\$]) - E\.[\w\$](?:[\w\$\-]*[\w\$])? - (?:\.[\w\$](?:[\w\$\-]*[\w\$])?)* - (?=[\(\{\[\?\!\s])''', + (?<![\w$]) + E\.[\w$](?:[\w$\-]*[\w$])? + (?:\.[\w$](?:[\w$\-]*[\w$])?)* + (?=[({\[?!\s])''', Name.Exception), ], 'control': [ (r'''(?x) - ([a-zA-Z$_](?:[a-zA-Z$0-9_-]*[a-zA-Z$0-9_])?) + ([a-zA-Z$_](?:[\w$-]*[\w$])?) (?!\n)\s+ (?!and|as|each\*|each|in|is|mod|of|or|when|where|with) - (?=(?:[+\-*/~^<>%&|?!@#.])?[a-zA-Z$_](?:[a-zA-Z$0-9_-]*[a-zA-Z$0-9_])?)''', + (?=(?:[+\-*/~^<>%&|?!@#.])?[a-zA-Z$_](?:[\w$-]*[\w$])?)''', Keyword.Control), - (r'([a-zA-Z$_](?:[a-zA-Z$0-9_-]*[a-zA-Z$0-9_])?)(?!\n)\s+(?=[\'"\d\{\[\(])', + (r'([a-zA-Z$_](?:[\w$-]*[\w$])?)(?!\n)\s+(?=[\'"\d{\[(])', Keyword.Control), (r'''(?x) (?: @@ -1324,28 +1349,28 @@ class EarlGreyLexer(RegexLexer): (?<=with|each|with)| (?<=each\*|where) )(\s+) - ([a-zA-Z$_](?:[a-zA-Z$0-9_\-]*[a-zA-Z$0-9_])?)(:)''', + ([a-zA-Z$_](?:[\w$-]*[\w$])?)(:)''', bygroups(Text, Keyword.Control, Punctuation)), (r'''(?x) (?<![+\-*/~^<>%&|?!@#.])(\s+) - ([a-zA-Z$_](?:[a-zA-Z$0-9_-]*[a-zA-Z$0-9_])?)(:)''', + ([a-zA-Z$_](?:[\w$-]*[\w$])?)(:)''', bygroups(Text, Keyword.Control, Punctuation)), ], 'nested': [ (r'''(?x) - (?<=[a-zA-Z$0-9_\]\}\)])(\.) - ([a-zA-Z$_](?:[a-zA-Z$0-9_-]*[a-zA-Z$0-9_])?) + (?<=[\w$\]})])(\.) + ([a-zA-Z$_](?:[\w$-]*[\w$])?) (?=\s+with(?:\s|\n))''', bygroups(Punctuation, Name.Function)), (r'''(?x) (?<!\s)(\.) - ([a-zA-Z$_](?:[a-zA-Z$0-9_-]*[a-zA-Z$0-9_])?) - (?=[\}\]\)\.,;:\s])''', + ([a-zA-Z$_](?:[\w$-]*[\w$])?) + (?=[}\]).,;:\s])''', bygroups(Punctuation, Name.Field)), (r'''(?x) - (?<=[a-zA-Z$0-9_\]\}\)])(\.) - ([a-zA-Z$_](?:[a-zA-Z$0-9_-]*[a-zA-Z$0-9_])?) - (?=[\[\{\(:])''', + (?<=[\w$\]})])(\.) + ([a-zA-Z$_](?:[\w$-]*[\w$])?) + (?=[\[{(:])''', bygroups(Punctuation, Name.Function)), ], 'keywords': [ @@ -1354,15 +1379,15 @@ class EarlGreyLexer(RegexLexer): 'continue', 'elif', 'expr-value', 'if', 'match', 'return', 'yield', 'pass', 'else', 'require', 'var', 'let', 'async', 'method', 'gen'), - prefix=r'(?<![\w\$\-\.])', suffix=r'(?![\w\$\-\.])'), + prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$.])'), Keyword.Pseudo), (words(('this', 'self', '@'), - prefix=r'(?<![\w\$\-\.])', suffix=r'(?![\w\$\-])'), + prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$])'), Keyword.Constant), (words(( 'Function', 'Object', 'Array', 'String', 'Number', 'Boolean', 'ErrorFactory', 'ENode', 'Promise'), - prefix=r'(?<![\w\$\-\.])', suffix=r'(?![\w\$\-])'), + prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$])'), Keyword.Type), ], 'builtins': [ @@ -1373,20 +1398,20 @@ class EarlGreyLexer(RegexLexer): 'getChecker', 'get-checker', 'getProperty', 'get-property', 'getProjector', 'get-projector', 'consume', 'take', 'promisify', 'spawn', 'constructor'), - prefix=r'(?<![\w\-#\.])', suffix=r'(?![\w\-\.])'), + prefix=r'(?<![\w\-#.])', suffix=r'(?![\w\-.])'), Name.Builtin), (words(( 'true', 'false', 'null', 'undefined'), - prefix=r'(?<![\w\$\-\.])', suffix=r'(?![\w\$\-\.])'), + prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$.])'), Name.Constant), ], 'name': [ - (r'@([a-zA-Z$_](?:[a-zA-Z$0-9_-]*[a-zA-Z$0-9_])?)', Name.Variable.Instance), - (r'([a-zA-Z$_](?:[a-zA-Z$0-9_-]*[a-zA-Z$0-9_])?)(\+\+|\-\-)?', + (r'@([a-zA-Z$_](?:[\w$-]*[\w$])?)', Name.Variable.Instance), + (r'([a-zA-Z$_](?:[\w$-]*[\w$])?)(\+\+|\-\-)?', bygroups(Name.Symbol, Operator.Word)) ], 'tuple': [ - (r'#[a-zA-Z_][a-zA-Z_\-0-9]*(?=[\s\{\(,;\n])', Name.Namespace) + (r'#[a-zA-Z_][\w\-]*(?=[\s{(,;])', Name.Namespace) ], 'interpoling_string': [ (r'\}', String.Interpol, '#pop'), @@ -1426,7 +1451,7 @@ class EarlGreyLexer(RegexLexer): (r'```', String.Backtick, '#pop'), (r'\n', String.Backtick), (r'\^=?', String.Escape), - (r'[^\`]+', String.Backtick), + (r'[^`]+', String.Backtick), ], 'numbers': [ (r'\d+\.(?!\.)\d*([eE][+-]?[0-9]+)?', Number.Float), @@ -1434,7 +1459,67 @@ class EarlGreyLexer(RegexLexer): (r'8r[0-7]+', Number.Oct), (r'2r[01]+', Number.Bin), (r'16r[a-fA-F0-9]+', Number.Hex), - (r'([3-79]|[1-2][0-9]|3[0-6])r[a-zA-Z\d]+(\.[a-zA-Z\d]+)?', Number.Radix), + (r'([3-79]|[12][0-9]|3[0-6])r[a-zA-Z\d]+(\.[a-zA-Z\d]+)?', Number.Radix), (r'\d+', Number.Integer) ], } + +class JuttleLexer(RegexLexer): + """ + For `Juttle`_ source code. + + .. _Juttle: https://github.com/juttle/juttle + + """ + + name = 'Juttle' + aliases = ['juttle', 'juttle'] + filenames = ['*.juttle'] + mimetypes = ['application/juttle', 'application/x-juttle', + 'text/x-juttle', 'text/juttle'] + + flags = re.DOTALL | re.UNICODE | re.MULTILINE + + tokens = { + 'commentsandwhitespace': [ + (r'\s+', Text), + (r'//.*?\n', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline) + ], + 'slashstartsregex': [ + include('commentsandwhitespace'), + (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' + r'([gim]+\b|\B)', String.Regex, '#pop'), + (r'(?=/)', Text, ('#pop', 'badregex')), + default('#pop') + ], + 'badregex': [ + (r'\n', Text, '#pop') + ], + 'root': [ + (r'^(?=\s|/)', Text, 'slashstartsregex'), + include('commentsandwhitespace'), + (r':\d{2}:\d{2}:\d{2}(\.\d*)?:', String.Moment), + (r':(now|beginning|end|forever|yesterday|today|tomorrow|(\d+(\.\d*)?|\.\d+)(ms|[smhdwMy])?):', String.Moment), + (r':\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d*)?)?(Z|[+-]\d{2}:\d{2}|[+-]\d{4})?:', String.Moment), + (r':((\d+(\.\d*)?|\.\d+)[ ]+)?(millisecond|second|minute|hour|day|week|month|year)[s]?' + r'(([ ]+and[ ]+(\d+[ ]+)?(millisecond|second|minute|hour|day|week|month|year)[s]?)' + r'|[ ]+(ago|from[ ]+now))*:', String.Moment), + (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|' + r'(==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'), + (r'[{(\[;,]', Punctuation, 'slashstartsregex'), + (r'[})\].]', Punctuation), + (r'(import|return|continue|if|else)\b', Keyword, 'slashstartsregex'), + (r'(var|const|function|reducer|sub|input)\b', Keyword.Declaration, 'slashstartsregex'), + (r'(batch|emit|filter|head|join|keep|pace|pass|put|read|reduce|remove|' + r'sequence|skip|sort|split|tail|unbatch|uniq|view|write)\b', Keyword.Reserved), + (r'(true|false|null|Infinity)\b', Keyword.Constant), + (r'(Array|Date|Juttle|Math|Number|Object|RegExp|String)\b', Name.Builtin), + (JS_IDENT, Name.Other), + (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), + (r'[0-9]+', Number.Integer), + (r'"(\\\\|\\"|[^"])*"', String.Double), + (r"'(\\\\|\\'|[^'])*'", String.Single) + ] + + } diff --git a/pygments/lexers/julia.py b/pygments/lexers/julia.py index d0aa6d35..67453aba 100644 --- a/pygments/lexers/julia.py +++ b/pygments/lexers/julia.py @@ -5,19 +5,24 @@ Lexers for the Julia language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re -from pygments.lexer import Lexer, RegexLexer, bygroups, combined, do_insertions +from pygments.lexer import Lexer, RegexLexer, bygroups, do_insertions, \ + words, include from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Generic from pygments.util import shebang_matches, unirange __all__ = ['JuliaLexer', 'JuliaConsoleLexer'] +allowed_variable = ( + u'(?:[a-zA-Z_\u00A1-\uffff]|%s)(?:[a-zA-Z_0-9\u00A1-\uffff]|%s)*!*' % + ((unirange(0x10000, 0x10ffff),) * 2)) + class JuliaLexer(RegexLexer): """ @@ -25,6 +30,7 @@ class JuliaLexer(RegexLexer): .. versionadded:: 1.6 """ + name = 'Julia' aliases = ['julia', 'jl'] filenames = ['*.jl'] @@ -32,51 +38,151 @@ class JuliaLexer(RegexLexer): flags = re.MULTILINE | re.UNICODE - builtins = [ - 'exit', 'whos', 'edit', 'load', 'is', 'isa', 'isequal', 'typeof', 'tuple', - 'ntuple', 'uid', 'hash', 'finalizer', 'convert', 'promote', 'subtype', - 'typemin', 'typemax', 'realmin', 'realmax', 'sizeof', 'eps', 'promote_type', - 'method_exists', 'applicable', 'invoke', 'dlopen', 'dlsym', 'system', - 'error', 'throw', 'assert', 'new', 'Inf', 'Nan', 'pi', 'im', - ] - tokens = { 'root': [ (r'\n', Text), (r'[^\S\n]+', Text), (r'#=', Comment.Multiline, "blockcomment"), (r'#.*$', Comment), - (r'[]{}:(),;[@]', Punctuation), - (r'\\\n', Text), - (r'\\', Text), + (r'[\[\]{}(),;]', Punctuation), # keywords - (r'(begin|while|for|in|return|break|continue|' - r'macro|quote|let|if|elseif|else|try|catch|end|' - r'bitstype|ccall|do|using|module|import|export|' - r'importall|baremodule|immutable)\b', Keyword), + (r'in\b', Keyword.Pseudo), + (r'(true|false)\b', Keyword.Constant), (r'(local|global|const)\b', Keyword.Declaration), - (r'(Bool|Int|Int8|Int16|Int32|Int64|Uint|Uint8|Uint16|Uint32|Uint64' - r'|Float32|Float64|Complex64|Complex128|Any|Nothing|None)\b', - Keyword.Type), - + (words([ + 'function', 'type', 'typealias', 'abstract', 'immutable', + 'baremodule', 'begin', 'bitstype', 'break', 'catch', 'ccall', + 'continue', 'do', 'else', 'elseif', 'end', 'export', 'finally', + 'for', 'if', 'import', 'importall', 'let', 'macro', 'module', + 'quote', 'return', 'try', 'using', 'while'], + suffix=r'\b'), Keyword), + + # NOTE + # Patterns below work only for definition sites and thus hardly reliable. + # # functions - (r'(function)((?:\s|\\\s)+)', - bygroups(Keyword, Name.Function), 'funcname'), - + # (r'(function)(\s+)(' + allowed_variable + ')', + # bygroups(Keyword, Text, Name.Function)), + # # types - (r'(type|typealias|abstract|immutable)((?:\s|\\\s)+)', - bygroups(Keyword, Name.Class), 'typename'), - - # operators - (r'==|!=|<=|>=|->|&&|\|\||::|<:|[-~+/*%=<>&^|.?!$]', Operator), - (r'\.\*|\.\^|\.\\|\.\/|\\', Operator), + # (r'(type|typealias|abstract|immutable)(\s+)(' + allowed_variable + ')', + # bygroups(Keyword, Text, Name.Class)), + + # type names + (words([ + 'ANY', 'ASCIIString', 'AbstractArray', 'AbstractChannel', + 'AbstractFloat', 'AbstractMatrix', 'AbstractRNG', + 'AbstractSparseArray', 'AbstractSparseMatrix', + 'AbstractSparseVector', 'AbstractString', 'AbstractVecOrMat', + 'AbstractVector', 'Any', 'ArgumentError', 'Array', + 'AssertionError', 'Associative', 'Base64DecodePipe', + 'Base64EncodePipe', 'Bidiagonal', 'BigFloat', 'BigInt', + 'BitArray', 'BitMatrix', 'BitVector', 'Bool', 'BoundsError', + 'Box', 'BufferStream', 'CapturedException', 'CartesianIndex', + 'CartesianRange', 'Cchar', 'Cdouble', 'Cfloat', 'Channel', + 'Char', 'Cint', 'Cintmax_t', 'Clong', 'Clonglong', + 'ClusterManager', 'Cmd', 'Coff_t', 'Colon', 'Complex', + 'Complex128', 'Complex32', 'Complex64', 'CompositeException', + 'Condition', 'Cptrdiff_t', 'Cshort', 'Csize_t', 'Cssize_t', + 'Cstring', 'Cuchar', 'Cuint', 'Cuintmax_t', 'Culong', + 'Culonglong', 'Cushort', 'Cwchar_t', 'Cwstring', 'DataType', + 'Date', 'DateTime', 'DenseArray', 'DenseMatrix', + 'DenseVecOrMat', 'DenseVector', 'Diagonal', 'Dict', + 'DimensionMismatch', 'Dims', 'DirectIndexString', 'Display', + 'DivideError', 'DomainError', 'EOFError', 'EachLine', 'Enum', + 'Enumerate', 'ErrorException', 'Exception', 'Expr', + 'Factorization', 'FileMonitor', 'FileOffset', 'Filter', + 'Float16', 'Float32', 'Float64', 'FloatRange', 'Function', + 'GenSym', 'GlobalRef', 'GotoNode', 'HTML', 'Hermitian', 'IO', + 'IOBuffer', 'IOStream', 'IPv4', 'IPv6', 'InexactError', + 'InitError', 'Int', 'Int128', 'Int16', 'Int32', 'Int64', 'Int8', + 'IntSet', 'Integer', 'InterruptException', 'IntrinsicFunction', + 'InvalidStateException', 'Irrational', 'KeyError', 'LabelNode', + 'LambdaStaticData', 'LinSpace', 'LineNumberNode', 'LoadError', + 'LocalProcess', 'LowerTriangular', 'MIME', 'Matrix', + 'MersenneTwister', 'Method', 'MethodError', 'MethodTable', + 'Module', 'NTuple', 'NewvarNode', 'NullException', 'Nullable', + 'Number', 'ObjectIdDict', 'OrdinalRange', 'OutOfMemoryError', + 'OverflowError', 'Pair', 'ParseError', 'PartialQuickSort', + 'Pipe', 'PollingFileWatcher', 'ProcessExitedException', + 'ProcessGroup', 'Ptr', 'QuoteNode', 'RandomDevice', 'Range', + 'Rational', 'RawFD', 'ReadOnlyMemoryError', 'Real', + 'ReentrantLock', 'Ref', 'Regex', 'RegexMatch', + 'RemoteException', 'RemoteRef', 'RepString', 'RevString', + 'RopeString', 'RoundingMode', 'SegmentationFault', + 'SerializationState', 'Set', 'SharedArray', 'SharedMatrix', + 'SharedVector', 'Signed', 'SimpleVector', 'SparseMatrixCSC', + 'StackOverflowError', 'StatStruct', 'StepRange', 'StridedArray', + 'StridedMatrix', 'StridedVecOrMat', 'StridedVector', 'SubArray', + 'SubString', 'SymTridiagonal', 'Symbol', 'SymbolNode', + 'Symmetric', 'SystemError', 'TCPSocket', 'Task', 'Text', + 'TextDisplay', 'Timer', 'TopNode', 'Tridiagonal', 'Tuple', + 'Type', 'TypeConstructor', 'TypeError', 'TypeName', 'TypeVar', + 'UDPSocket', 'UInt', 'UInt128', 'UInt16', 'UInt32', 'UInt64', + 'UInt8', 'UTF16String', 'UTF32String', 'UTF8String', + 'UndefRefError', 'UndefVarError', 'UnicodeError', 'UniformScaling', + 'Union', 'UnitRange', 'Unsigned', 'UpperTriangular', 'Val', + 'Vararg', 'VecOrMat', 'Vector', 'VersionNumber', 'Void', 'WString', + 'WeakKeyDict', 'WeakRef', 'WorkerConfig', 'Zip'], suffix=r'\b'), + Keyword.Type), # builtins - ('(' + '|'.join(builtins) + r')\b', Name.Builtin), + (words([ + u'ARGS', u'CPU_CORES', u'C_NULL', u'DevNull', u'ENDIAN_BOM', + u'ENV', u'I', u'Inf', u'Inf16', u'Inf32', u'Inf64', + u'InsertionSort', u'JULIA_HOME', u'LOAD_PATH', u'MergeSort', + u'NaN', u'NaN16', u'NaN32', u'NaN64', u'OS_NAME', + u'QuickSort', u'RoundDown', u'RoundFromZero', u'RoundNearest', + u'RoundNearestTiesAway', u'RoundNearestTiesUp', + u'RoundToZero', u'RoundUp', u'STDERR', u'STDIN', u'STDOUT', + u'VERSION', u'WORD_SIZE', u'catalan', u'e', u'eu', + u'eulergamma', u'golden', u'im', u'nothing', u'pi', u'γ', + u'π', u'φ'], + suffix=r'\b'), Name.Builtin), - # backticks - (r'`(?s).*?`', String.Backtick), + # operators + # see: https://github.com/JuliaLang/julia/blob/master/src/julia-parser.scm + (words([ + # prec-assignment + u'=', u':=', u'+=', u'-=', u'*=', u'/=', u'//=', u'.//=', u'.*=', u'./=', + u'\=', u'.\=', u'^=', u'.^=', u'÷=', u'.÷=', u'%=', u'.%=', u'|=', u'&=', + u'$=', u'=>', u'<<=', u'>>=', u'>>>=', u'~', u'.+=', u'.-=', + # prec-conditional + u'?', + # prec-arrow + u'--', u'-->', + # prec-lazy-or + u'||', + # prec-lazy-and + u'&&', + # prec-comparison + u'>', u'<', u'>=', u'≥', u'<=', u'≤', u'==', u'===', u'≡', u'!=', u'≠', + u'!==', u'≢', u'.>', u'.<', u'.>=', u'.≥', u'.<=', u'.≤', u'.==', u'.!=', + u'.≠', u'.=', u'.!', u'<:', u'>:', u'∈', u'∉', u'∋', u'∌', u'⊆', + u'⊈', u'⊂', + u'⊄', u'⊊', + # prec-pipe + u'|>', u'<|', + # prec-colon + u':', + # prec-plus + u'+', u'-', u'.+', u'.-', u'|', u'∪', u'$', + # prec-bitshift + u'<<', u'>>', u'>>>', u'.<<', u'.>>', u'.>>>', + # prec-times + u'*', u'/', u'./', u'÷', u'.÷', u'%', u'⋅', u'.%', u'.*', u'\\', u'.\\', u'&', u'∩', + # prec-rational + u'//', u'.//', + # prec-power + u'^', u'.^', + # prec-decl + u'::', + # prec-dot + u'.', + # unary op + u'+', u'-', u'!', u'~', u'√', u'∛', u'∜' + ]), Operator), # chars (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|" @@ -86,13 +192,19 @@ class JuliaLexer(RegexLexer): (r'(?<=[.\w)\]])\'+', Operator), # strings - (r'(?:[IL])"', String, 'string'), - (r'[E]?"', String, combined('stringescape', 'string')), + (r'"""', String, 'tqstring'), + (r'"', String, 'string'), + + # regular expressions + (r'r"""', String.Regex, 'tqregex'), + (r'r"', String.Regex, 'regex'), + + # backticks + (r'`', String.Backtick, 'command'), # names - (r'@[\w.]+', Name.Decorator), - (u'(?:[a-zA-Z_\u00A1-\uffff]|%s)(?:[a-zA-Z_0-9\u00A1-\uffff]|%s)*!*' % - ((unirange(0x10000, 0x10ffff),)*2), Name), + (allowed_variable, Name), + (r'@' + allowed_variable, Name.Decorator), # numbers (r'(\d+(_\d+)+\.\d*|\d*\.\d+(_\d+)+)([eEf][+-]?[0-9]+)?', Number.Float), @@ -109,45 +221,59 @@ class JuliaLexer(RegexLexer): (r'\d+', Number.Integer) ], - 'funcname': [ - ('[a-zA-Z_]\w*', Name.Function, '#pop'), - ('\([^\s\w{]{1,2}\)', Operator, '#pop'), - ('[^\s\w{]{1,2}', Operator, '#pop'), - ], - - 'typename': [ - ('[a-zA-Z_]\w*', Name.Class, '#pop') - ], - - 'stringescape': [ - (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|' - r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) - ], "blockcomment": [ (r'[^=#]', Comment.Multiline), (r'#=', Comment.Multiline, '#push'), (r'=#', Comment.Multiline, '#pop'), (r'[=#]', Comment.Multiline), ], + 'string': [ (r'"', String, '#pop'), - (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings + # FIXME: This escape pattern is not perfect. + (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape), # Interpolation is defined as "$" followed by the shortest full # expression, which is something we can't parse. # Include the most common cases here: $word, and $(paren'd expr). - (r'\$[a-zA-Z_]+', String.Interpol), - (r'\$\(', String.Interpol, 'in-intp'), + (r'\$' + allowed_variable, String.Interpol), + # (r'\$[a-zA-Z_]+', String.Interpol), + (r'(\$)(\()', bygroups(String.Interpol, Punctuation), 'in-intp'), # @printf and @sprintf formats - (r'%[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[diouxXeEfFgGcrs%]', + (r'%[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[E-GXc-giorsux%]', String.Interpol), - (r'[^$%"\\]+', String), - # unhandled special signs - (r'[$%"\\]', String), + (r'.|\s', String), ], + + 'tqstring': [ + (r'"""', String, '#pop'), + (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape), + (r'\$' + allowed_variable, String.Interpol), + (r'(\$)(\()', bygroups(String.Interpol, Punctuation), 'in-intp'), + (r'.|\s', String), + ], + + 'regex': [ + (r'"', String.Regex, '#pop'), + (r'\\"', String.Regex), + (r'.|\s', String.Regex), + ], + + 'tqregex': [ + (r'"""', String.Regex, '#pop'), + (r'.|\s', String.Regex), + ], + + 'command': [ + (r'`', String.Backtick, '#pop'), + (r'\$' + allowed_variable, String.Interpol), + (r'(\$)(\()', bygroups(String.Interpol, Punctuation), 'in-intp'), + (r'.|\s', String.Backtick) + ], + 'in-intp': [ - (r'[^()]+', String.Interpol), - (r'\(', String.Interpol, '#push'), - (r'\)', String.Interpol, '#pop'), + (r'\(', Punctuation, '#push'), + (r'\)', Punctuation, '#pop'), + include('root'), ] } @@ -155,9 +281,6 @@ class JuliaLexer(RegexLexer): return shebang_matches(text, r'julia') -line_re = re.compile('.*?\n') - - class JuliaConsoleLexer(Lexer): """ For Julia console sessions. Modeled after MatlabSessionLexer. @@ -169,27 +292,26 @@ class JuliaConsoleLexer(Lexer): def get_tokens_unprocessed(self, text): jllexer = JuliaLexer(**self.options) - + start = 0 curcode = '' insertions = [] + output = False + error = False - for match in line_re.finditer(text): - line = match.group() - + for line in text.splitlines(True): if line.startswith('julia>'): - insertions.append((len(curcode), - [(0, Generic.Prompt, line[:6])])) + insertions.append((len(curcode), [(0, Generic.Prompt, line[:6])])) + curcode += line[6:] + output = False + error = False + elif line.startswith('help?>') or line.startswith('shell>'): + yield start, Generic.Prompt, line[:6] + yield start + 6, Text, line[6:] + output = False + error = False + elif line.startswith(' ') and not output: + insertions.append((len(curcode), [(0, Text, line[:6])])) curcode += line[6:] - - elif line.startswith(' '): - - idx = len(curcode) - - # without is showing error on same line as before...? - line = "\n" + line - token = (0, Generic.Traceback, line) - insertions.append((idx, [token])) - else: if curcode: for item in do_insertions( @@ -197,10 +319,15 @@ class JuliaConsoleLexer(Lexer): yield item curcode = '' insertions = [] - - yield match.start(), Generic.Output, line - - if curcode: # or item: + if line.startswith('ERROR: ') or error: + yield start, Generic.Error, line + error = True + else: + yield start, Generic.Output, line + output = True + start += len(line) + + if curcode: for item in do_insertions( insertions, jllexer.get_tokens_unprocessed(curcode)): yield item diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index 41fc0fdb..f4392839 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -5,7 +5,7 @@ Pygments lexers for JVM languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -59,25 +59,27 @@ class JavaLexer(RegexLexer): Keyword.Type), (r'(package)(\s+)', bygroups(Keyword.Namespace, Text), 'import'), (r'(true|false|null)\b', Keyword.Constant), - (r'(class|interface)(\s+)', bygroups(Keyword.Declaration, Text), 'class'), - (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'), + (r'(class|interface)(\s+)', bygroups(Keyword.Declaration, Text), + 'class'), + (r'(import(?:\s+static)?)(\s+)', bygroups(Keyword.Namespace, Text), + 'import'), (r'"(\\\\|\\"|[^"])*"', String), (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char), (r'(\.)((?:[^\W\d]|\$)[\w$]*)', bygroups(Operator, Name.Attribute)), (r'^\s*([^\W\d]|\$)[\w$]*:', Name.Label), (r'([^\W\d]|\$)[\w$]*', Name), - (r'([0-9](_*[0-9]+)*\.([0-9](_*[0-9]+)*)?|' - r'([0-9](_*[0-9]+)*)?\.[0-9](_*[0-9]+)*)' - r'([eE][+\-]?[0-9](_*[0-9]+)*)?[fFdD]?|' - r'[0-9][eE][+\-]?[0-9](_*[0-9]+)*[fFdD]?|' - r'[0-9]([eE][+\-]?[0-9](_*[0-9]+)*)?[fFdD]|' - r'0[xX]([0-9a-fA-F](_*[0-9a-fA-F]+)*\.?|' - r'([0-9a-fA-F](_*[0-9a-fA-F]+)*)?\.[0-9a-fA-F](_*[0-9a-fA-F]+)*)' - r'[pP][+\-]?[0-9](_*[0-9]+)*[fFdD]?', Number.Float), - (r'0[xX][0-9a-fA-F](_*[0-9a-fA-F]+)*[lL]?', Number.Hex), - (r'0[bB][01](_*[01]+)*[lL]?', Number.Bin), - (r'0(_*[0-7]+)+[lL]?', Number.Oct), - (r'0|[1-9](_*[0-9]+)*[lL]?', Number.Integer), + (r'([0-9][0-9_]*\.([0-9][0-9_]*)?|' + r'\.[0-9][0-9_]*)' + r'([eE][+\-]?[0-9][0-9_]*)?[fFdD]?|' + r'[0-9][eE][+\-]?[0-9][0-9_]*[fFdD]?|' + r'[0-9]([eE][+\-]?[0-9][0-9_]*)?[fFdD]|' + r'0[xX]([0-9a-fA-F][0-9a-fA-F_]*\.?|' + r'([0-9a-fA-F][0-9a-fA-F_]*)?\.[0-9a-fA-F][0-9a-fA-F_]*)' + r'[pP][+\-]?[0-9][0-9_]*[fFdD]?', Number.Float), + (r'0[xX][0-9a-fA-F][0-9a-fA-F_]*[lL]?', Number.Hex), + (r'0[bB][01][01_]*[lL]?', Number.Bin), + (r'0[0-7_]+[lL]?', Number.Oct), + (r'0|[1-9][0-9_]*[lL]?', Number.Integer), (r'[~^*!%&\[\](){}<>|+=:;,./?-]', Operator), (r'\n', Text) ], @@ -564,14 +566,14 @@ class IokeLexer(RegexLexer): ], 'slashRegexp': [ - (r'(?<!\\)/[oxpniums]*', String.Regex, '#pop'), + (r'(?<!\\)/[im-psux]*', String.Regex, '#pop'), include('interpolatableText'), (r'\\/', String.Regex), (r'[^/]', String.Regex) ], 'squareRegexp': [ - (r'(?<!\\)][oxpniums]*', String.Regex, '#pop'), + (r'(?<!\\)][im-psux]*', String.Regex, '#pop'), include('interpolatableText'), (r'\\]', String.Regex), (r'[^\]]', String.Regex) diff --git a/pygments/lexers/lisp.py b/pygments/lexers/lisp.py index 84720fab..e258c347 100644 --- a/pygments/lexers/lisp.py +++ b/pygments/lexers/lisp.py @@ -5,7 +5,7 @@ Lexers for Lispy languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -18,7 +18,8 @@ from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ from pygments.lexers.python import PythonLexer __all__ = ['SchemeLexer', 'CommonLispLexer', 'HyLexer', 'RacketLexer', - 'NewLispLexer', 'EmacsLispLexer', 'ShenLexer', 'CPSALexer'] + 'NewLispLexer', 'EmacsLispLexer', 'ShenLexer', 'CPSALexer', + 'XtlangLexer'] class SchemeLexer(RegexLexer): @@ -470,731 +471,779 @@ class RacketLexer(RegexLexer): # Generated by example.rkt _keywords = ( - '#%app', '#%datum', '#%declare', '#%expression', '#%module-begin', - '#%plain-app', '#%plain-lambda', '#%plain-module-begin', - '#%printing-module-begin', '#%provide', '#%require', - '#%stratified-body', '#%top', '#%top-interaction', - '#%variable-reference', '->', '->*', '->*m', '->d', '->dm', '->i', - '->m', '...', ':do-in', '==', '=>', '_', 'absent', 'abstract', - 'all-defined-out', 'all-from-out', 'and', 'any', 'augment', 'augment*', - 'augment-final', 'augment-final*', 'augride', 'augride*', 'begin', - 'begin-for-syntax', 'begin0', 'case', 'case->', 'case->m', - 'case-lambda', 'class', 'class*', 'class-field-accessor', - 'class-field-mutator', 'class/c', 'class/derived', 'combine-in', - 'combine-out', 'command-line', 'compound-unit', 'compound-unit/infer', - 'cond', 'contract', 'contract-out', 'contract-struct', 'contracted', - 'define', 'define-compound-unit', 'define-compound-unit/infer', - 'define-contract-struct', 'define-custom-hash-types', - 'define-custom-set-types', 'define-for-syntax', - 'define-local-member-name', 'define-logger', 'define-match-expander', - 'define-member-name', 'define-module-boundary-contract', - 'define-namespace-anchor', 'define-opt/c', 'define-sequence-syntax', - 'define-serializable-class', 'define-serializable-class*', - 'define-signature', 'define-signature-form', 'define-struct', - 'define-struct/contract', 'define-struct/derived', 'define-syntax', - 'define-syntax-rule', 'define-syntaxes', 'define-unit', - 'define-unit-binding', 'define-unit-from-context', - 'define-unit/contract', 'define-unit/new-import-export', - 'define-unit/s', 'define-values', 'define-values-for-export', - 'define-values-for-syntax', 'define-values/invoke-unit', - 'define-values/invoke-unit/infer', 'define/augment', - 'define/augment-final', 'define/augride', 'define/contract', - 'define/final-prop', 'define/match', 'define/overment', - 'define/override', 'define/override-final', 'define/private', - 'define/public', 'define/public-final', 'define/pubment', - 'define/subexpression-pos-prop', 'delay', 'delay/idle', 'delay/name', - 'delay/strict', 'delay/sync', 'delay/thread', 'do', 'else', 'except', - 'except-in', 'except-out', 'export', 'extends', 'failure-cont', - 'false', 'false/c', 'field', 'field-bound?', 'file', - 'flat-murec-contract', 'flat-rec-contract', 'for', 'for*', 'for*/and', - 'for*/first', 'for*/fold', 'for*/fold/derived', 'for*/hash', - 'for*/hasheq', 'for*/hasheqv', 'for*/last', 'for*/list', 'for*/lists', - 'for*/mutable-set', 'for*/mutable-seteq', 'for*/mutable-seteqv', - 'for*/or', 'for*/product', 'for*/set', 'for*/seteq', 'for*/seteqv', - 'for*/sum', 'for*/vector', 'for*/weak-set', 'for*/weak-seteq', - 'for*/weak-seteqv', 'for-label', 'for-meta', 'for-syntax', - 'for-template', 'for/and', 'for/first', 'for/fold', 'for/fold/derived', - 'for/hash', 'for/hasheq', 'for/hasheqv', 'for/last', 'for/list', - 'for/lists', 'for/mutable-set', 'for/mutable-seteq', - 'for/mutable-seteqv', 'for/or', 'for/product', 'for/set', 'for/seteq', - 'for/seteqv', 'for/sum', 'for/vector', 'for/weak-set', - 'for/weak-seteq', 'for/weak-seteqv', 'gen:custom-write', 'gen:dict', - 'gen:equal+hash', 'gen:set', 'gen:stream', 'generic', 'get-field', - 'if', 'implies', 'import', 'include', 'include-at/relative-to', - 'include-at/relative-to/reader', 'include/reader', 'inherit', - 'inherit-field', 'inherit/inner', 'inherit/super', 'init', - 'init-depend', 'init-field', 'init-rest', 'inner', 'inspect', - 'instantiate', 'interface', 'interface*', 'invoke-unit', - 'invoke-unit/infer', 'lambda', 'lazy', 'let', 'let*', 'let*-values', - 'let-syntax', 'let-syntaxes', 'let-values', 'let/cc', 'let/ec', - 'letrec', 'letrec-syntax', 'letrec-syntaxes', 'letrec-syntaxes+values', - 'letrec-values', 'lib', 'link', 'local', 'local-require', 'log-debug', - 'log-error', 'log-fatal', 'log-info', 'log-warning', 'match', 'match*', - 'match*/derived', 'match-define', 'match-define-values', - 'match-lambda', 'match-lambda*', 'match-lambda**', 'match-let', - 'match-let*', 'match-let*-values', 'match-let-values', 'match-letrec', - 'match/derived', 'match/values', 'member-name-key', 'method-contract?', - 'mixin', 'module', 'module*', 'module+', 'nand', 'new', 'nor', - 'object-contract', 'object/c', 'only', 'only-in', 'only-meta-in', - 'open', 'opt/c', 'or', 'overment', 'overment*', 'override', - 'override*', 'override-final', 'override-final*', 'parameterize', - 'parameterize*', 'parameterize-break', 'parametric->/c', 'place', - 'place*', 'planet', 'prefix', 'prefix-in', 'prefix-out', 'private', - 'private*', 'prompt-tag/c', 'protect-out', 'provide', - 'provide-signature-elements', 'provide/contract', 'public', 'public*', - 'public-final', 'public-final*', 'pubment', 'pubment*', 'quasiquote', - 'quasisyntax', 'quasisyntax/loc', 'quote', 'quote-syntax', - 'quote-syntax/prune', 'recontract-out', 'recursive-contract', - 'relative-in', 'rename', 'rename-in', 'rename-inner', 'rename-out', - 'rename-super', 'require', 'send', 'send*', 'send+', 'send-generic', - 'send/apply', 'send/keyword-apply', 'set!', 'set!-values', - 'set-field!', 'shared', 'stream', 'stream-cons', 'struct', 'struct*', - 'struct-copy', 'struct-field-index', 'struct-out', 'struct/c', - 'struct/ctc', 'struct/dc', 'submod', 'super', 'super-instantiate', - 'super-make-object', 'super-new', 'syntax', 'syntax-case', - 'syntax-case*', 'syntax-id-rules', 'syntax-rules', 'syntax/loc', 'tag', - 'this', 'this%', 'thunk', 'thunk*', 'time', 'unconstrained-domain->', - 'unit', 'unit-from-context', 'unit/c', 'unit/new-import-export', - 'unit/s', 'unless', 'unquote', 'unquote-splicing', 'unsyntax', - 'unsyntax-splicing', 'values/drop', 'when', 'with-continuation-mark', - 'with-contract', 'with-handlers', 'with-handlers*', 'with-method', - 'with-syntax', u'λ' + u'#%app', u'#%datum', u'#%declare', u'#%expression', u'#%module-begin', + u'#%plain-app', u'#%plain-lambda', u'#%plain-module-begin', + u'#%printing-module-begin', u'#%provide', u'#%require', + u'#%stratified-body', u'#%top', u'#%top-interaction', + u'#%variable-reference', u'->', u'->*', u'->*m', u'->d', u'->dm', u'->i', + u'->m', u'...', u':do-in', u'==', u'=>', u'_', u'absent', u'abstract', + u'all-defined-out', u'all-from-out', u'and', u'any', u'augment', u'augment*', + u'augment-final', u'augment-final*', u'augride', u'augride*', u'begin', + u'begin-for-syntax', u'begin0', u'case', u'case->', u'case->m', + u'case-lambda', u'class', u'class*', u'class-field-accessor', + u'class-field-mutator', u'class/c', u'class/derived', u'combine-in', + u'combine-out', u'command-line', u'compound-unit', u'compound-unit/infer', + u'cond', u'cons/dc', u'contract', u'contract-out', u'contract-struct', + u'contracted', u'define', u'define-compound-unit', + u'define-compound-unit/infer', u'define-contract-struct', + u'define-custom-hash-types', u'define-custom-set-types', + u'define-for-syntax', u'define-local-member-name', u'define-logger', + u'define-match-expander', u'define-member-name', + u'define-module-boundary-contract', u'define-namespace-anchor', + u'define-opt/c', u'define-sequence-syntax', u'define-serializable-class', + u'define-serializable-class*', u'define-signature', + u'define-signature-form', u'define-struct', u'define-struct/contract', + u'define-struct/derived', u'define-syntax', u'define-syntax-rule', + u'define-syntaxes', u'define-unit', u'define-unit-binding', + u'define-unit-from-context', u'define-unit/contract', + u'define-unit/new-import-export', u'define-unit/s', u'define-values', + u'define-values-for-export', u'define-values-for-syntax', + u'define-values/invoke-unit', u'define-values/invoke-unit/infer', + u'define/augment', u'define/augment-final', u'define/augride', + u'define/contract', u'define/final-prop', u'define/match', + u'define/overment', u'define/override', u'define/override-final', + u'define/private', u'define/public', u'define/public-final', + u'define/pubment', u'define/subexpression-pos-prop', + u'define/subexpression-pos-prop/name', u'delay', u'delay/idle', + u'delay/name', u'delay/strict', u'delay/sync', u'delay/thread', u'do', + u'else', u'except', u'except-in', u'except-out', u'export', u'extends', + u'failure-cont', u'false', u'false/c', u'field', u'field-bound?', u'file', + u'flat-murec-contract', u'flat-rec-contract', u'for', u'for*', u'for*/and', + u'for*/async', u'for*/first', u'for*/fold', u'for*/fold/derived', + u'for*/hash', u'for*/hasheq', u'for*/hasheqv', u'for*/last', u'for*/list', + u'for*/lists', u'for*/mutable-set', u'for*/mutable-seteq', + u'for*/mutable-seteqv', u'for*/or', u'for*/product', u'for*/set', + u'for*/seteq', u'for*/seteqv', u'for*/stream', u'for*/sum', u'for*/vector', + u'for*/weak-set', u'for*/weak-seteq', u'for*/weak-seteqv', u'for-label', + u'for-meta', u'for-syntax', u'for-template', u'for/and', u'for/async', + u'for/first', u'for/fold', u'for/fold/derived', u'for/hash', u'for/hasheq', + u'for/hasheqv', u'for/last', u'for/list', u'for/lists', u'for/mutable-set', + u'for/mutable-seteq', u'for/mutable-seteqv', u'for/or', u'for/product', + u'for/set', u'for/seteq', u'for/seteqv', u'for/stream', u'for/sum', + u'for/vector', u'for/weak-set', u'for/weak-seteq', u'for/weak-seteqv', + u'gen:custom-write', u'gen:dict', u'gen:equal+hash', u'gen:set', + u'gen:stream', u'generic', u'get-field', u'hash/dc', u'if', u'implies', + u'import', u'include', u'include-at/relative-to', + u'include-at/relative-to/reader', u'include/reader', u'inherit', + u'inherit-field', u'inherit/inner', u'inherit/super', u'init', + u'init-depend', u'init-field', u'init-rest', u'inner', u'inspect', + u'instantiate', u'interface', u'interface*', u'invariant-assertion', + u'invoke-unit', u'invoke-unit/infer', u'lambda', u'lazy', u'let', u'let*', + u'let*-values', u'let-syntax', u'let-syntaxes', u'let-values', u'let/cc', + u'let/ec', u'letrec', u'letrec-syntax', u'letrec-syntaxes', + u'letrec-syntaxes+values', u'letrec-values', u'lib', u'link', u'local', + u'local-require', u'log-debug', u'log-error', u'log-fatal', u'log-info', + u'log-warning', u'match', u'match*', u'match*/derived', u'match-define', + u'match-define-values', u'match-lambda', u'match-lambda*', + u'match-lambda**', u'match-let', u'match-let*', u'match-let*-values', + u'match-let-values', u'match-letrec', u'match-letrec-values', + u'match/derived', u'match/values', u'member-name-key', u'mixin', u'module', + u'module*', u'module+', u'nand', u'new', u'nor', u'object-contract', + u'object/c', u'only', u'only-in', u'only-meta-in', u'open', u'opt/c', u'or', + u'overment', u'overment*', u'override', u'override*', u'override-final', + u'override-final*', u'parameterize', u'parameterize*', + u'parameterize-break', u'parametric->/c', u'place', u'place*', + u'place/context', u'planet', u'prefix', u'prefix-in', u'prefix-out', + u'private', u'private*', u'prompt-tag/c', u'protect-out', u'provide', + u'provide-signature-elements', u'provide/contract', u'public', u'public*', + u'public-final', u'public-final*', u'pubment', u'pubment*', u'quasiquote', + u'quasisyntax', u'quasisyntax/loc', u'quote', u'quote-syntax', + u'quote-syntax/prune', u'recontract-out', u'recursive-contract', + u'relative-in', u'rename', u'rename-in', u'rename-inner', u'rename-out', + u'rename-super', u'require', u'send', u'send*', u'send+', u'send-generic', + u'send/apply', u'send/keyword-apply', u'set!', u'set!-values', + u'set-field!', u'shared', u'stream', u'stream*', u'stream-cons', u'struct', + u'struct*', u'struct-copy', u'struct-field-index', u'struct-out', + u'struct/c', u'struct/ctc', u'struct/dc', u'submod', u'super', + u'super-instantiate', u'super-make-object', u'super-new', u'syntax', + u'syntax-case', u'syntax-case*', u'syntax-id-rules', u'syntax-rules', + u'syntax/loc', u'tag', u'this', u'this%', u'thunk', u'thunk*', u'time', + u'unconstrained-domain->', u'unit', u'unit-from-context', u'unit/c', + u'unit/new-import-export', u'unit/s', u'unless', u'unquote', + u'unquote-splicing', u'unsyntax', u'unsyntax-splicing', u'values/drop', + u'when', u'with-continuation-mark', u'with-contract', + u'with-contract-continuation-mark', u'with-handlers', u'with-handlers*', + u'with-method', u'with-syntax', u'λ' ) # Generated by example.rkt _builtins = ( - '*', '+', '-', '/', '<', '</c', '<=', '<=/c', '=', '=/c', '>', '>/c', - '>=', '>=/c', 'abort-current-continuation', 'abs', 'absolute-path?', - 'acos', 'add-between', 'add1', 'alarm-evt', 'always-evt', 'and/c', - 'andmap', 'angle', 'any/c', 'append', 'append*', 'append-map', 'apply', - 'argmax', 'argmin', 'arithmetic-shift', 'arity-at-least', - 'arity-at-least-value', 'arity-at-least?', 'arity-checking-wrapper', - 'arity-includes?', 'arity=?', 'asin', 'assf', 'assoc', 'assq', 'assv', - 'atan', 'bad-number-of-results', 'banner', 'base->-doms/c', - 'base->-rngs/c', 'base->?', 'between/c', 'bitwise-and', - 'bitwise-bit-field', 'bitwise-bit-set?', 'bitwise-ior', 'bitwise-not', - 'bitwise-xor', 'blame-add-car-context', 'blame-add-cdr-context', - 'blame-add-context', 'blame-add-missing-party', - 'blame-add-nth-arg-context', 'blame-add-or-context', - 'blame-add-range-context', 'blame-add-unknown-context', - 'blame-context', 'blame-contract', 'blame-fmt->-string', - 'blame-negative', 'blame-original?', 'blame-positive', - 'blame-replace-negative', 'blame-source', 'blame-swap', - 'blame-swapped?', 'blame-update', 'blame-value', 'blame?', 'boolean=?', - 'boolean?', 'bound-identifier=?', 'box', 'box-cas!', 'box-immutable', - 'box-immutable/c', 'box/c', 'box?', 'break-enabled', 'break-thread', - 'build-chaperone-contract-property', 'build-compound-type-name', - 'build-contract-property', 'build-flat-contract-property', - 'build-list', 'build-path', 'build-path/convention-type', - 'build-string', 'build-vector', 'byte-pregexp', 'byte-pregexp?', - 'byte-ready?', 'byte-regexp', 'byte-regexp?', 'byte?', 'bytes', - 'bytes->immutable-bytes', 'bytes->list', 'bytes->path', - 'bytes->path-element', 'bytes->string/latin-1', 'bytes->string/locale', - 'bytes->string/utf-8', 'bytes-append', 'bytes-append*', - 'bytes-close-converter', 'bytes-convert', 'bytes-convert-end', - 'bytes-converter?', 'bytes-copy', 'bytes-copy!', - 'bytes-environment-variable-name?', 'bytes-fill!', 'bytes-join', - 'bytes-length', 'bytes-no-nuls?', 'bytes-open-converter', 'bytes-ref', - 'bytes-set!', 'bytes-utf-8-index', 'bytes-utf-8-length', - 'bytes-utf-8-ref', 'bytes<?', 'bytes=?', 'bytes>?', 'bytes?', 'caaaar', - 'caaadr', 'caaar', 'caadar', 'caaddr', 'caadr', 'caar', 'cadaar', - 'cadadr', 'cadar', 'caddar', 'cadddr', 'caddr', 'cadr', - 'call-in-nested-thread', 'call-with-atomic-output-file', - 'call-with-break-parameterization', - 'call-with-composable-continuation', 'call-with-continuation-barrier', - 'call-with-continuation-prompt', 'call-with-current-continuation', - 'call-with-default-reading-parameterization', - 'call-with-escape-continuation', 'call-with-exception-handler', - 'call-with-file-lock/timeout', 'call-with-immediate-continuation-mark', - 'call-with-input-bytes', 'call-with-input-file', - 'call-with-input-file*', 'call-with-input-string', - 'call-with-output-bytes', 'call-with-output-file', - 'call-with-output-file*', 'call-with-output-string', - 'call-with-parameterization', 'call-with-semaphore', - 'call-with-semaphore/enable-break', 'call-with-values', 'call/cc', - 'call/ec', 'car', 'cdaaar', 'cdaadr', 'cdaar', 'cdadar', 'cdaddr', - 'cdadr', 'cdar', 'cddaar', 'cddadr', 'cddar', 'cdddar', 'cddddr', - 'cdddr', 'cddr', 'cdr', 'ceiling', 'channel-get', 'channel-put', - 'channel-put-evt', 'channel-put-evt?', 'channel-try-get', 'channel/c', - 'channel?', 'chaperone-box', 'chaperone-channel', - 'chaperone-continuation-mark-key', 'chaperone-contract-property?', - 'chaperone-contract?', 'chaperone-evt', 'chaperone-hash', - 'chaperone-of?', 'chaperone-procedure', 'chaperone-prompt-tag', - 'chaperone-struct', 'chaperone-struct-type', 'chaperone-vector', - 'chaperone?', 'char->integer', 'char-alphabetic?', 'char-blank?', - 'char-ci<=?', 'char-ci<?', 'char-ci=?', 'char-ci>=?', 'char-ci>?', - 'char-downcase', 'char-foldcase', 'char-general-category', - 'char-graphic?', 'char-iso-control?', 'char-lower-case?', - 'char-numeric?', 'char-punctuation?', 'char-ready?', 'char-symbolic?', - 'char-title-case?', 'char-titlecase', 'char-upcase', - 'char-upper-case?', 'char-utf-8-length', 'char-whitespace?', 'char<=?', - 'char<?', 'char=?', 'char>=?', 'char>?', 'char?', - 'check-duplicate-identifier', 'checked-procedure-check-and-extract', - 'choice-evt', 'class->interface', 'class-info', 'class?', - 'cleanse-path', 'close-input-port', 'close-output-port', - 'coerce-chaperone-contract', 'coerce-chaperone-contracts', - 'coerce-contract', 'coerce-contract/f', 'coerce-contracts', - 'coerce-flat-contract', 'coerce-flat-contracts', 'collect-garbage', - 'collection-file-path', 'collection-path', 'compile', - 'compile-allow-set!-undefined', 'compile-context-preservation-enabled', - 'compile-enforce-module-constants', 'compile-syntax', - 'compiled-expression?', 'compiled-module-expression?', - 'complete-path?', 'complex?', 'compose', 'compose1', 'conjugate', - 'cons', 'cons/c', 'cons?', 'const', 'continuation-mark-key/c', - 'continuation-mark-key?', 'continuation-mark-set->context', - 'continuation-mark-set->list', 'continuation-mark-set->list*', - 'continuation-mark-set-first', 'continuation-mark-set?', - 'continuation-marks', 'continuation-prompt-available?', - 'continuation-prompt-tag?', 'continuation?', - 'contract-continuation-mark-key', 'contract-first-order', - 'contract-first-order-passes?', 'contract-name', 'contract-proc', - 'contract-projection', 'contract-property?', - 'contract-random-generate', 'contract-stronger?', - 'contract-struct-exercise', 'contract-struct-generate', - 'contract-val-first-projection', 'contract?', 'convert-stream', - 'copy-directory/files', 'copy-file', 'copy-port', 'cos', 'cosh', - 'count', 'current-blame-format', 'current-break-parameterization', - 'current-code-inspector', 'current-command-line-arguments', - 'current-compile', 'current-compiled-file-roots', - 'current-continuation-marks', 'current-contract-region', - 'current-custodian', 'current-directory', 'current-directory-for-user', - 'current-drive', 'current-environment-variables', 'current-error-port', - 'current-eval', 'current-evt-pseudo-random-generator', - 'current-future', 'current-gc-milliseconds', - 'current-get-interaction-input-port', 'current-inexact-milliseconds', - 'current-input-port', 'current-inspector', - 'current-library-collection-links', 'current-library-collection-paths', - 'current-load', 'current-load-extension', - 'current-load-relative-directory', 'current-load/use-compiled', - 'current-locale', 'current-logger', 'current-memory-use', - 'current-milliseconds', 'current-module-declare-name', - 'current-module-declare-source', 'current-module-name-resolver', - 'current-module-path-for-load', 'current-namespace', - 'current-output-port', 'current-parameterization', - 'current-preserved-thread-cell-values', 'current-print', - 'current-process-milliseconds', 'current-prompt-read', - 'current-pseudo-random-generator', 'current-read-interaction', - 'current-reader-guard', 'current-readtable', 'current-seconds', - 'current-security-guard', 'current-subprocess-custodian-mode', - 'current-thread', 'current-thread-group', - 'current-thread-initial-stack-size', - 'current-write-relative-directory', 'curry', 'curryr', - 'custodian-box-value', 'custodian-box?', 'custodian-limit-memory', - 'custodian-managed-list', 'custodian-memory-accounting-available?', - 'custodian-require-memory', 'custodian-shutdown-all', 'custodian?', - 'custom-print-quotable-accessor', 'custom-print-quotable?', - 'custom-write-accessor', 'custom-write-property-proc', 'custom-write?', - 'date', 'date*', 'date*-nanosecond', 'date*-time-zone-name', 'date*?', - 'date-day', 'date-dst?', 'date-hour', 'date-minute', 'date-month', - 'date-second', 'date-time-zone-offset', 'date-week-day', 'date-year', - 'date-year-day', 'date?', 'datum->syntax', 'datum-intern-literal', - 'default-continuation-prompt-tag', 'degrees->radians', - 'delete-directory', 'delete-directory/files', 'delete-file', - 'denominator', 'dict->list', 'dict-can-functional-set?', - 'dict-can-remove-keys?', 'dict-clear', 'dict-clear!', 'dict-copy', - 'dict-count', 'dict-empty?', 'dict-for-each', 'dict-has-key?', - 'dict-implements/c', 'dict-implements?', 'dict-iter-contract', - 'dict-iterate-first', 'dict-iterate-key', 'dict-iterate-next', - 'dict-iterate-value', 'dict-key-contract', 'dict-keys', 'dict-map', - 'dict-mutable?', 'dict-ref', 'dict-ref!', 'dict-remove', - 'dict-remove!', 'dict-set', 'dict-set!', 'dict-set*', 'dict-set*!', - 'dict-update', 'dict-update!', 'dict-value-contract', 'dict-values', - 'dict?', 'directory-exists?', 'directory-list', 'display', - 'display-lines', 'display-lines-to-file', 'display-to-file', - 'displayln', 'double-flonum?', 'drop', 'drop-right', 'dropf', - 'dropf-right', 'dump-memory-stats', 'dup-input-port', - 'dup-output-port', 'dynamic-get-field', 'dynamic-place', - 'dynamic-place*', 'dynamic-require', 'dynamic-require-for-syntax', - 'dynamic-send', 'dynamic-set-field!', 'dynamic-wind', 'eighth', - 'empty', 'empty-sequence', 'empty-stream', 'empty?', - 'environment-variables-copy', 'environment-variables-names', - 'environment-variables-ref', 'environment-variables-set!', - 'environment-variables?', 'eof', 'eof-evt', 'eof-object?', - 'ephemeron-value', 'ephemeron?', 'eprintf', 'eq-contract-val', - 'eq-contract?', 'eq-hash-code', 'eq?', 'equal-contract-val', - 'equal-contract?', 'equal-hash-code', 'equal-secondary-hash-code', - 'equal<%>', 'equal?', 'equal?/recur', 'eqv-hash-code', 'eqv?', 'error', - 'error-display-handler', 'error-escape-handler', - 'error-print-context-length', 'error-print-source-location', - 'error-print-width', 'error-value->string-handler', 'eval', - 'eval-jit-enabled', 'eval-syntax', 'even?', 'evt/c', 'evt?', - 'exact->inexact', 'exact-ceiling', 'exact-floor', 'exact-integer?', - 'exact-nonnegative-integer?', 'exact-positive-integer?', 'exact-round', - 'exact-truncate', 'exact?', 'executable-yield-handler', 'exit', - 'exit-handler', 'exn', 'exn-continuation-marks', 'exn-message', - 'exn:break', 'exn:break-continuation', 'exn:break:hang-up', - 'exn:break:hang-up?', 'exn:break:terminate', 'exn:break:terminate?', - 'exn:break?', 'exn:fail', 'exn:fail:contract', - 'exn:fail:contract:arity', 'exn:fail:contract:arity?', - 'exn:fail:contract:blame', 'exn:fail:contract:blame-object', - 'exn:fail:contract:blame?', 'exn:fail:contract:continuation', - 'exn:fail:contract:continuation?', 'exn:fail:contract:divide-by-zero', - 'exn:fail:contract:divide-by-zero?', - 'exn:fail:contract:non-fixnum-result', - 'exn:fail:contract:non-fixnum-result?', 'exn:fail:contract:variable', - 'exn:fail:contract:variable-id', 'exn:fail:contract:variable?', - 'exn:fail:contract?', 'exn:fail:filesystem', - 'exn:fail:filesystem:errno', 'exn:fail:filesystem:errno-errno', - 'exn:fail:filesystem:errno?', 'exn:fail:filesystem:exists', - 'exn:fail:filesystem:exists?', 'exn:fail:filesystem:missing-module', - 'exn:fail:filesystem:missing-module-path', - 'exn:fail:filesystem:missing-module?', 'exn:fail:filesystem:version', - 'exn:fail:filesystem:version?', 'exn:fail:filesystem?', - 'exn:fail:network', 'exn:fail:network:errno', - 'exn:fail:network:errno-errno', 'exn:fail:network:errno?', - 'exn:fail:network?', 'exn:fail:object', 'exn:fail:object?', - 'exn:fail:out-of-memory', 'exn:fail:out-of-memory?', 'exn:fail:read', - 'exn:fail:read-srclocs', 'exn:fail:read:eof', 'exn:fail:read:eof?', - 'exn:fail:read:non-char', 'exn:fail:read:non-char?', 'exn:fail:read?', - 'exn:fail:syntax', 'exn:fail:syntax-exprs', - 'exn:fail:syntax:missing-module', - 'exn:fail:syntax:missing-module-path', - 'exn:fail:syntax:missing-module?', 'exn:fail:syntax:unbound', - 'exn:fail:syntax:unbound?', 'exn:fail:syntax?', 'exn:fail:unsupported', - 'exn:fail:unsupported?', 'exn:fail:user', 'exn:fail:user?', - 'exn:fail?', 'exn:misc:match?', 'exn:missing-module-accessor', - 'exn:missing-module?', 'exn:srclocs-accessor', 'exn:srclocs?', 'exn?', - 'exp', 'expand', 'expand-once', 'expand-syntax', 'expand-syntax-once', - 'expand-syntax-to-top-form', 'expand-to-top-form', 'expand-user-path', - 'explode-path', 'expt', 'externalizable<%>', 'false?', 'field-names', - 'fifth', 'file->bytes', 'file->bytes-lines', 'file->lines', - 'file->list', 'file->string', 'file->value', 'file-exists?', - 'file-name-from-path', 'file-or-directory-identity', - 'file-or-directory-modify-seconds', 'file-or-directory-permissions', - 'file-position', 'file-position*', 'file-size', - 'file-stream-buffer-mode', 'file-stream-port?', 'file-truncate', - 'filename-extension', 'filesystem-change-evt', - 'filesystem-change-evt-cancel', 'filesystem-change-evt?', - 'filesystem-root-list', 'filter', 'filter-map', 'filter-not', - 'filter-read-input-port', 'find-executable-path', 'find-files', - 'find-library-collection-links', 'find-library-collection-paths', - 'find-relative-path', 'find-system-path', 'findf', 'first', 'fixnum?', - 'flat-contract', 'flat-contract-predicate', 'flat-contract-property?', - 'flat-contract?', 'flat-named-contract', 'flatten', - 'floating-point-bytes->real', 'flonum?', 'floor', 'flush-output', - 'fold-files', 'foldl', 'foldr', 'for-each', 'force', 'format', - 'fourth', 'fprintf', 'free-identifier=?', 'free-label-identifier=?', - 'free-template-identifier=?', 'free-transformer-identifier=?', - 'fsemaphore-count', 'fsemaphore-post', 'fsemaphore-try-wait?', - 'fsemaphore-wait', 'fsemaphore?', 'future', 'future?', - 'futures-enabled?', 'gcd', 'generate-member-key', - 'generate-temporaries', 'generic-set?', 'generic?', 'gensym', - 'get-output-bytes', 'get-output-string', 'get-preference', - 'get/build-val-first-projection', 'getenv', - 'global-port-print-handler', 'group-execute-bit', 'group-read-bit', - 'group-write-bit', 'guard-evt', 'handle-evt', 'handle-evt?', - 'has-contract?', 'hash', 'hash->list', 'hash-clear', 'hash-clear!', - 'hash-copy', 'hash-copy-clear', 'hash-count', 'hash-empty?', - 'hash-eq?', 'hash-equal?', 'hash-eqv?', 'hash-for-each', - 'hash-has-key?', 'hash-iterate-first', 'hash-iterate-key', - 'hash-iterate-next', 'hash-iterate-value', 'hash-keys', 'hash-map', - 'hash-placeholder?', 'hash-ref', 'hash-ref!', 'hash-remove', - 'hash-remove!', 'hash-set', 'hash-set!', 'hash-set*', 'hash-set*!', - 'hash-update', 'hash-update!', 'hash-values', 'hash-weak?', 'hash/c', - 'hash?', 'hasheq', 'hasheqv', 'identifier-binding', - 'identifier-binding-symbol', 'identifier-label-binding', - 'identifier-prune-lexical-context', - 'identifier-prune-to-source-module', - 'identifier-remove-from-definition-context', - 'identifier-template-binding', 'identifier-transformer-binding', - 'identifier?', 'identity', 'imag-part', 'immutable?', - 'impersonate-box', 'impersonate-channel', - 'impersonate-continuation-mark-key', 'impersonate-hash', - 'impersonate-procedure', 'impersonate-prompt-tag', - 'impersonate-struct', 'impersonate-vector', 'impersonator-contract?', - 'impersonator-ephemeron', 'impersonator-of?', - 'impersonator-prop:application-mark', 'impersonator-prop:contracted', - 'impersonator-property-accessor-procedure?', 'impersonator-property?', - 'impersonator?', 'implementation?', 'implementation?/c', 'in-bytes', - 'in-bytes-lines', 'in-cycle', 'in-dict', 'in-dict-keys', - 'in-dict-pairs', 'in-dict-values', 'in-directory', 'in-hash', - 'in-hash-keys', 'in-hash-pairs', 'in-hash-values', 'in-indexed', - 'in-input-port-bytes', 'in-input-port-chars', 'in-lines', 'in-list', - 'in-mlist', 'in-naturals', 'in-parallel', 'in-permutations', 'in-port', - 'in-producer', 'in-range', 'in-sequences', 'in-set', 'in-stream', - 'in-string', 'in-value', 'in-values*-sequence', 'in-values-sequence', - 'in-vector', 'inexact->exact', 'inexact-real?', 'inexact?', - 'infinite?', 'input-port-append', 'input-port?', 'inspector?', - 'instanceof/c', 'integer->char', 'integer->integer-bytes', - 'integer-bytes->integer', 'integer-in', 'integer-length', - 'integer-sqrt', 'integer-sqrt/remainder', 'integer?', - 'interface->method-names', 'interface-extension?', 'interface?', - 'internal-definition-context-seal', 'internal-definition-context?', - 'is-a?', 'is-a?/c', 'keyword->string', 'keyword-apply', 'keyword<?', - 'keyword?', 'keywords-match', 'kill-thread', 'last', 'last-pair', - 'lcm', 'length', 'liberal-define-context?', 'link-exists?', 'list', - 'list*', 'list->bytes', 'list->mutable-set', 'list->mutable-seteq', - 'list->mutable-seteqv', 'list->set', 'list->seteq', 'list->seteqv', - 'list->string', 'list->vector', 'list->weak-set', 'list->weak-seteq', - 'list->weak-seteqv', 'list-ref', 'list-tail', 'list/c', 'list?', - 'listof', 'load', 'load-extension', 'load-on-demand-enabled', - 'load-relative', 'load-relative-extension', 'load/cd', - 'load/use-compiled', 'local-expand', 'local-expand/capture-lifts', - 'local-transformer-expand', 'local-transformer-expand/capture-lifts', - 'locale-string-encoding', 'log', 'log-level?', 'log-max-level', - 'log-message', 'log-receiver?', 'logger-name', 'logger?', 'magnitude', - 'make-arity-at-least', 'make-base-empty-namespace', - 'make-base-namespace', 'make-bytes', 'make-channel', - 'make-chaperone-contract', 'make-continuation-mark-key', - 'make-continuation-prompt-tag', 'make-contract', 'make-custodian', - 'make-custodian-box', 'make-custom-hash', 'make-custom-hash-types', - 'make-custom-set', 'make-custom-set-types', 'make-date', 'make-date*', - 'make-derived-parameter', 'make-directory', 'make-directory*', - 'make-do-sequence', 'make-empty-namespace', - 'make-environment-variables', 'make-ephemeron', 'make-exn', - 'make-exn:break', 'make-exn:break:hang-up', 'make-exn:break:terminate', - 'make-exn:fail', 'make-exn:fail:contract', - 'make-exn:fail:contract:arity', 'make-exn:fail:contract:blame', - 'make-exn:fail:contract:continuation', - 'make-exn:fail:contract:divide-by-zero', - 'make-exn:fail:contract:non-fixnum-result', - 'make-exn:fail:contract:variable', 'make-exn:fail:filesystem', - 'make-exn:fail:filesystem:errno', 'make-exn:fail:filesystem:exists', - 'make-exn:fail:filesystem:missing-module', - 'make-exn:fail:filesystem:version', 'make-exn:fail:network', - 'make-exn:fail:network:errno', 'make-exn:fail:object', - 'make-exn:fail:out-of-memory', 'make-exn:fail:read', - 'make-exn:fail:read:eof', 'make-exn:fail:read:non-char', - 'make-exn:fail:syntax', 'make-exn:fail:syntax:missing-module', - 'make-exn:fail:syntax:unbound', 'make-exn:fail:unsupported', - 'make-exn:fail:user', 'make-file-or-directory-link', - 'make-flat-contract', 'make-fsemaphore', 'make-generic', - 'make-handle-get-preference-locked', 'make-hash', - 'make-hash-placeholder', 'make-hasheq', 'make-hasheq-placeholder', - 'make-hasheqv', 'make-hasheqv-placeholder', - 'make-immutable-custom-hash', 'make-immutable-hash', - 'make-immutable-hasheq', 'make-immutable-hasheqv', - 'make-impersonator-property', 'make-input-port', - 'make-input-port/read-to-peek', 'make-inspector', - 'make-keyword-procedure', 'make-known-char-range-list', - 'make-limited-input-port', 'make-list', 'make-lock-file-name', - 'make-log-receiver', 'make-logger', 'make-mixin-contract', - 'make-mutable-custom-set', 'make-none/c', 'make-object', - 'make-output-port', 'make-parameter', 'make-phantom-bytes', - 'make-pipe', 'make-pipe-with-specials', 'make-placeholder', - 'make-polar', 'make-prefab-struct', 'make-primitive-class', - 'make-proj-contract', 'make-pseudo-random-generator', - 'make-reader-graph', 'make-readtable', 'make-rectangular', - 'make-rename-transformer', 'make-resolved-module-path', - 'make-security-guard', 'make-semaphore', 'make-set!-transformer', - 'make-shared-bytes', 'make-sibling-inspector', 'make-special-comment', - 'make-srcloc', 'make-string', 'make-struct-field-accessor', - 'make-struct-field-mutator', 'make-struct-type', - 'make-struct-type-property', 'make-syntax-delta-introducer', - 'make-syntax-introducer', 'make-temporary-file', - 'make-tentative-pretty-print-output-port', 'make-thread-cell', - 'make-thread-group', 'make-vector', 'make-weak-box', - 'make-weak-custom-hash', 'make-weak-custom-set', 'make-weak-hash', - 'make-weak-hasheq', 'make-weak-hasheqv', 'make-will-executor', 'map', - 'match-equality-test', 'matches-arity-exactly?', 'max', 'mcar', 'mcdr', - 'mcons', 'member', 'member-name-key-hash-code', 'member-name-key=?', - 'member-name-key?', 'memf', 'memq', 'memv', 'merge-input', - 'method-in-interface?', 'min', 'mixin-contract', 'module->exports', - 'module->imports', 'module->language-info', 'module->namespace', - 'module-compiled-cross-phase-persistent?', 'module-compiled-exports', - 'module-compiled-imports', 'module-compiled-language-info', - 'module-compiled-name', 'module-compiled-submodules', - 'module-declared?', 'module-path-index-join', - 'module-path-index-resolve', 'module-path-index-split', - 'module-path-index-submodule', 'module-path-index?', 'module-path?', - 'module-predefined?', 'module-provide-protected?', 'modulo', 'mpair?', - 'mutable-set', 'mutable-seteq', 'mutable-seteqv', 'n->th', - 'nack-guard-evt', 'namespace-anchor->empty-namespace', - 'namespace-anchor->namespace', 'namespace-anchor?', - 'namespace-attach-module', 'namespace-attach-module-declaration', - 'namespace-base-phase', 'namespace-mapped-symbols', - 'namespace-module-identifier', 'namespace-module-registry', - 'namespace-require', 'namespace-require/constant', - 'namespace-require/copy', 'namespace-require/expansion-time', - 'namespace-set-variable-value!', 'namespace-symbol->identifier', - 'namespace-syntax-introduce', 'namespace-undefine-variable!', - 'namespace-unprotect-module', 'namespace-variable-value', 'namespace?', - 'nan?', 'natural-number/c', 'negate', 'negative?', 'never-evt', - u'new-∀/c', u'new-∃/c', 'newline', 'ninth', 'non-empty-listof', - 'none/c', 'normal-case-path', 'normalize-arity', 'normalize-path', - 'normalized-arity?', 'not', 'not/c', 'null', 'null?', 'number->string', - 'number?', 'numerator', 'object%', 'object->vector', 'object-info', - 'object-interface', 'object-method-arity-includes?', 'object-name', - 'object=?', 'object?', 'odd?', 'one-of/c', 'open-input-bytes', - 'open-input-file', 'open-input-output-file', 'open-input-string', - 'open-output-bytes', 'open-output-file', 'open-output-nowhere', - 'open-output-string', 'or/c', 'order-of-magnitude', 'ormap', - 'other-execute-bit', 'other-read-bit', 'other-write-bit', - 'output-port?', 'pair?', 'parameter-procedure=?', 'parameter/c', - 'parameter?', 'parameterization?', 'parse-command-line', 'partition', - 'path->bytes', 'path->complete-path', 'path->directory-path', - 'path->string', 'path-add-suffix', 'path-convention-type', - 'path-element->bytes', 'path-element->string', 'path-element?', - 'path-for-some-system?', 'path-list-string->path-list', 'path-only', - 'path-replace-suffix', 'path-string?', 'path<?', 'path?', - 'pathlist-closure', 'peek-byte', 'peek-byte-or-special', 'peek-bytes', - 'peek-bytes!', 'peek-bytes!-evt', 'peek-bytes-avail!', - 'peek-bytes-avail!*', 'peek-bytes-avail!-evt', - 'peek-bytes-avail!/enable-break', 'peek-bytes-evt', 'peek-char', - 'peek-char-or-special', 'peek-string', 'peek-string!', - 'peek-string!-evt', 'peek-string-evt', 'peeking-input-port', - 'permutations', 'phantom-bytes?', 'pi', 'pi.f', 'pipe-content-length', - 'place-break', 'place-channel', 'place-channel-get', - 'place-channel-put', 'place-channel-put/get', 'place-channel?', - 'place-dead-evt', 'place-enabled?', 'place-kill', 'place-location?', - 'place-message-allowed?', 'place-sleep', 'place-wait', 'place?', - 'placeholder-get', 'placeholder-set!', 'placeholder?', - 'poll-guard-evt', 'port->bytes', 'port->bytes-lines', 'port->lines', - 'port->list', 'port->string', 'port-closed-evt', 'port-closed?', - 'port-commit-peeked', 'port-count-lines!', 'port-count-lines-enabled', - 'port-counts-lines?', 'port-display-handler', 'port-file-identity', - 'port-file-unlock', 'port-next-location', 'port-print-handler', - 'port-progress-evt', 'port-provides-progress-evts?', - 'port-read-handler', 'port-try-file-lock?', 'port-write-handler', - 'port-writes-atomic?', 'port-writes-special?', 'port?', 'positive?', - 'predicate/c', 'prefab-key->struct-type', 'prefab-key?', - 'prefab-struct-key', 'preferences-lock-file-mode', 'pregexp', - 'pregexp?', 'pretty-display', 'pretty-format', 'pretty-print', - 'pretty-print-.-symbol-without-bars', - 'pretty-print-abbreviate-read-macros', 'pretty-print-columns', - 'pretty-print-current-style-table', 'pretty-print-depth', - 'pretty-print-exact-as-decimal', 'pretty-print-extend-style-table', - 'pretty-print-handler', 'pretty-print-newline', - 'pretty-print-post-print-hook', 'pretty-print-pre-print-hook', - 'pretty-print-print-hook', 'pretty-print-print-line', - 'pretty-print-remap-stylable', 'pretty-print-show-inexactness', - 'pretty-print-size-hook', 'pretty-print-style-table?', - 'pretty-printing', 'pretty-write', 'primitive-closure?', - 'primitive-result-arity', 'primitive?', 'print', 'print-as-expression', - 'print-boolean-long-form', 'print-box', 'print-graph', - 'print-hash-table', 'print-mpair-curly-braces', - 'print-pair-curly-braces', 'print-reader-abbreviations', - 'print-struct', 'print-syntax-width', 'print-unreadable', - 'print-vector-length', 'printable/c', 'printable<%>', 'printf', - 'procedure->method', 'procedure-arity', 'procedure-arity-includes/c', - 'procedure-arity-includes?', 'procedure-arity?', - 'procedure-closure-contents-eq?', 'procedure-extract-target', - 'procedure-keywords', 'procedure-reduce-arity', - 'procedure-reduce-keyword-arity', 'procedure-rename', - 'procedure-struct-type?', 'procedure?', 'process', 'process*', - 'process*/ports', 'process/ports', 'processor-count', 'progress-evt?', - 'promise-forced?', 'promise-running?', 'promise/c', 'promise?', - 'prop:arity-string', 'prop:chaperone-contract', - 'prop:checked-procedure', 'prop:contract', 'prop:contracted', - 'prop:custom-print-quotable', 'prop:custom-write', 'prop:dict', - 'prop:dict/contract', 'prop:equal+hash', 'prop:evt', - 'prop:exn:missing-module', 'prop:exn:srclocs', 'prop:flat-contract', - 'prop:impersonator-of', 'prop:input-port', - 'prop:liberal-define-context', 'prop:opt-chaperone-contract', - 'prop:opt-chaperone-contract-get-test', 'prop:opt-chaperone-contract?', - 'prop:output-port', 'prop:place-location', 'prop:procedure', - 'prop:rename-transformer', 'prop:sequence', 'prop:set!-transformer', - 'prop:stream', 'proper-subset?', 'pseudo-random-generator->vector', - 'pseudo-random-generator-vector?', 'pseudo-random-generator?', - 'put-preferences', 'putenv', 'quotient', 'quotient/remainder', - 'radians->degrees', 'raise', 'raise-argument-error', - 'raise-arguments-error', 'raise-arity-error', 'raise-blame-error', - 'raise-contract-error', 'raise-mismatch-error', - 'raise-not-cons-blame-error', 'raise-range-error', - 'raise-result-error', 'raise-syntax-error', 'raise-type-error', - 'raise-user-error', 'random', 'random-seed', 'range', 'rational?', - 'rationalize', 'read', 'read-accept-bar-quote', 'read-accept-box', - 'read-accept-compiled', 'read-accept-dot', 'read-accept-graph', - 'read-accept-infix-dot', 'read-accept-lang', 'read-accept-quasiquote', - 'read-accept-reader', 'read-byte', 'read-byte-or-special', - 'read-bytes', 'read-bytes!', 'read-bytes!-evt', 'read-bytes-avail!', - 'read-bytes-avail!*', 'read-bytes-avail!-evt', - 'read-bytes-avail!/enable-break', 'read-bytes-evt', 'read-bytes-line', - 'read-bytes-line-evt', 'read-case-sensitive', 'read-char', - 'read-char-or-special', 'read-curly-brace-as-paren', - 'read-decimal-as-inexact', 'read-eval-print-loop', 'read-language', - 'read-line', 'read-line-evt', 'read-on-demand-source', - 'read-square-bracket-as-paren', 'read-string', 'read-string!', - 'read-string!-evt', 'read-string-evt', 'read-syntax', - 'read-syntax/recursive', 'read/recursive', 'readtable-mapping', - 'readtable?', 'real->decimal-string', 'real->double-flonum', - 'real->floating-point-bytes', 'real->single-flonum', 'real-in', - 'real-part', 'real?', 'reencode-input-port', 'reencode-output-port', - 'regexp', 'regexp-match', 'regexp-match*', 'regexp-match-evt', - 'regexp-match-exact?', 'regexp-match-peek', - 'regexp-match-peek-immediate', 'regexp-match-peek-positions', - 'regexp-match-peek-positions*', - 'regexp-match-peek-positions-immediate', - 'regexp-match-peek-positions-immediate/end', - 'regexp-match-peek-positions/end', 'regexp-match-positions', - 'regexp-match-positions*', 'regexp-match-positions/end', - 'regexp-match/end', 'regexp-match?', 'regexp-max-lookbehind', - 'regexp-quote', 'regexp-replace', 'regexp-replace*', - 'regexp-replace-quote', 'regexp-replaces', 'regexp-split', - 'regexp-try-match', 'regexp?', 'relative-path?', 'relocate-input-port', - 'relocate-output-port', 'remainder', 'remove', 'remove*', - 'remove-duplicates', 'remq', 'remq*', 'remv', 'remv*', - 'rename-file-or-directory', 'rename-transformer-target', - 'rename-transformer?', 'reroot-path', 'resolve-path', - 'resolved-module-path-name', 'resolved-module-path?', 'rest', - 'reverse', 'round', 'second', 'seconds->date', 'security-guard?', - 'semaphore-peek-evt', 'semaphore-peek-evt?', 'semaphore-post', - 'semaphore-try-wait?', 'semaphore-wait', 'semaphore-wait/enable-break', - 'semaphore?', 'sequence->list', 'sequence->stream', - 'sequence-add-between', 'sequence-andmap', 'sequence-append', - 'sequence-count', 'sequence-filter', 'sequence-fold', - 'sequence-for-each', 'sequence-generate', 'sequence-generate*', - 'sequence-length', 'sequence-map', 'sequence-ormap', 'sequence-ref', - 'sequence-tail', 'sequence?', 'set', 'set!-transformer-procedure', - 'set!-transformer?', 'set->list', 'set->stream', 'set-add', 'set-add!', - 'set-box!', 'set-clear', 'set-clear!', 'set-copy', 'set-copy-clear', - 'set-count', 'set-empty?', 'set-eq?', 'set-equal?', 'set-eqv?', - 'set-first', 'set-for-each', 'set-implements/c', 'set-implements?', - 'set-intersect', 'set-intersect!', 'set-map', 'set-mcar!', 'set-mcdr!', - 'set-member?', 'set-mutable?', 'set-phantom-bytes!', - 'set-port-next-location!', 'set-remove', 'set-remove!', 'set-rest', - 'set-subtract', 'set-subtract!', 'set-symmetric-difference', - 'set-symmetric-difference!', 'set-union', 'set-union!', 'set-weak?', - 'set/c', 'set=?', 'set?', 'seteq', 'seteqv', 'seventh', 'sgn', - 'shared-bytes', 'shell-execute', 'shrink-path-wrt', 'shuffle', - 'simple-form-path', 'simplify-path', 'sin', 'single-flonum?', 'sinh', - 'sixth', 'skip-projection-wrapper?', 'sleep', - 'some-system-path->string', 'sort', 'special-comment-value', - 'special-comment?', 'special-filter-input-port', 'split-at', - 'split-at-right', 'split-path', 'splitf-at', 'splitf-at-right', 'sqr', - 'sqrt', 'srcloc', 'srcloc->string', 'srcloc-column', 'srcloc-line', - 'srcloc-position', 'srcloc-source', 'srcloc-span', 'srcloc?', - 'stop-after', 'stop-before', 'stream->list', 'stream-add-between', - 'stream-andmap', 'stream-append', 'stream-count', 'stream-empty?', - 'stream-filter', 'stream-first', 'stream-fold', 'stream-for-each', - 'stream-length', 'stream-map', 'stream-ormap', 'stream-ref', - 'stream-rest', 'stream-tail', 'stream?', 'string', - 'string->bytes/latin-1', 'string->bytes/locale', 'string->bytes/utf-8', - 'string->immutable-string', 'string->keyword', 'string->list', - 'string->number', 'string->path', 'string->path-element', - 'string->some-system-path', 'string->symbol', - 'string->uninterned-symbol', 'string->unreadable-symbol', - 'string-append', 'string-append*', 'string-ci<=?', 'string-ci<?', - 'string-ci=?', 'string-ci>=?', 'string-ci>?', 'string-copy', - 'string-copy!', 'string-downcase', 'string-environment-variable-name?', - 'string-fill!', 'string-foldcase', 'string-join', 'string-len/c', - 'string-length', 'string-locale-ci<?', 'string-locale-ci=?', - 'string-locale-ci>?', 'string-locale-downcase', 'string-locale-upcase', - 'string-locale<?', 'string-locale=?', 'string-locale>?', - 'string-no-nuls?', 'string-normalize-nfc', 'string-normalize-nfd', - 'string-normalize-nfkc', 'string-normalize-nfkd', - 'string-normalize-spaces', 'string-ref', 'string-replace', - 'string-set!', 'string-split', 'string-titlecase', 'string-trim', - 'string-upcase', 'string-utf-8-length', 'string<=?', 'string<?', - 'string=?', 'string>=?', 'string>?', 'string?', 'struct->vector', - 'struct-accessor-procedure?', 'struct-constructor-procedure?', - 'struct-info', 'struct-mutator-procedure?', - 'struct-predicate-procedure?', 'struct-type-info', - 'struct-type-make-constructor', 'struct-type-make-predicate', - 'struct-type-property-accessor-procedure?', 'struct-type-property/c', - 'struct-type-property?', 'struct-type?', 'struct:arity-at-least', - 'struct:date', 'struct:date*', 'struct:exn', 'struct:exn:break', - 'struct:exn:break:hang-up', 'struct:exn:break:terminate', - 'struct:exn:fail', 'struct:exn:fail:contract', - 'struct:exn:fail:contract:arity', 'struct:exn:fail:contract:blame', - 'struct:exn:fail:contract:continuation', - 'struct:exn:fail:contract:divide-by-zero', - 'struct:exn:fail:contract:non-fixnum-result', - 'struct:exn:fail:contract:variable', 'struct:exn:fail:filesystem', - 'struct:exn:fail:filesystem:errno', - 'struct:exn:fail:filesystem:exists', - 'struct:exn:fail:filesystem:missing-module', - 'struct:exn:fail:filesystem:version', 'struct:exn:fail:network', - 'struct:exn:fail:network:errno', 'struct:exn:fail:object', - 'struct:exn:fail:out-of-memory', 'struct:exn:fail:read', - 'struct:exn:fail:read:eof', 'struct:exn:fail:read:non-char', - 'struct:exn:fail:syntax', 'struct:exn:fail:syntax:missing-module', - 'struct:exn:fail:syntax:unbound', 'struct:exn:fail:unsupported', - 'struct:exn:fail:user', 'struct:srcloc', - 'struct:wrapped-extra-arg-arrow', 'struct?', 'sub1', 'subbytes', - 'subclass?', 'subclass?/c', 'subprocess', 'subprocess-group-enabled', - 'subprocess-kill', 'subprocess-pid', 'subprocess-status', - 'subprocess-wait', 'subprocess?', 'subset?', 'substring', - 'symbol->string', 'symbol-interned?', 'symbol-unreadable?', 'symbol<?', - 'symbol=?', 'symbol?', 'symbols', 'sync', 'sync/enable-break', - 'sync/timeout', 'sync/timeout/enable-break', 'syntax->datum', - 'syntax->list', 'syntax-arm', 'syntax-column', 'syntax-disarm', - 'syntax-e', 'syntax-line', 'syntax-local-bind-syntaxes', - 'syntax-local-certifier', 'syntax-local-context', - 'syntax-local-expand-expression', 'syntax-local-get-shadower', - 'syntax-local-introduce', 'syntax-local-lift-context', - 'syntax-local-lift-expression', - 'syntax-local-lift-module-end-declaration', - 'syntax-local-lift-provide', 'syntax-local-lift-require', - 'syntax-local-lift-values-expression', - 'syntax-local-make-definition-context', - 'syntax-local-make-delta-introducer', - 'syntax-local-module-defined-identifiers', - 'syntax-local-module-exports', - 'syntax-local-module-required-identifiers', 'syntax-local-name', - 'syntax-local-phase-level', 'syntax-local-submodules', - 'syntax-local-transforming-module-provides?', 'syntax-local-value', - 'syntax-local-value/immediate', 'syntax-original?', 'syntax-position', - 'syntax-property', 'syntax-property-symbol-keys', 'syntax-protect', - 'syntax-rearm', 'syntax-recertify', 'syntax-shift-phase-level', - 'syntax-source', 'syntax-source-module', 'syntax-span', 'syntax-taint', - 'syntax-tainted?', 'syntax-track-origin', - 'syntax-transforming-module-expression?', 'syntax-transforming?', - 'syntax/c', 'syntax?', 'system', 'system*', 'system*/exit-code', - 'system-big-endian?', 'system-idle-evt', 'system-language+country', - 'system-library-subpath', 'system-path-convention-type', 'system-type', - 'system/exit-code', 'tail-marks-match?', 'take', 'take-right', 'takef', - 'takef-right', 'tan', 'tanh', 'tcp-abandon-port', 'tcp-accept', - 'tcp-accept-evt', 'tcp-accept-ready?', 'tcp-accept/enable-break', - 'tcp-addresses', 'tcp-close', 'tcp-connect', - 'tcp-connect/enable-break', 'tcp-listen', 'tcp-listener?', 'tcp-port?', - 'tentative-pretty-print-port-cancel', - 'tentative-pretty-print-port-transfer', 'tenth', 'terminal-port?', - 'the-unsupplied-arg', 'third', 'thread', 'thread-cell-ref', - 'thread-cell-set!', 'thread-cell-values?', 'thread-cell?', - 'thread-dead-evt', 'thread-dead?', 'thread-group?', 'thread-receive', - 'thread-receive-evt', 'thread-resume', 'thread-resume-evt', - 'thread-rewind-receive', 'thread-running?', 'thread-send', - 'thread-suspend', 'thread-suspend-evt', 'thread-try-receive', - 'thread-wait', 'thread/suspend-to-kill', 'thread?', 'time-apply', - 'touch', 'transplant-input-port', 'transplant-output-port', 'true', - 'truncate', 'udp-addresses', 'udp-bind!', 'udp-bound?', 'udp-close', - 'udp-connect!', 'udp-connected?', 'udp-multicast-interface', - 'udp-multicast-join-group!', 'udp-multicast-leave-group!', - 'udp-multicast-loopback?', 'udp-multicast-set-interface!', - 'udp-multicast-set-loopback!', 'udp-multicast-set-ttl!', - 'udp-multicast-ttl', 'udp-open-socket', 'udp-receive!', - 'udp-receive!*', 'udp-receive!-evt', 'udp-receive!/enable-break', - 'udp-receive-ready-evt', 'udp-send', 'udp-send*', 'udp-send-evt', - 'udp-send-ready-evt', 'udp-send-to', 'udp-send-to*', 'udp-send-to-evt', - 'udp-send-to/enable-break', 'udp-send/enable-break', 'udp?', 'unbox', - 'uncaught-exception-handler', 'unit?', 'unspecified-dom', - 'unsupplied-arg?', 'use-collection-link-paths', - 'use-compiled-file-paths', 'use-user-specific-search-paths', - 'user-execute-bit', 'user-read-bit', 'user-write-bit', - 'value-contract', 'values', 'variable-reference->empty-namespace', - 'variable-reference->module-base-phase', - 'variable-reference->module-declaration-inspector', - 'variable-reference->module-path-index', - 'variable-reference->module-source', 'variable-reference->namespace', - 'variable-reference->phase', - 'variable-reference->resolved-module-path', - 'variable-reference-constant?', 'variable-reference?', 'vector', - 'vector->immutable-vector', 'vector->list', - 'vector->pseudo-random-generator', 'vector->pseudo-random-generator!', - 'vector->values', 'vector-append', 'vector-argmax', 'vector-argmin', - 'vector-copy', 'vector-copy!', 'vector-count', 'vector-drop', - 'vector-drop-right', 'vector-fill!', 'vector-filter', - 'vector-filter-not', 'vector-immutable', 'vector-immutable/c', - 'vector-immutableof', 'vector-length', 'vector-map', 'vector-map!', - 'vector-member', 'vector-memq', 'vector-memv', 'vector-ref', - 'vector-set!', 'vector-set*!', 'vector-set-performance-stats!', - 'vector-split-at', 'vector-split-at-right', 'vector-take', - 'vector-take-right', 'vector/c', 'vector?', 'vectorof', 'version', - 'void', 'void?', 'weak-box-value', 'weak-box?', 'weak-set', - 'weak-seteq', 'weak-seteqv', 'will-execute', 'will-executor?', - 'will-register', 'will-try-execute', 'with-input-from-bytes', - 'with-input-from-file', 'with-input-from-string', - 'with-output-to-bytes', 'with-output-to-file', 'with-output-to-string', - 'would-be-future', 'wrap-evt', 'wrapped-extra-arg-arrow', - 'wrapped-extra-arg-arrow-extra-neg-party-argument', - 'wrapped-extra-arg-arrow-real-func', 'wrapped-extra-arg-arrow?', - 'writable<%>', 'write', 'write-byte', 'write-bytes', - 'write-bytes-avail', 'write-bytes-avail*', 'write-bytes-avail-evt', - 'write-bytes-avail/enable-break', 'write-char', 'write-special', - 'write-special-avail*', 'write-special-evt', 'write-string', - 'write-to-file', 'xor', 'zero?', '~.a', '~.s', '~.v', '~a', '~e', '~r', - '~s', '~v' + u'*', u'*list/c', u'+', u'-', u'/', u'<', u'</c', u'<=', u'<=/c', u'=', u'=/c', + u'>', u'>/c', u'>=', u'>=/c', u'abort-current-continuation', u'abs', + u'absolute-path?', u'acos', u'add-between', u'add1', u'alarm-evt', + u'always-evt', u'and/c', u'andmap', u'angle', u'any/c', u'append', u'append*', + u'append-map', u'apply', u'argmax', u'argmin', u'arithmetic-shift', + u'arity-at-least', u'arity-at-least-value', u'arity-at-least?', + u'arity-checking-wrapper', u'arity-includes?', u'arity=?', + u'arrow-contract-info', u'arrow-contract-info-accepts-arglist', + u'arrow-contract-info-chaperone-procedure', + u'arrow-contract-info-check-first-order', u'arrow-contract-info?', + u'asin', u'assf', u'assoc', u'assq', u'assv', u'atan', + u'bad-number-of-results', u'banner', u'base->-doms/c', u'base->-rngs/c', + u'base->?', u'between/c', u'bitwise-and', u'bitwise-bit-field', + u'bitwise-bit-set?', u'bitwise-ior', u'bitwise-not', u'bitwise-xor', + u'blame-add-car-context', u'blame-add-cdr-context', u'blame-add-context', + u'blame-add-missing-party', u'blame-add-nth-arg-context', + u'blame-add-range-context', u'blame-add-unknown-context', + u'blame-context', u'blame-contract', u'blame-fmt->-string', + u'blame-missing-party?', u'blame-negative', u'blame-original?', + u'blame-positive', u'blame-replace-negative', u'blame-source', + u'blame-swap', u'blame-swapped?', u'blame-update', u'blame-value', + u'blame?', u'boolean=?', u'boolean?', u'bound-identifier=?', u'box', + u'box-cas!', u'box-immutable', u'box-immutable/c', u'box/c', u'box?', + u'break-enabled', u'break-parameterization?', u'break-thread', + u'build-chaperone-contract-property', u'build-compound-type-name', + u'build-contract-property', u'build-flat-contract-property', + u'build-list', u'build-path', u'build-path/convention-type', + u'build-string', u'build-vector', u'byte-pregexp', u'byte-pregexp?', + u'byte-ready?', u'byte-regexp', u'byte-regexp?', u'byte?', u'bytes', + u'bytes->immutable-bytes', u'bytes->list', u'bytes->path', + u'bytes->path-element', u'bytes->string/latin-1', u'bytes->string/locale', + u'bytes->string/utf-8', u'bytes-append', u'bytes-append*', + u'bytes-close-converter', u'bytes-convert', u'bytes-convert-end', + u'bytes-converter?', u'bytes-copy', u'bytes-copy!', + u'bytes-environment-variable-name?', u'bytes-fill!', u'bytes-join', + u'bytes-length', u'bytes-no-nuls?', u'bytes-open-converter', u'bytes-ref', + u'bytes-set!', u'bytes-utf-8-index', u'bytes-utf-8-length', + u'bytes-utf-8-ref', u'bytes<?', u'bytes=?', u'bytes>?', u'bytes?', u'caaaar', + u'caaadr', u'caaar', u'caadar', u'caaddr', u'caadr', u'caar', u'cadaar', + u'cadadr', u'cadar', u'caddar', u'cadddr', u'caddr', u'cadr', + u'call-in-nested-thread', u'call-with-atomic-output-file', + u'call-with-break-parameterization', + u'call-with-composable-continuation', u'call-with-continuation-barrier', + u'call-with-continuation-prompt', u'call-with-current-continuation', + u'call-with-default-reading-parameterization', + u'call-with-escape-continuation', u'call-with-exception-handler', + u'call-with-file-lock/timeout', u'call-with-immediate-continuation-mark', + u'call-with-input-bytes', u'call-with-input-file', + u'call-with-input-file*', u'call-with-input-string', + u'call-with-output-bytes', u'call-with-output-file', + u'call-with-output-file*', u'call-with-output-string', + u'call-with-parameterization', u'call-with-semaphore', + u'call-with-semaphore/enable-break', u'call-with-values', u'call/cc', + u'call/ec', u'car', u'cartesian-product', u'cdaaar', u'cdaadr', u'cdaar', + u'cdadar', u'cdaddr', u'cdadr', u'cdar', u'cddaar', u'cddadr', u'cddar', + u'cdddar', u'cddddr', u'cdddr', u'cddr', u'cdr', u'ceiling', u'channel-get', + u'channel-put', u'channel-put-evt', u'channel-put-evt?', + u'channel-try-get', u'channel/c', u'channel?', u'chaperone-box', + u'chaperone-channel', u'chaperone-continuation-mark-key', + u'chaperone-contract-property?', u'chaperone-contract?', u'chaperone-evt', + u'chaperone-hash', u'chaperone-hash-set', u'chaperone-of?', + u'chaperone-procedure', u'chaperone-procedure*', u'chaperone-prompt-tag', + u'chaperone-struct', u'chaperone-struct-type', u'chaperone-vector', + u'chaperone?', u'char->integer', u'char-alphabetic?', u'char-blank?', + u'char-ci<=?', u'char-ci<?', u'char-ci=?', u'char-ci>=?', u'char-ci>?', + u'char-downcase', u'char-foldcase', u'char-general-category', + u'char-graphic?', u'char-in', u'char-in/c', u'char-iso-control?', + u'char-lower-case?', u'char-numeric?', u'char-punctuation?', + u'char-ready?', u'char-symbolic?', u'char-title-case?', u'char-titlecase', + u'char-upcase', u'char-upper-case?', u'char-utf-8-length', + u'char-whitespace?', u'char<=?', u'char<?', u'char=?', u'char>=?', u'char>?', + u'char?', u'check-duplicate-identifier', u'check-duplicates', + u'checked-procedure-check-and-extract', u'choice-evt', + u'class->interface', u'class-info', u'class-seal', u'class-unseal', + u'class?', u'cleanse-path', u'close-input-port', u'close-output-port', + u'coerce-chaperone-contract', u'coerce-chaperone-contracts', + u'coerce-contract', u'coerce-contract/f', u'coerce-contracts', + u'coerce-flat-contract', u'coerce-flat-contracts', u'collect-garbage', + u'collection-file-path', u'collection-path', u'combinations', u'compile', + u'compile-allow-set!-undefined', u'compile-context-preservation-enabled', + u'compile-enforce-module-constants', u'compile-syntax', + u'compiled-expression-recompile', u'compiled-expression?', + u'compiled-module-expression?', u'complete-path?', u'complex?', u'compose', + u'compose1', u'conjoin', u'conjugate', u'cons', u'cons/c', u'cons?', u'const', + u'continuation-mark-key/c', u'continuation-mark-key?', + u'continuation-mark-set->context', u'continuation-mark-set->list', + u'continuation-mark-set->list*', u'continuation-mark-set-first', + u'continuation-mark-set?', u'continuation-marks', + u'continuation-prompt-available?', u'continuation-prompt-tag?', + u'continuation?', u'contract-continuation-mark-key', + u'contract-custom-write-property-proc', u'contract-exercise', + u'contract-first-order', u'contract-first-order-passes?', + u'contract-late-neg-projection', u'contract-name', u'contract-proc', + u'contract-projection', u'contract-property?', + u'contract-random-generate', u'contract-random-generate-fail', + u'contract-random-generate-fail?', + u'contract-random-generate-get-current-environment', + u'contract-random-generate-stash', u'contract-random-generate/choose', + u'contract-stronger?', u'contract-struct-exercise', + u'contract-struct-generate', u'contract-struct-late-neg-projection', + u'contract-struct-list-contract?', u'contract-val-first-projection', + u'contract?', u'convert-stream', u'copy-directory/files', u'copy-file', + u'copy-port', u'cos', u'cosh', u'count', u'current-blame-format', + u'current-break-parameterization', u'current-code-inspector', + u'current-command-line-arguments', u'current-compile', + u'current-compiled-file-roots', u'current-continuation-marks', + u'current-contract-region', u'current-custodian', u'current-directory', + u'current-directory-for-user', u'current-drive', + u'current-environment-variables', u'current-error-port', u'current-eval', + u'current-evt-pseudo-random-generator', + u'current-force-delete-permissions', u'current-future', + u'current-gc-milliseconds', u'current-get-interaction-input-port', + u'current-inexact-milliseconds', u'current-input-port', + u'current-inspector', u'current-library-collection-links', + u'current-library-collection-paths', u'current-load', + u'current-load-extension', u'current-load-relative-directory', + u'current-load/use-compiled', u'current-locale', u'current-logger', + u'current-memory-use', u'current-milliseconds', + u'current-module-declare-name', u'current-module-declare-source', + u'current-module-name-resolver', u'current-module-path-for-load', + u'current-namespace', u'current-output-port', u'current-parameterization', + u'current-plumber', u'current-preserved-thread-cell-values', + u'current-print', u'current-process-milliseconds', u'current-prompt-read', + u'current-pseudo-random-generator', u'current-read-interaction', + u'current-reader-guard', u'current-readtable', u'current-seconds', + u'current-security-guard', u'current-subprocess-custodian-mode', + u'current-thread', u'current-thread-group', + u'current-thread-initial-stack-size', + u'current-write-relative-directory', u'curry', u'curryr', + u'custodian-box-value', u'custodian-box?', u'custodian-limit-memory', + u'custodian-managed-list', u'custodian-memory-accounting-available?', + u'custodian-require-memory', u'custodian-shutdown-all', u'custodian?', + u'custom-print-quotable-accessor', u'custom-print-quotable?', + u'custom-write-accessor', u'custom-write-property-proc', u'custom-write?', + u'date', u'date*', u'date*-nanosecond', u'date*-time-zone-name', u'date*?', + u'date-day', u'date-dst?', u'date-hour', u'date-minute', u'date-month', + u'date-second', u'date-time-zone-offset', u'date-week-day', u'date-year', + u'date-year-day', u'date?', u'datum->syntax', u'datum-intern-literal', + u'default-continuation-prompt-tag', u'degrees->radians', + u'delete-directory', u'delete-directory/files', u'delete-file', + u'denominator', u'dict->list', u'dict-can-functional-set?', + u'dict-can-remove-keys?', u'dict-clear', u'dict-clear!', u'dict-copy', + u'dict-count', u'dict-empty?', u'dict-for-each', u'dict-has-key?', + u'dict-implements/c', u'dict-implements?', u'dict-iter-contract', + u'dict-iterate-first', u'dict-iterate-key', u'dict-iterate-next', + u'dict-iterate-value', u'dict-key-contract', u'dict-keys', u'dict-map', + u'dict-mutable?', u'dict-ref', u'dict-ref!', u'dict-remove', + u'dict-remove!', u'dict-set', u'dict-set!', u'dict-set*', u'dict-set*!', + u'dict-update', u'dict-update!', u'dict-value-contract', u'dict-values', + u'dict?', u'directory-exists?', u'directory-list', u'disjoin', u'display', + u'display-lines', u'display-lines-to-file', u'display-to-file', + u'displayln', u'double-flonum?', u'drop', u'drop-common-prefix', + u'drop-right', u'dropf', u'dropf-right', u'dump-memory-stats', + u'dup-input-port', u'dup-output-port', u'dynamic->*', u'dynamic-get-field', + u'dynamic-object/c', u'dynamic-place', u'dynamic-place*', + u'dynamic-require', u'dynamic-require-for-syntax', u'dynamic-send', + u'dynamic-set-field!', u'dynamic-wind', u'eighth', u'empty', + u'empty-sequence', u'empty-stream', u'empty?', + u'environment-variables-copy', u'environment-variables-names', + u'environment-variables-ref', u'environment-variables-set!', + u'environment-variables?', u'eof', u'eof-evt', u'eof-object?', + u'ephemeron-value', u'ephemeron?', u'eprintf', u'eq-contract-val', + u'eq-contract?', u'eq-hash-code', u'eq?', u'equal-contract-val', + u'equal-contract?', u'equal-hash-code', u'equal-secondary-hash-code', + u'equal<%>', u'equal?', u'equal?/recur', u'eqv-hash-code', u'eqv?', u'error', + u'error-display-handler', u'error-escape-handler', + u'error-print-context-length', u'error-print-source-location', + u'error-print-width', u'error-value->string-handler', u'eval', + u'eval-jit-enabled', u'eval-syntax', u'even?', u'evt/c', u'evt?', + u'exact->inexact', u'exact-ceiling', u'exact-floor', u'exact-integer?', + u'exact-nonnegative-integer?', u'exact-positive-integer?', u'exact-round', + u'exact-truncate', u'exact?', u'executable-yield-handler', u'exit', + u'exit-handler', u'exn', u'exn-continuation-marks', u'exn-message', + u'exn:break', u'exn:break-continuation', u'exn:break:hang-up', + u'exn:break:hang-up?', u'exn:break:terminate', u'exn:break:terminate?', + u'exn:break?', u'exn:fail', u'exn:fail:contract', + u'exn:fail:contract:arity', u'exn:fail:contract:arity?', + u'exn:fail:contract:blame', u'exn:fail:contract:blame-object', + u'exn:fail:contract:blame?', u'exn:fail:contract:continuation', + u'exn:fail:contract:continuation?', u'exn:fail:contract:divide-by-zero', + u'exn:fail:contract:divide-by-zero?', + u'exn:fail:contract:non-fixnum-result', + u'exn:fail:contract:non-fixnum-result?', u'exn:fail:contract:variable', + u'exn:fail:contract:variable-id', u'exn:fail:contract:variable?', + u'exn:fail:contract?', u'exn:fail:filesystem', + u'exn:fail:filesystem:errno', u'exn:fail:filesystem:errno-errno', + u'exn:fail:filesystem:errno?', u'exn:fail:filesystem:exists', + u'exn:fail:filesystem:exists?', u'exn:fail:filesystem:missing-module', + u'exn:fail:filesystem:missing-module-path', + u'exn:fail:filesystem:missing-module?', u'exn:fail:filesystem:version', + u'exn:fail:filesystem:version?', u'exn:fail:filesystem?', + u'exn:fail:network', u'exn:fail:network:errno', + u'exn:fail:network:errno-errno', u'exn:fail:network:errno?', + u'exn:fail:network?', u'exn:fail:object', u'exn:fail:object?', + u'exn:fail:out-of-memory', u'exn:fail:out-of-memory?', u'exn:fail:read', + u'exn:fail:read-srclocs', u'exn:fail:read:eof', u'exn:fail:read:eof?', + u'exn:fail:read:non-char', u'exn:fail:read:non-char?', u'exn:fail:read?', + u'exn:fail:syntax', u'exn:fail:syntax-exprs', + u'exn:fail:syntax:missing-module', + u'exn:fail:syntax:missing-module-path', + u'exn:fail:syntax:missing-module?', u'exn:fail:syntax:unbound', + u'exn:fail:syntax:unbound?', u'exn:fail:syntax?', u'exn:fail:unsupported', + u'exn:fail:unsupported?', u'exn:fail:user', u'exn:fail:user?', + u'exn:fail?', u'exn:misc:match?', u'exn:missing-module-accessor', + u'exn:missing-module?', u'exn:srclocs-accessor', u'exn:srclocs?', u'exn?', + u'exp', u'expand', u'expand-once', u'expand-syntax', u'expand-syntax-once', + u'expand-syntax-to-top-form', u'expand-to-top-form', u'expand-user-path', + u'explode-path', u'expt', u'externalizable<%>', u'failure-result/c', + u'false?', u'field-names', u'fifth', u'file->bytes', u'file->bytes-lines', + u'file->lines', u'file->list', u'file->string', u'file->value', + u'file-exists?', u'file-name-from-path', u'file-or-directory-identity', + u'file-or-directory-modify-seconds', u'file-or-directory-permissions', + u'file-position', u'file-position*', u'file-size', + u'file-stream-buffer-mode', u'file-stream-port?', u'file-truncate', + u'filename-extension', u'filesystem-change-evt', + u'filesystem-change-evt-cancel', u'filesystem-change-evt?', + u'filesystem-root-list', u'filter', u'filter-map', u'filter-not', + u'filter-read-input-port', u'find-executable-path', u'find-files', + u'find-library-collection-links', u'find-library-collection-paths', + u'find-relative-path', u'find-system-path', u'findf', u'first', + u'first-or/c', u'fixnum?', u'flat-contract', u'flat-contract-predicate', + u'flat-contract-property?', u'flat-contract?', u'flat-named-contract', + u'flatten', u'floating-point-bytes->real', u'flonum?', u'floor', + u'flush-output', u'fold-files', u'foldl', u'foldr', u'for-each', u'force', + u'format', u'fourth', u'fprintf', u'free-identifier=?', + u'free-label-identifier=?', u'free-template-identifier=?', + u'free-transformer-identifier=?', u'fsemaphore-count', u'fsemaphore-post', + u'fsemaphore-try-wait?', u'fsemaphore-wait', u'fsemaphore?', u'future', + u'future?', u'futures-enabled?', u'gcd', u'generate-member-key', + u'generate-temporaries', u'generic-set?', u'generic?', u'gensym', + u'get-output-bytes', u'get-output-string', u'get-preference', + u'get/build-late-neg-projection', u'get/build-val-first-projection', + u'getenv', u'global-port-print-handler', u'group-by', u'group-execute-bit', + u'group-read-bit', u'group-write-bit', u'guard-evt', u'handle-evt', + u'handle-evt?', u'has-blame?', u'has-contract?', u'hash', u'hash->list', + u'hash-clear', u'hash-clear!', u'hash-copy', u'hash-copy-clear', + u'hash-count', u'hash-empty?', u'hash-eq?', u'hash-equal?', u'hash-eqv?', + u'hash-for-each', u'hash-has-key?', u'hash-iterate-first', + u'hash-iterate-key', u'hash-iterate-key+value', u'hash-iterate-next', + u'hash-iterate-pair', u'hash-iterate-value', u'hash-keys', u'hash-map', + u'hash-placeholder?', u'hash-ref', u'hash-ref!', u'hash-remove', + u'hash-remove!', u'hash-set', u'hash-set!', u'hash-set*', u'hash-set*!', + u'hash-update', u'hash-update!', u'hash-values', u'hash-weak?', u'hash/c', + u'hash?', u'hasheq', u'hasheqv', u'identifier-binding', + u'identifier-binding-symbol', u'identifier-label-binding', + u'identifier-prune-lexical-context', + u'identifier-prune-to-source-module', + u'identifier-remove-from-definition-context', + u'identifier-template-binding', u'identifier-transformer-binding', + u'identifier?', u'identity', u'if/c', u'imag-part', u'immutable?', + u'impersonate-box', u'impersonate-channel', + u'impersonate-continuation-mark-key', u'impersonate-hash', + u'impersonate-hash-set', u'impersonate-procedure', + u'impersonate-procedure*', u'impersonate-prompt-tag', + u'impersonate-struct', u'impersonate-vector', u'impersonator-contract?', + u'impersonator-ephemeron', u'impersonator-of?', + u'impersonator-prop:application-mark', u'impersonator-prop:blame', + u'impersonator-prop:contracted', + u'impersonator-property-accessor-procedure?', u'impersonator-property?', + u'impersonator?', u'implementation?', u'implementation?/c', u'in-bytes', + u'in-bytes-lines', u'in-combinations', u'in-cycle', u'in-dict', + u'in-dict-keys', u'in-dict-pairs', u'in-dict-values', u'in-directory', + u'in-hash', u'in-hash-keys', u'in-hash-pairs', u'in-hash-values', + u'in-immutable-hash', u'in-immutable-hash-keys', + u'in-immutable-hash-pairs', u'in-immutable-hash-values', + u'in-immutable-set', u'in-indexed', u'in-input-port-bytes', + u'in-input-port-chars', u'in-lines', u'in-list', u'in-mlist', + u'in-mutable-hash', u'in-mutable-hash-keys', u'in-mutable-hash-pairs', + u'in-mutable-hash-values', u'in-mutable-set', u'in-naturals', + u'in-parallel', u'in-permutations', u'in-port', u'in-producer', u'in-range', + u'in-sequences', u'in-set', u'in-slice', u'in-stream', u'in-string', + u'in-syntax', u'in-value', u'in-values*-sequence', u'in-values-sequence', + u'in-vector', u'in-weak-hash', u'in-weak-hash-keys', u'in-weak-hash-pairs', + u'in-weak-hash-values', u'in-weak-set', u'inexact->exact', + u'inexact-real?', u'inexact?', u'infinite?', u'input-port-append', + u'input-port?', u'inspector?', u'instanceof/c', u'integer->char', + u'integer->integer-bytes', u'integer-bytes->integer', u'integer-in', + u'integer-length', u'integer-sqrt', u'integer-sqrt/remainder', u'integer?', + u'interface->method-names', u'interface-extension?', u'interface?', + u'internal-definition-context-binding-identifiers', + u'internal-definition-context-introduce', + u'internal-definition-context-seal', u'internal-definition-context?', + u'is-a?', u'is-a?/c', u'keyword->string', u'keyword-apply', u'keyword<?', + u'keyword?', u'keywords-match', u'kill-thread', u'last', u'last-pair', + u'lcm', u'length', u'liberal-define-context?', u'link-exists?', u'list', + u'list*', u'list*of', u'list->bytes', u'list->mutable-set', + u'list->mutable-seteq', u'list->mutable-seteqv', u'list->set', + u'list->seteq', u'list->seteqv', u'list->string', u'list->vector', + u'list->weak-set', u'list->weak-seteq', u'list->weak-seteqv', + u'list-contract?', u'list-prefix?', u'list-ref', u'list-set', u'list-tail', + u'list-update', u'list/c', u'list?', u'listen-port-number?', u'listof', + u'load', u'load-extension', u'load-on-demand-enabled', u'load-relative', + u'load-relative-extension', u'load/cd', u'load/use-compiled', + u'local-expand', u'local-expand/capture-lifts', + u'local-transformer-expand', u'local-transformer-expand/capture-lifts', + u'locale-string-encoding', u'log', u'log-all-levels', u'log-level-evt', + u'log-level?', u'log-max-level', u'log-message', u'log-receiver?', + u'logger-name', u'logger?', u'magnitude', u'make-arity-at-least', + u'make-base-empty-namespace', u'make-base-namespace', u'make-bytes', + u'make-channel', u'make-chaperone-contract', + u'make-continuation-mark-key', u'make-continuation-prompt-tag', + u'make-contract', u'make-custodian', u'make-custodian-box', + u'make-custom-hash', u'make-custom-hash-types', u'make-custom-set', + u'make-custom-set-types', u'make-date', u'make-date*', + u'make-derived-parameter', u'make-directory', u'make-directory*', + u'make-do-sequence', u'make-empty-namespace', + u'make-environment-variables', u'make-ephemeron', u'make-exn', + u'make-exn:break', u'make-exn:break:hang-up', u'make-exn:break:terminate', + u'make-exn:fail', u'make-exn:fail:contract', + u'make-exn:fail:contract:arity', u'make-exn:fail:contract:blame', + u'make-exn:fail:contract:continuation', + u'make-exn:fail:contract:divide-by-zero', + u'make-exn:fail:contract:non-fixnum-result', + u'make-exn:fail:contract:variable', u'make-exn:fail:filesystem', + u'make-exn:fail:filesystem:errno', u'make-exn:fail:filesystem:exists', + u'make-exn:fail:filesystem:missing-module', + u'make-exn:fail:filesystem:version', u'make-exn:fail:network', + u'make-exn:fail:network:errno', u'make-exn:fail:object', + u'make-exn:fail:out-of-memory', u'make-exn:fail:read', + u'make-exn:fail:read:eof', u'make-exn:fail:read:non-char', + u'make-exn:fail:syntax', u'make-exn:fail:syntax:missing-module', + u'make-exn:fail:syntax:unbound', u'make-exn:fail:unsupported', + u'make-exn:fail:user', u'make-file-or-directory-link', + u'make-flat-contract', u'make-fsemaphore', u'make-generic', + u'make-handle-get-preference-locked', u'make-hash', + u'make-hash-placeholder', u'make-hasheq', u'make-hasheq-placeholder', + u'make-hasheqv', u'make-hasheqv-placeholder', + u'make-immutable-custom-hash', u'make-immutable-hash', + u'make-immutable-hasheq', u'make-immutable-hasheqv', + u'make-impersonator-property', u'make-input-port', + u'make-input-port/read-to-peek', u'make-inspector', + u'make-keyword-procedure', u'make-known-char-range-list', + u'make-limited-input-port', u'make-list', u'make-lock-file-name', + u'make-log-receiver', u'make-logger', u'make-mixin-contract', + u'make-mutable-custom-set', u'make-none/c', u'make-object', + u'make-output-port', u'make-parameter', u'make-parent-directory*', + u'make-phantom-bytes', u'make-pipe', u'make-pipe-with-specials', + u'make-placeholder', u'make-plumber', u'make-polar', u'make-prefab-struct', + u'make-primitive-class', u'make-proj-contract', + u'make-pseudo-random-generator', u'make-reader-graph', u'make-readtable', + u'make-rectangular', u'make-rename-transformer', + u'make-resolved-module-path', u'make-security-guard', u'make-semaphore', + u'make-set!-transformer', u'make-shared-bytes', u'make-sibling-inspector', + u'make-special-comment', u'make-srcloc', u'make-string', + u'make-struct-field-accessor', u'make-struct-field-mutator', + u'make-struct-type', u'make-struct-type-property', + u'make-syntax-delta-introducer', u'make-syntax-introducer', + u'make-temporary-file', u'make-tentative-pretty-print-output-port', + u'make-thread-cell', u'make-thread-group', u'make-vector', + u'make-weak-box', u'make-weak-custom-hash', u'make-weak-custom-set', + u'make-weak-hash', u'make-weak-hasheq', u'make-weak-hasheqv', + u'make-will-executor', u'map', u'match-equality-test', + u'matches-arity-exactly?', u'max', u'mcar', u'mcdr', u'mcons', u'member', + u'member-name-key-hash-code', u'member-name-key=?', u'member-name-key?', + u'memf', u'memq', u'memv', u'merge-input', u'method-in-interface?', u'min', + u'mixin-contract', u'module->exports', u'module->imports', + u'module->language-info', u'module->namespace', + u'module-compiled-cross-phase-persistent?', u'module-compiled-exports', + u'module-compiled-imports', u'module-compiled-language-info', + u'module-compiled-name', u'module-compiled-submodules', + u'module-declared?', u'module-path-index-join', + u'module-path-index-resolve', u'module-path-index-split', + u'module-path-index-submodule', u'module-path-index?', u'module-path?', + u'module-predefined?', u'module-provide-protected?', u'modulo', u'mpair?', + u'mutable-set', u'mutable-seteq', u'mutable-seteqv', u'n->th', + u'nack-guard-evt', u'namespace-anchor->empty-namespace', + u'namespace-anchor->namespace', u'namespace-anchor?', + u'namespace-attach-module', u'namespace-attach-module-declaration', + u'namespace-base-phase', u'namespace-mapped-symbols', + u'namespace-module-identifier', u'namespace-module-registry', + u'namespace-require', u'namespace-require/constant', + u'namespace-require/copy', u'namespace-require/expansion-time', + u'namespace-set-variable-value!', u'namespace-symbol->identifier', + u'namespace-syntax-introduce', u'namespace-undefine-variable!', + u'namespace-unprotect-module', u'namespace-variable-value', u'namespace?', + u'nan?', u'natural-number/c', u'negate', u'negative?', u'never-evt', + u'new-∀/c', u'new-∃/c', u'newline', u'ninth', u'non-empty-listof', + u'non-empty-string?', u'none/c', u'normal-case-path', u'normalize-arity', + u'normalize-path', u'normalized-arity?', u'not', u'not/c', u'null', u'null?', + u'number->string', u'number?', u'numerator', u'object%', u'object->vector', + u'object-info', u'object-interface', u'object-method-arity-includes?', + u'object-name', u'object-or-false=?', u'object=?', u'object?', u'odd?', + u'one-of/c', u'open-input-bytes', u'open-input-file', + u'open-input-output-file', u'open-input-string', u'open-output-bytes', + u'open-output-file', u'open-output-nowhere', u'open-output-string', + u'or/c', u'order-of-magnitude', u'ormap', u'other-execute-bit', + u'other-read-bit', u'other-write-bit', u'output-port?', u'pair?', + u'parameter-procedure=?', u'parameter/c', u'parameter?', + u'parameterization?', u'parse-command-line', u'partition', u'path->bytes', + u'path->complete-path', u'path->directory-path', u'path->string', + u'path-add-suffix', u'path-convention-type', u'path-element->bytes', + u'path-element->string', u'path-element?', u'path-for-some-system?', + u'path-list-string->path-list', u'path-only', u'path-replace-suffix', + u'path-string?', u'path<?', u'path?', u'pathlist-closure', u'peek-byte', + u'peek-byte-or-special', u'peek-bytes', u'peek-bytes!', u'peek-bytes!-evt', + u'peek-bytes-avail!', u'peek-bytes-avail!*', u'peek-bytes-avail!-evt', + u'peek-bytes-avail!/enable-break', u'peek-bytes-evt', u'peek-char', + u'peek-char-or-special', u'peek-string', u'peek-string!', + u'peek-string!-evt', u'peek-string-evt', u'peeking-input-port', + u'permutations', u'phantom-bytes?', u'pi', u'pi.f', u'pipe-content-length', + u'place-break', u'place-channel', u'place-channel-get', + u'place-channel-put', u'place-channel-put/get', u'place-channel?', + u'place-dead-evt', u'place-enabled?', u'place-kill', u'place-location?', + u'place-message-allowed?', u'place-sleep', u'place-wait', u'place?', + u'placeholder-get', u'placeholder-set!', u'placeholder?', + u'plumber-add-flush!', u'plumber-flush-all', + u'plumber-flush-handle-remove!', u'plumber-flush-handle?', u'plumber?', + u'poll-guard-evt', u'port->bytes', u'port->bytes-lines', u'port->lines', + u'port->list', u'port->string', u'port-closed-evt', u'port-closed?', + u'port-commit-peeked', u'port-count-lines!', u'port-count-lines-enabled', + u'port-counts-lines?', u'port-display-handler', u'port-file-identity', + u'port-file-unlock', u'port-next-location', u'port-number?', + u'port-print-handler', u'port-progress-evt', + u'port-provides-progress-evts?', u'port-read-handler', + u'port-try-file-lock?', u'port-write-handler', u'port-writes-atomic?', + u'port-writes-special?', u'port?', u'positive?', u'predicate/c', + u'prefab-key->struct-type', u'prefab-key?', u'prefab-struct-key', + u'preferences-lock-file-mode', u'pregexp', u'pregexp?', u'pretty-display', + u'pretty-format', u'pretty-print', u'pretty-print-.-symbol-without-bars', + u'pretty-print-abbreviate-read-macros', u'pretty-print-columns', + u'pretty-print-current-style-table', u'pretty-print-depth', + u'pretty-print-exact-as-decimal', u'pretty-print-extend-style-table', + u'pretty-print-handler', u'pretty-print-newline', + u'pretty-print-post-print-hook', u'pretty-print-pre-print-hook', + u'pretty-print-print-hook', u'pretty-print-print-line', + u'pretty-print-remap-stylable', u'pretty-print-show-inexactness', + u'pretty-print-size-hook', u'pretty-print-style-table?', + u'pretty-printing', u'pretty-write', u'primitive-closure?', + u'primitive-result-arity', u'primitive?', u'print', u'print-as-expression', + u'print-boolean-long-form', u'print-box', u'print-graph', + u'print-hash-table', u'print-mpair-curly-braces', + u'print-pair-curly-braces', u'print-reader-abbreviations', + u'print-struct', u'print-syntax-width', u'print-unreadable', + u'print-vector-length', u'printable/c', u'printable<%>', u'printf', + u'println', u'procedure->method', u'procedure-arity', + u'procedure-arity-includes/c', u'procedure-arity-includes?', + u'procedure-arity?', u'procedure-closure-contents-eq?', + u'procedure-extract-target', u'procedure-keywords', + u'procedure-reduce-arity', u'procedure-reduce-keyword-arity', + u'procedure-rename', u'procedure-result-arity', u'procedure-specialize', + u'procedure-struct-type?', u'procedure?', u'process', u'process*', + u'process*/ports', u'process/ports', u'processor-count', u'progress-evt?', + u'promise-forced?', u'promise-running?', u'promise/c', u'promise/name?', + u'promise?', u'prop:arity-string', u'prop:arrow-contract', + u'prop:arrow-contract-get-info', u'prop:arrow-contract?', u'prop:blame', + u'prop:chaperone-contract', u'prop:checked-procedure', u'prop:contract', + u'prop:contracted', u'prop:custom-print-quotable', u'prop:custom-write', + u'prop:dict', u'prop:dict/contract', u'prop:equal+hash', u'prop:evt', + u'prop:exn:missing-module', u'prop:exn:srclocs', + u'prop:expansion-contexts', u'prop:flat-contract', + u'prop:impersonator-of', u'prop:input-port', + u'prop:liberal-define-context', u'prop:object-name', + u'prop:opt-chaperone-contract', u'prop:opt-chaperone-contract-get-test', + u'prop:opt-chaperone-contract?', u'prop:orc-contract', + u'prop:orc-contract-get-subcontracts', u'prop:orc-contract?', + u'prop:output-port', u'prop:place-location', u'prop:procedure', + u'prop:recursive-contract', u'prop:recursive-contract-unroll', + u'prop:recursive-contract?', u'prop:rename-transformer', u'prop:sequence', + u'prop:set!-transformer', u'prop:stream', u'proper-subset?', + u'pseudo-random-generator->vector', u'pseudo-random-generator-vector?', + u'pseudo-random-generator?', u'put-preferences', u'putenv', u'quotient', + u'quotient/remainder', u'radians->degrees', u'raise', + u'raise-argument-error', u'raise-arguments-error', u'raise-arity-error', + u'raise-blame-error', u'raise-contract-error', u'raise-mismatch-error', + u'raise-not-cons-blame-error', u'raise-range-error', + u'raise-result-error', u'raise-syntax-error', u'raise-type-error', + u'raise-user-error', u'random', u'random-seed', u'range', u'rational?', + u'rationalize', u'read', u'read-accept-bar-quote', u'read-accept-box', + u'read-accept-compiled', u'read-accept-dot', u'read-accept-graph', + u'read-accept-infix-dot', u'read-accept-lang', u'read-accept-quasiquote', + u'read-accept-reader', u'read-byte', u'read-byte-or-special', + u'read-bytes', u'read-bytes!', u'read-bytes!-evt', u'read-bytes-avail!', + u'read-bytes-avail!*', u'read-bytes-avail!-evt', + u'read-bytes-avail!/enable-break', u'read-bytes-evt', u'read-bytes-line', + u'read-bytes-line-evt', u'read-case-sensitive', u'read-cdot', u'read-char', + u'read-char-or-special', u'read-curly-brace-as-paren', + u'read-curly-brace-with-tag', u'read-decimal-as-inexact', + u'read-eval-print-loop', u'read-language', u'read-line', u'read-line-evt', + u'read-on-demand-source', u'read-square-bracket-as-paren', + u'read-square-bracket-with-tag', u'read-string', u'read-string!', + u'read-string!-evt', u'read-string-evt', u'read-syntax', + u'read-syntax/recursive', u'read/recursive', u'readtable-mapping', + u'readtable?', u'real->decimal-string', u'real->double-flonum', + u'real->floating-point-bytes', u'real->single-flonum', u'real-in', + u'real-part', u'real?', u'reencode-input-port', u'reencode-output-port', + u'regexp', u'regexp-match', u'regexp-match*', u'regexp-match-evt', + u'regexp-match-exact?', u'regexp-match-peek', + u'regexp-match-peek-immediate', u'regexp-match-peek-positions', + u'regexp-match-peek-positions*', + u'regexp-match-peek-positions-immediate', + u'regexp-match-peek-positions-immediate/end', + u'regexp-match-peek-positions/end', u'regexp-match-positions', + u'regexp-match-positions*', u'regexp-match-positions/end', + u'regexp-match/end', u'regexp-match?', u'regexp-max-lookbehind', + u'regexp-quote', u'regexp-replace', u'regexp-replace*', + u'regexp-replace-quote', u'regexp-replaces', u'regexp-split', + u'regexp-try-match', u'regexp?', u'relative-path?', u'relocate-input-port', + u'relocate-output-port', u'remainder', u'remf', u'remf*', u'remove', + u'remove*', u'remove-duplicates', u'remq', u'remq*', u'remv', u'remv*', + u'rename-contract', u'rename-file-or-directory', + u'rename-transformer-target', u'rename-transformer?', u'replace-evt', + u'reroot-path', u'resolve-path', u'resolved-module-path-name', + u'resolved-module-path?', u'rest', u'reverse', u'round', u'second', + u'seconds->date', u'security-guard?', u'semaphore-peek-evt', + u'semaphore-peek-evt?', u'semaphore-post', u'semaphore-try-wait?', + u'semaphore-wait', u'semaphore-wait/enable-break', u'semaphore?', + u'sequence->list', u'sequence->stream', u'sequence-add-between', + u'sequence-andmap', u'sequence-append', u'sequence-count', + u'sequence-filter', u'sequence-fold', u'sequence-for-each', + u'sequence-generate', u'sequence-generate*', u'sequence-length', + u'sequence-map', u'sequence-ormap', u'sequence-ref', u'sequence-tail', + u'sequence/c', u'sequence?', u'set', u'set!-transformer-procedure', + u'set!-transformer?', u'set->list', u'set->stream', u'set-add', u'set-add!', + u'set-box!', u'set-clear', u'set-clear!', u'set-copy', u'set-copy-clear', + u'set-count', u'set-empty?', u'set-eq?', u'set-equal?', u'set-eqv?', + u'set-first', u'set-for-each', u'set-implements/c', u'set-implements?', + u'set-intersect', u'set-intersect!', u'set-map', u'set-mcar!', u'set-mcdr!', + u'set-member?', u'set-mutable?', u'set-phantom-bytes!', + u'set-port-next-location!', u'set-remove', u'set-remove!', u'set-rest', + u'set-some-basic-contracts!', u'set-subtract', u'set-subtract!', + u'set-symmetric-difference', u'set-symmetric-difference!', u'set-union', + u'set-union!', u'set-weak?', u'set/c', u'set=?', u'set?', u'seteq', u'seteqv', + u'seventh', u'sgn', u'shared-bytes', u'shell-execute', u'shrink-path-wrt', + u'shuffle', u'simple-form-path', u'simplify-path', u'sin', + u'single-flonum?', u'sinh', u'sixth', u'skip-projection-wrapper?', u'sleep', + u'some-system-path->string', u'sort', u'special-comment-value', + u'special-comment?', u'special-filter-input-port', u'split-at', + u'split-at-right', u'split-common-prefix', u'split-path', u'splitf-at', + u'splitf-at-right', u'sqr', u'sqrt', u'srcloc', u'srcloc->string', + u'srcloc-column', u'srcloc-line', u'srcloc-position', u'srcloc-source', + u'srcloc-span', u'srcloc?', u'stop-after', u'stop-before', u'stream->list', + u'stream-add-between', u'stream-andmap', u'stream-append', u'stream-count', + u'stream-empty?', u'stream-filter', u'stream-first', u'stream-fold', + u'stream-for-each', u'stream-length', u'stream-map', u'stream-ormap', + u'stream-ref', u'stream-rest', u'stream-tail', u'stream/c', u'stream?', + u'string', u'string->bytes/latin-1', u'string->bytes/locale', + u'string->bytes/utf-8', u'string->immutable-string', u'string->keyword', + u'string->list', u'string->number', u'string->path', + u'string->path-element', u'string->some-system-path', u'string->symbol', + u'string->uninterned-symbol', u'string->unreadable-symbol', + u'string-append', u'string-append*', u'string-ci<=?', u'string-ci<?', + u'string-ci=?', u'string-ci>=?', u'string-ci>?', u'string-contains?', + u'string-copy', u'string-copy!', u'string-downcase', + u'string-environment-variable-name?', u'string-fill!', u'string-foldcase', + u'string-join', u'string-len/c', u'string-length', u'string-locale-ci<?', + u'string-locale-ci=?', u'string-locale-ci>?', u'string-locale-downcase', + u'string-locale-upcase', u'string-locale<?', u'string-locale=?', + u'string-locale>?', u'string-no-nuls?', u'string-normalize-nfc', + u'string-normalize-nfd', u'string-normalize-nfkc', + u'string-normalize-nfkd', u'string-normalize-spaces', u'string-port?', + u'string-prefix?', u'string-ref', u'string-replace', u'string-set!', + u'string-split', u'string-suffix?', u'string-titlecase', u'string-trim', + u'string-upcase', u'string-utf-8-length', u'string<=?', u'string<?', + u'string=?', u'string>=?', u'string>?', u'string?', u'struct->vector', + u'struct-accessor-procedure?', u'struct-constructor-procedure?', + u'struct-info', u'struct-mutator-procedure?', + u'struct-predicate-procedure?', u'struct-type-info', + u'struct-type-make-constructor', u'struct-type-make-predicate', + u'struct-type-property-accessor-procedure?', u'struct-type-property/c', + u'struct-type-property?', u'struct-type?', u'struct:arity-at-least', + u'struct:arrow-contract-info', u'struct:date', u'struct:date*', + u'struct:exn', u'struct:exn:break', u'struct:exn:break:hang-up', + u'struct:exn:break:terminate', u'struct:exn:fail', + u'struct:exn:fail:contract', u'struct:exn:fail:contract:arity', + u'struct:exn:fail:contract:blame', + u'struct:exn:fail:contract:continuation', + u'struct:exn:fail:contract:divide-by-zero', + u'struct:exn:fail:contract:non-fixnum-result', + u'struct:exn:fail:contract:variable', u'struct:exn:fail:filesystem', + u'struct:exn:fail:filesystem:errno', + u'struct:exn:fail:filesystem:exists', + u'struct:exn:fail:filesystem:missing-module', + u'struct:exn:fail:filesystem:version', u'struct:exn:fail:network', + u'struct:exn:fail:network:errno', u'struct:exn:fail:object', + u'struct:exn:fail:out-of-memory', u'struct:exn:fail:read', + u'struct:exn:fail:read:eof', u'struct:exn:fail:read:non-char', + u'struct:exn:fail:syntax', u'struct:exn:fail:syntax:missing-module', + u'struct:exn:fail:syntax:unbound', u'struct:exn:fail:unsupported', + u'struct:exn:fail:user', u'struct:srcloc', + u'struct:wrapped-extra-arg-arrow', u'struct?', u'sub1', u'subbytes', + u'subclass?', u'subclass?/c', u'subprocess', u'subprocess-group-enabled', + u'subprocess-kill', u'subprocess-pid', u'subprocess-status', + u'subprocess-wait', u'subprocess?', u'subset?', u'substring', u'suggest/c', + u'symbol->string', u'symbol-interned?', u'symbol-unreadable?', u'symbol<?', + u'symbol=?', u'symbol?', u'symbols', u'sync', u'sync/enable-break', + u'sync/timeout', u'sync/timeout/enable-break', u'syntax->datum', + u'syntax->list', u'syntax-arm', u'syntax-column', u'syntax-debug-info', + u'syntax-disarm', u'syntax-e', u'syntax-line', + u'syntax-local-bind-syntaxes', u'syntax-local-certifier', + u'syntax-local-context', u'syntax-local-expand-expression', + u'syntax-local-get-shadower', u'syntax-local-identifier-as-binding', + u'syntax-local-introduce', u'syntax-local-lift-context', + u'syntax-local-lift-expression', u'syntax-local-lift-module', + u'syntax-local-lift-module-end-declaration', + u'syntax-local-lift-provide', u'syntax-local-lift-require', + u'syntax-local-lift-values-expression', + u'syntax-local-make-definition-context', + u'syntax-local-make-delta-introducer', + u'syntax-local-module-defined-identifiers', + u'syntax-local-module-exports', + u'syntax-local-module-required-identifiers', u'syntax-local-name', + u'syntax-local-phase-level', u'syntax-local-submodules', + u'syntax-local-transforming-module-provides?', u'syntax-local-value', + u'syntax-local-value/immediate', u'syntax-original?', u'syntax-position', + u'syntax-property', u'syntax-property-preserved?', + u'syntax-property-symbol-keys', u'syntax-protect', u'syntax-rearm', + u'syntax-recertify', u'syntax-shift-phase-level', u'syntax-source', + u'syntax-source-module', u'syntax-span', u'syntax-taint', + u'syntax-tainted?', u'syntax-track-origin', + u'syntax-transforming-module-expression?', + u'syntax-transforming-with-lifts?', u'syntax-transforming?', u'syntax/c', + u'syntax?', u'system', u'system*', u'system*/exit-code', + u'system-big-endian?', u'system-idle-evt', u'system-language+country', + u'system-library-subpath', u'system-path-convention-type', u'system-type', + u'system/exit-code', u'tail-marks-match?', u'take', u'take-common-prefix', + u'take-right', u'takef', u'takef-right', u'tan', u'tanh', + u'tcp-abandon-port', u'tcp-accept', u'tcp-accept-evt', + u'tcp-accept-ready?', u'tcp-accept/enable-break', u'tcp-addresses', + u'tcp-close', u'tcp-connect', u'tcp-connect/enable-break', u'tcp-listen', + u'tcp-listener?', u'tcp-port?', u'tentative-pretty-print-port-cancel', + u'tentative-pretty-print-port-transfer', u'tenth', u'terminal-port?', + u'the-unsupplied-arg', u'third', u'thread', u'thread-cell-ref', + u'thread-cell-set!', u'thread-cell-values?', u'thread-cell?', + u'thread-dead-evt', u'thread-dead?', u'thread-group?', u'thread-receive', + u'thread-receive-evt', u'thread-resume', u'thread-resume-evt', + u'thread-rewind-receive', u'thread-running?', u'thread-send', + u'thread-suspend', u'thread-suspend-evt', u'thread-try-receive', + u'thread-wait', u'thread/suspend-to-kill', u'thread?', u'time-apply', + u'touch', u'transplant-input-port', u'transplant-output-port', u'true', + u'truncate', u'udp-addresses', u'udp-bind!', u'udp-bound?', u'udp-close', + u'udp-connect!', u'udp-connected?', u'udp-multicast-interface', + u'udp-multicast-join-group!', u'udp-multicast-leave-group!', + u'udp-multicast-loopback?', u'udp-multicast-set-interface!', + u'udp-multicast-set-loopback!', u'udp-multicast-set-ttl!', + u'udp-multicast-ttl', u'udp-open-socket', u'udp-receive!', + u'udp-receive!*', u'udp-receive!-evt', u'udp-receive!/enable-break', + u'udp-receive-ready-evt', u'udp-send', u'udp-send*', u'udp-send-evt', + u'udp-send-ready-evt', u'udp-send-to', u'udp-send-to*', u'udp-send-to-evt', + u'udp-send-to/enable-break', u'udp-send/enable-break', u'udp?', u'unbox', + u'uncaught-exception-handler', u'unit?', u'unspecified-dom', + u'unsupplied-arg?', u'use-collection-link-paths', + u'use-compiled-file-paths', u'use-user-specific-search-paths', + u'user-execute-bit', u'user-read-bit', u'user-write-bit', u'value-blame', + u'value-contract', u'values', u'variable-reference->empty-namespace', + u'variable-reference->module-base-phase', + u'variable-reference->module-declaration-inspector', + u'variable-reference->module-path-index', + u'variable-reference->module-source', u'variable-reference->namespace', + u'variable-reference->phase', + u'variable-reference->resolved-module-path', + u'variable-reference-constant?', u'variable-reference?', u'vector', + u'vector->immutable-vector', u'vector->list', + u'vector->pseudo-random-generator', u'vector->pseudo-random-generator!', + u'vector->values', u'vector-append', u'vector-argmax', u'vector-argmin', + u'vector-copy', u'vector-copy!', u'vector-count', u'vector-drop', + u'vector-drop-right', u'vector-fill!', u'vector-filter', + u'vector-filter-not', u'vector-immutable', u'vector-immutable/c', + u'vector-immutableof', u'vector-length', u'vector-map', u'vector-map!', + u'vector-member', u'vector-memq', u'vector-memv', u'vector-ref', + u'vector-set!', u'vector-set*!', u'vector-set-performance-stats!', + u'vector-split-at', u'vector-split-at-right', u'vector-take', + u'vector-take-right', u'vector/c', u'vector?', u'vectorof', u'version', + u'void', u'void?', u'weak-box-value', u'weak-box?', u'weak-set', + u'weak-seteq', u'weak-seteqv', u'will-execute', u'will-executor?', + u'will-register', u'will-try-execute', u'with-input-from-bytes', + u'with-input-from-file', u'with-input-from-string', + u'with-output-to-bytes', u'with-output-to-file', u'with-output-to-string', + u'would-be-future', u'wrap-evt', u'wrapped-extra-arg-arrow', + u'wrapped-extra-arg-arrow-extra-neg-party-argument', + u'wrapped-extra-arg-arrow-real-func', u'wrapped-extra-arg-arrow?', + u'writable<%>', u'write', u'write-byte', u'write-bytes', + u'write-bytes-avail', u'write-bytes-avail*', u'write-bytes-avail-evt', + u'write-bytes-avail/enable-break', u'write-char', u'write-special', + u'write-special-avail*', u'write-special-evt', u'write-string', + u'write-to-file', u'writeln', u'xor', u'zero?', u'~.a', u'~.s', u'~.v', u'~a', + u'~e', u'~r', u'~s', u'~v' ) _opening_parenthesis = r'[([{]' @@ -1358,7 +1407,7 @@ class NewLispLexer(RegexLexer): name = 'NewLisp' aliases = ['newlisp'] - filenames = ['*.lsp', '*.nl'] + filenames = ['*.lsp', '*.nl', '*.kif'] mimetypes = ['text/x-newlisp', 'application/x-newlisp'] flags = re.IGNORECASE | re.MULTILINE | re.UNICODE @@ -2135,49 +2184,52 @@ class ShenLexer(RegexLexer): filenames = ['*.shen'] mimetypes = ['text/x-shen', 'application/x-shen'] - DECLARATIONS = re.findall(r'\S+', """ - datatype define defmacro defprolog defcc synonyms declare package - type function - """) - - SPECIAL_FORMS = re.findall(r'\S+', """ - lambda get let if cases cond put time freeze value load $ - protect or and not do output prolog? trap-error error - make-string /. set @p @s @v - """) - - BUILTINS = re.findall(r'\S+', """ - == = * + - / < > >= <= <-address <-vector abort absvector - absvector? address-> adjoin append arity assoc bind boolean? - bound? call cd close cn compile concat cons cons? cut destroy - difference element? empty? enable-type-theory error-to-string - eval eval-kl exception explode external fail fail-if file - findall fix fst fwhen gensym get-time hash hd hdstr hdv head - identical implementation in include include-all-but inferences - input input+ integer? intern intersection is kill language - length limit lineread loaded macro macroexpand map mapcan - maxinferences mode n->string nl nth null number? occurrences - occurs-check open os out port porters pos pr preclude - preclude-all-but print profile profile-results ps quit read - read+ read-byte read-file read-file-as-bytelist - read-file-as-string read-from-string release remove return - reverse run save set simple-error snd specialise spy step - stinput stoutput str string->n string->symbol string? subst - symbol? systemf tail tc tc? thaw tl tlstr tlv track tuple? - undefmacro unify unify! union unprofile unspecialise untrack - variable? vector vector-> vector? verified version warn when - write-byte write-to-file y-or-n? - """) - - BUILTINS_ANYWHERE = re.findall(r'\S+', """ - where skip >> _ ! <e> <!> - """) + DECLARATIONS = ( + 'datatype', 'define', 'defmacro', 'defprolog', 'defcc', + 'synonyms', 'declare', 'package', 'type', 'function', + ) + + SPECIAL_FORMS = ( + 'lambda', 'get', 'let', 'if', 'cases', 'cond', 'put', 'time', 'freeze', + 'value', 'load', '$', 'protect', 'or', 'and', 'not', 'do', 'output', + 'prolog?', 'trap-error', 'error', 'make-string', '/.', 'set', '@p', + '@s', '@v', + ) + + BUILTINS = ( + '==', '=', '*', '+', '-', '/', '<', '>', '>=', '<=', '<-address', + '<-vector', 'abort', 'absvector', 'absvector?', 'address->', 'adjoin', + 'append', 'arity', 'assoc', 'bind', 'boolean?', 'bound?', 'call', 'cd', + 'close', 'cn', 'compile', 'concat', 'cons', 'cons?', 'cut', 'destroy', + 'difference', 'element?', 'empty?', 'enable-type-theory', + 'error-to-string', 'eval', 'eval-kl', 'exception', 'explode', 'external', + 'fail', 'fail-if', 'file', 'findall', 'fix', 'fst', 'fwhen', 'gensym', + 'get-time', 'hash', 'hd', 'hdstr', 'hdv', 'head', 'identical', + 'implementation', 'in', 'include', 'include-all-but', 'inferences', + 'input', 'input+', 'integer?', 'intern', 'intersection', 'is', 'kill', + 'language', 'length', 'limit', 'lineread', 'loaded', 'macro', 'macroexpand', + 'map', 'mapcan', 'maxinferences', 'mode', 'n->string', 'nl', 'nth', 'null', + 'number?', 'occurrences', 'occurs-check', 'open', 'os', 'out', 'port', + 'porters', 'pos', 'pr', 'preclude', 'preclude-all-but', 'print', 'profile', + 'profile-results', 'ps', 'quit', 'read', 'read+', 'read-byte', 'read-file', + 'read-file-as-bytelist', 'read-file-as-string', 'read-from-string', + 'release', 'remove', 'return', 'reverse', 'run', 'save', 'set', + 'simple-error', 'snd', 'specialise', 'spy', 'step', 'stinput', 'stoutput', + 'str', 'string->n', 'string->symbol', 'string?', 'subst', 'symbol?', + 'systemf', 'tail', 'tc', 'tc?', 'thaw', 'tl', 'tlstr', 'tlv', 'track', + 'tuple?', 'undefmacro', 'unify', 'unify!', 'union', 'unprofile', + 'unspecialise', 'untrack', 'variable?', 'vector', 'vector->', 'vector?', + 'verified', 'version', 'warn', 'when', 'write-byte', 'write-to-file', + 'y-or-n?', + ) + + BUILTINS_ANYWHERE = ('where', 'skip', '>>', '_', '!', '<e>', '<!>') MAPPINGS = dict((s, Keyword) for s in DECLARATIONS) MAPPINGS.update((s, Name.Builtin) for s in BUILTINS) MAPPINGS.update((s, Keyword) for s in SPECIAL_FORMS) - valid_symbol_chars = r'[\w!$%*+,<=>?/.\'@&#:_-]' + valid_symbol_chars = r'[\w!$%*+,<=>?/.\'@&#:-]' valid_name = '%s+' % valid_symbol_chars symbol_name = r'[a-z!$%%*+,<=>?/.\'@&#_-]%s*' % valid_symbol_chars variable = r'[A-Z]%s*' % valid_symbol_chars @@ -2313,7 +2365,7 @@ class CPSALexer(SchemeLexer): # valid names for identifiers # well, names can only not consist fully of numbers # but this should be good enough for now - valid_name = r'[a-zA-Z0-9!$%&*+,/:<=>?@^_~|-]+' + valid_name = r'[\w!$%&*+,/:<=>?@^~|-]+' tokens = { 'root': [ @@ -2334,7 +2386,7 @@ class CPSALexer(SchemeLexer): # strings, symbols and characters (r'"(\\\\|\\"|[^"])*"', String), (r"'" + valid_name, String.Symbol), - (r"#\\([()/'\"._!§$%& ?=+-]{1}|[a-zA-Z0-9]+)", String.Char), + (r"#\\([()/'\"._!§$%& ?=+-]|[a-zA-Z0-9]+)", String.Char), # constants (r'(#t|#f)', Name.Constant), @@ -2363,3 +2415,207 @@ class CPSALexer(SchemeLexer): (r'(\[|\])', Punctuation), ], } + + +class XtlangLexer(RegexLexer): + """An xtlang lexer for the `Extempore programming environment + <http://extempore.moso.com.au>`_. + + This is a mixture of Scheme and xtlang, really. Keyword lists are + taken from the Extempore Emacs mode + (https://github.com/extemporelang/extempore-emacs-mode) + + .. versionadded:: 2.2 + """ + name = 'xtlang' + aliases = ['extempore'] + filenames = ['*.xtm'] + mimetypes = [] + + common_keywords = ( + 'lambda', 'define', 'if', 'else', 'cond', 'and', + 'or', 'let', 'begin', 'set!', 'map', 'for-each', + ) + scheme_keywords = ( + 'do', 'delay', 'quasiquote', 'unquote', 'unquote-splicing', 'eval', + 'case', 'let*', 'letrec', 'quote', + ) + xtlang_bind_keywords = ( + 'bind-func', 'bind-val', 'bind-lib', 'bind-type', 'bind-alias', + 'bind-poly', 'bind-dylib', 'bind-lib-func', 'bind-lib-val', + ) + xtlang_keywords = ( + 'letz', 'memzone', 'cast', 'convert', 'dotimes', 'doloop', + ) + common_functions = ( + '*', '+', '-', '/', '<', '<=', '=', '>', '>=', '%', 'abs', 'acos', + 'angle', 'append', 'apply', 'asin', 'assoc', 'assq', 'assv', + 'atan', 'boolean?', 'caaaar', 'caaadr', 'caaar', 'caadar', + 'caaddr', 'caadr', 'caar', 'cadaar', 'cadadr', 'cadar', + 'caddar', 'cadddr', 'caddr', 'cadr', 'car', 'cdaaar', + 'cdaadr', 'cdaar', 'cdadar', 'cdaddr', 'cdadr', 'cdar', + 'cddaar', 'cddadr', 'cddar', 'cdddar', 'cddddr', 'cdddr', + 'cddr', 'cdr', 'ceiling', 'cons', 'cos', 'floor', 'length', + 'list', 'log', 'max', 'member', 'min', 'modulo', 'not', + 'reverse', 'round', 'sin', 'sqrt', 'substring', 'tan', + 'println', 'random', 'null?', 'callback', 'now', + ) + scheme_functions = ( + 'call-with-current-continuation', 'call-with-input-file', + 'call-with-output-file', 'call-with-values', 'call/cc', + 'char->integer', 'char-alphabetic?', 'char-ci<=?', 'char-ci<?', + 'char-ci=?', 'char-ci>=?', 'char-ci>?', 'char-downcase', + 'char-lower-case?', 'char-numeric?', 'char-ready?', + 'char-upcase', 'char-upper-case?', 'char-whitespace?', + 'char<=?', 'char<?', 'char=?', 'char>=?', 'char>?', 'char?', + 'close-input-port', 'close-output-port', 'complex?', + 'current-input-port', 'current-output-port', 'denominator', + 'display', 'dynamic-wind', 'eof-object?', 'eq?', 'equal?', + 'eqv?', 'even?', 'exact->inexact', 'exact?', 'exp', 'expt', + 'force', 'gcd', 'imag-part', 'inexact->exact', 'inexact?', + 'input-port?', 'integer->char', 'integer?', + 'interaction-environment', 'lcm', 'list->string', + 'list->vector', 'list-ref', 'list-tail', 'list?', 'load', + 'magnitude', 'make-polar', 'make-rectangular', 'make-string', + 'make-vector', 'memq', 'memv', 'negative?', 'newline', + 'null-environment', 'number->string', 'number?', + 'numerator', 'odd?', 'open-input-file', 'open-output-file', + 'output-port?', 'pair?', 'peek-char', 'port?', 'positive?', + 'procedure?', 'quotient', 'rational?', 'rationalize', 'read', + 'read-char', 'real-part', 'real?', + 'remainder', 'scheme-report-environment', 'set-car!', 'set-cdr!', + 'string', 'string->list', 'string->number', 'string->symbol', + 'string-append', 'string-ci<=?', 'string-ci<?', 'string-ci=?', + 'string-ci>=?', 'string-ci>?', 'string-copy', 'string-fill!', + 'string-length', 'string-ref', 'string-set!', 'string<=?', + 'string<?', 'string=?', 'string>=?', 'string>?', 'string?', + 'symbol->string', 'symbol?', 'transcript-off', 'transcript-on', + 'truncate', 'values', 'vector', 'vector->list', 'vector-fill!', + 'vector-length', 'vector?', + 'with-input-from-file', 'with-output-to-file', 'write', + 'write-char', 'zero?', + ) + xtlang_functions = ( + 'toString', 'afill!', 'pfill!', 'tfill!', 'tbind', 'vfill!', + 'array-fill!', 'pointer-fill!', 'tuple-fill!', 'vector-fill!', 'free', + 'array', 'tuple', 'list', '~', 'cset!', 'cref', '&', 'bor', + 'ang-names', '<<', '>>', 'nil', 'printf', 'sprintf', 'null', 'now', + 'pset!', 'pref-ptr', 'vset!', 'vref', 'aset!', 'aref', 'aref-ptr', + 'tset!', 'tref', 'tref-ptr', 'salloc', 'halloc', 'zalloc', 'alloc', + 'schedule', 'exp', 'log', 'sin', 'cos', 'tan', 'asin', 'acos', 'atan', + 'sqrt', 'expt', 'floor', 'ceiling', 'truncate', 'round', + 'llvm_printf', 'push_zone', 'pop_zone', 'memzone', 'callback', + 'llvm_sprintf', 'make-array', 'array-set!', 'array-ref', + 'array-ref-ptr', 'pointer-set!', 'pointer-ref', 'pointer-ref-ptr', + 'stack-alloc', 'heap-alloc', 'zone-alloc', 'make-tuple', 'tuple-set!', + 'tuple-ref', 'tuple-ref-ptr', 'closure-set!', 'closure-ref', 'pref', + 'pdref', 'impc_null', 'bitcast', 'void', 'ifret', 'ret->', 'clrun->', + 'make-env-zone', 'make-env', '<>', 'dtof', 'ftod', 'i1tof', + 'i1tod', 'i1toi8', 'i1toi32', 'i1toi64', 'i8tof', 'i8tod', + 'i8toi1', 'i8toi32', 'i8toi64', 'i32tof', 'i32tod', 'i32toi1', + 'i32toi8', 'i32toi64', 'i64tof', 'i64tod', 'i64toi1', + 'i64toi8', 'i64toi32', + ) + + # valid names for Scheme identifiers (names cannot consist fully + # of numbers, but this should be good enough for now) + valid_scheme_name = r'[\w!$%&*+,/:<=>?@^~|-]+' + + # valid characters in xtlang names & types + valid_xtlang_name = r'[\w.!-]+' + valid_xtlang_type = r'[]{}[\w<>,*/|!-]+' + + tokens = { + # keep track of when we're exiting the xtlang form + 'xtlang': [ + (r'\(', Punctuation, '#push'), + (r'\)', Punctuation, '#pop'), + + (r'(?<=bind-func\s)' + valid_xtlang_name, Name.Function), + (r'(?<=bind-val\s)' + valid_xtlang_name, Name.Function), + (r'(?<=bind-type\s)' + valid_xtlang_name, Name.Function), + (r'(?<=bind-alias\s)' + valid_xtlang_name, Name.Function), + (r'(?<=bind-poly\s)' + valid_xtlang_name, Name.Function), + (r'(?<=bind-lib\s)' + valid_xtlang_name, Name.Function), + (r'(?<=bind-dylib\s)' + valid_xtlang_name, Name.Function), + (r'(?<=bind-lib-func\s)' + valid_xtlang_name, Name.Function), + (r'(?<=bind-lib-val\s)' + valid_xtlang_name, Name.Function), + + # type annotations + (r':' + valid_xtlang_type, Keyword.Type), + + # types + (r'(<' + valid_xtlang_type + r'>|\|' + valid_xtlang_type + r'\||/' + + valid_xtlang_type + r'/|' + valid_xtlang_type + r'\*)\**', + Keyword.Type), + + # keywords + (words(xtlang_keywords, prefix=r'(?<=\()'), Keyword), + + # builtins + (words(xtlang_functions, prefix=r'(?<=\()'), Name.Function), + + include('common'), + + # variables + (valid_xtlang_name, Name.Variable), + ], + 'scheme': [ + # quoted symbols + (r"'" + valid_scheme_name, String.Symbol), + + # char literals + (r"#\\([()/'\"._!§$%& ?=+-]|[a-zA-Z0-9]+)", String.Char), + + # special operators + (r"('|#|`|,@|,|\.)", Operator), + + # keywords + (words(scheme_keywords, prefix=r'(?<=\()'), Keyword), + + # builtins + (words(scheme_functions, prefix=r'(?<=\()'), Name.Function), + + include('common'), + + # variables + (valid_scheme_name, Name.Variable), + ], + # common to both xtlang and Scheme + 'common': [ + # comments + (r';.*$', Comment.Single), + + # whitespaces - usually not relevant + (r'\s+', Text), + + # numbers + (r'-?\d+\.\d+', Number.Float), + (r'-?\d+', Number.Integer), + + # binary/oct/hex literals + (r'(#b|#o|#x)[\d.]+', Number), + + # strings + (r'"(\\\\|\\"|[^"])*"', String), + + # true/false constants + (r'(#t|#f)', Name.Constant), + + # keywords + (words(common_keywords, prefix=r'(?<=\()'), Keyword), + + # builtins + (words(common_functions, prefix=r'(?<=\()'), Name.Function), + + # the famous parentheses! + (r'(\(|\))', Punctuation), + ], + 'root': [ + # go into xtlang mode + (words(xtlang_bind_keywords, prefix=r'(?<=\()', suffix=r'\b'), + Keyword, 'xtlang'), + + include('scheme') + ], + } diff --git a/pygments/lexers/make.py b/pygments/lexers/make.py index f5eac127..b222b672 100644 --- a/pygments/lexers/make.py +++ b/pygments/lexers/make.py @@ -5,7 +5,7 @@ Lexers for Makefiles and similar. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -90,7 +90,7 @@ class BaseMakefileLexer(RegexLexer): bygroups(Keyword, Text), 'export'), (r'export\s+', Keyword), # assignment - (r'([\w${}.-]+)(\s*)([!?:+]?=)([ \t]*)((?:.*\\\n)+|.*\n)', + (r'([\w${}().-]+)(\s*)([!?:+]?=)([ \t]*)((?:.*\\\n)+|.*\n)', bygroups(Name.Variable, Text, Operator, Text, using(BashLexer))), # strings (r'(?s)"(\\\\|\\.|[^"\\])*"', String.Double), @@ -102,7 +102,7 @@ class BaseMakefileLexer(RegexLexer): (r'\$\(', Keyword, 'expansion'), ], 'expansion': [ - (r'[^$a-zA-Z_)]+', Text), + (r'[^$a-zA-Z_()]+', Text), (r'[a-zA-Z_]+', Name.Variable), (r'\$', Keyword), (r'\(', Keyword, '#push'), diff --git a/pygments/lexers/markup.py b/pygments/lexers/markup.py index aac8d27e..92dc9e7a 100644 --- a/pygments/lexers/markup.py +++ b/pygments/lexers/markup.py @@ -5,7 +5,7 @@ Lexers for non-HTML markup languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -24,7 +24,7 @@ from pygments.util import get_bool_opt, ClassNotFound __all__ = ['BBCodeLexer', 'MoinWikiLexer', 'RstLexer', 'TexLexer', 'GroffLexer', 'MozPreprocHashLexer', 'MozPreprocPercentLexer', 'MozPreprocXulLexer', 'MozPreprocJavascriptLexer', - 'MozPreprocCssLexer'] + 'MozPreprocCssLexer', 'MarkdownLexer'] class BBCodeLexer(RegexLexer): @@ -500,3 +500,96 @@ class MozPreprocCssLexer(DelegatingLexer): super(MozPreprocCssLexer, self).__init__( CssLexer, MozPreprocPercentLexer, **options) + +class MarkdownLexer(RegexLexer): + """ + For `Markdown <https://help.github.com/categories/writing-on-github/>`_ markup. + + .. versionadded:: 2.2 + """ + name = 'markdown' + aliases = ['md'] + filenames = ['*.md'] + mimetypes = ["text/x-markdown"] + flags = re.MULTILINE + + def _handle_codeblock(self, match): + """ + match args: 1:backticks, 2:lang_name, 3:newline, 4:code, 5:backticks + """ + from pygments.lexers import get_lexer_by_name + + # section header + yield match.start(1), String , match.group(1) + yield match.start(2), String , match.group(2) + yield match.start(3), Text , match.group(3) + + # lookup lexer if wanted and existing + lexer = None + if self.handlecodeblocks: + try: + lexer = get_lexer_by_name( match.group(2).strip() ) + except ClassNotFound: + pass + code = match.group(4) + + # no lexer for this language. handle it like it was a code block + if lexer is None: + yield match.start(4), String, code + return + + for item in do_insertions([], lexer.get_tokens_unprocessed(code)): + yield item + + yield match.start(5), String , match.group(5) + + tokens = { + 'root': [ + # heading with pound prefix + (r'^(#)([^#].+\n)', bygroups(Generic.Heading, Text)), + (r'^(#{2,6})(.+\n)', bygroups(Generic.Subheading, Text)), + # task list + (r'^(\s*)([*-] )(\[[ xX]\])( .+\n)', + bygroups(Text, Keyword, Keyword, using(this, state='inline'))), + # bulleted lists + (r'^(\s*)([*-])(\s)(.+\n)', + bygroups(Text, Keyword, Text, using(this, state='inline'))), + # numbered lists + (r'^(\s*)([0-9]+\.)( .+\n)', + bygroups(Text, Keyword, using(this, state='inline'))), + # quote + (r'^(\s*>\s)(.+\n)', bygroups(Keyword, Generic.Emph)), + # text block + (r'^(```\n)([\w\W]*?)(^```$)', bygroups(String, Text, String)), + # code block with language + (r'^(```)(\w+)(\n)([\w\W]*?)(^```$)', _handle_codeblock), + + include('inline'), + ], + 'inline': [ + # escape + (r'\\.', Text), + # italics + (r'(\s)([*_][^*_]+[*_])(\W|\n)', bygroups(Text, Generic.Emph, Text)), + # bold + # warning: the following rule eats internal tags. eg. **foo _bar_ baz** bar is not italics + (r'(\s)((\*\*|__).*\3)((?=\W|\n))', bygroups(Text, Generic.Strong, None, Text)), + # "proper way" (r'(\s)([*_]{2}[^*_]+[*_]{2})((?=\W|\n))', bygroups(Text, Generic.Strong, Text)), + # strikethrough + (r'(\s)(~~[^~]+~~)((?=\W|\n))', bygroups(Text, Generic.Deleted, Text)), + # inline code + (r'`[^`]+`', String.Backtick), + # mentions and topics (twitter and github stuff) + (r'[@#][\w/:]+', Name.Entity), + # (image?) links eg:  + (r'(!?\[)([^]]+)(\])(\()([^)]+)(\))', bygroups(Text, Name.Tag, Text, Text, Name.Attribute, Text)), + + # general text, must come last! + (r'[^\\\s]+', Text), + (r'.', Text), + ], + } + + def __init__(self, **options): + self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True) + RegexLexer.__init__(self, **options) diff --git a/pygments/lexers/math.py b/pygments/lexers/math.py index 7a92f5bb..ea0ebee2 100644 --- a/pygments/lexers/math.py +++ b/pygments/lexers/math.py @@ -5,7 +5,7 @@ Just export lexers that were contained in this module. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/matlab.py b/pygments/lexers/matlab.py index ccb11a5d..56a0f6d6 100644 --- a/pygments/lexers/matlab.py +++ b/pygments/lexers/matlab.py @@ -5,7 +5,7 @@ Lexers for Matlab and related languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/ml.py b/pygments/lexers/ml.py index 4f10edd0..f80d5bfa 100644 --- a/pygments/lexers/ml.py +++ b/pygments/lexers/ml.py @@ -5,7 +5,7 @@ Lexers for ML family languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/modeling.py b/pygments/lexers/modeling.py index a6b0cb77..b354f1cf 100644 --- a/pygments/lexers/modeling.py +++ b/pygments/lexers/modeling.py @@ -5,7 +5,7 @@ Lexers for modeling languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/modula2.py b/pygments/lexers/modula2.py index a5fcbf78..c0a69b40 100644 --- a/pygments/lexers/modula2.py +++ b/pygments/lexers/modula2.py @@ -5,7 +5,7 @@ Multi-Dialect Lexer for Modula-2. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -290,7 +290,7 @@ class Modula2Lexer(RegexLexer): ], 'unigraph_punctuation': [ # Common Punctuation - (r'[\(\)\[\]{},.:;\|]', Punctuation), + (r'[()\[\]{},.:;|]', Punctuation), # Case Label Separator Synonym (r'!', Punctuation), # ISO # Blueprint Punctuation diff --git a/pygments/lexers/monte.py b/pygments/lexers/monte.py new file mode 100644 index 00000000..ed6e20f8 --- /dev/null +++ b/pygments/lexers/monte.py @@ -0,0 +1,204 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.monte + ~~~~~~~~~~~~~~~~~~~~~ + + Lexer for the Monte programming language. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.token import Comment, Error, Keyword, Name, Number, Operator, \ + Punctuation, String, Whitespace +from pygments.lexer import RegexLexer, include, words + +__all__ = ['MonteLexer'] + + +# `var` handled separately +# `interface` handled separately +_declarations = ['bind', 'def', 'fn', 'object'] +_methods = ['method', 'to'] +_keywords = [ + 'as', 'break', 'catch', 'continue', 'else', 'escape', 'exit', 'exports', + 'extends', 'finally', 'for', 'guards', 'if', 'implements', 'import', + 'in', 'match', 'meta', 'pass', 'return', 'switch', 'try', 'via', 'when', + 'while', +] +_operators = [ + # Unary + '~', '!', + # Binary + '+', '-', '*', '/', '%', '**', '&', '|', '^', '<<', '>>', + # Binary augmented + '+=', '-=', '*=', '/=', '%=', '**=', '&=', '|=', '^=', '<<=', '>>=', + # Comparison + '==', '!=', '<', '<=', '>', '>=', '<=>', + # Patterns and assignment + ':=', '?', '=~', '!~', '=>', + # Calls and sends + '.', '<-', '->', +] +_escape_pattern = ( + r'(?:\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|' + r'\\["\'\\bftnr])') +# _char = _escape_chars + [('.', String.Char)] +_identifier = r'[_a-zA-Z]\w*' + +_constants = [ + # Void constants + 'null', + # Bool constants + 'false', 'true', + # Double constants + 'Infinity', 'NaN', + # Special objects + 'M', 'Ref', 'throw', 'traceln', +] + +_guards = [ + 'Any', 'Binding', 'Bool', 'Bytes', 'Char', 'DeepFrozen', 'Double', + 'Empty', 'Int', 'List', 'Map', 'Near', 'NullOk', 'Same', 'Selfless', + 'Set', 'Str', 'SubrangeGuard', 'Transparent', 'Void', +] + +_safeScope = [ + '_accumulateList', '_accumulateMap', '_auditedBy', '_bind', + '_booleanFlow', '_comparer', '_equalizer', '_iterForever', '_loop', + '_makeBytes', '_makeDouble', '_makeFinalSlot', '_makeInt', '_makeList', + '_makeMap', '_makeMessageDesc', '_makeOrderedSpace', '_makeParamDesc', + '_makeProtocolDesc', '_makeSourceSpan', '_makeString', '_makeVarSlot', + '_makeVerbFacet', '_mapExtract', '_matchSame', '_quasiMatcher', + '_slotToBinding', '_splitList', '_suchThat', '_switchFailed', + '_validateFor', 'b__quasiParser', 'eval', 'import', 'm__quasiParser', + 'makeBrandPair', 'makeLazySlot', 'safeScope', 'simple__quasiParser', +] + + +class MonteLexer(RegexLexer): + """ + Lexer for the `Monte <https://monte.readthedocs.io/>`_ programming language. + + .. versionadded:: 2.2 + """ + name = 'Monte' + aliases = ['monte'] + filenames = ['*.mt'] + + tokens = { + 'root': [ + # Comments + (r'#[^\n]*\n', Comment), + + # Docstrings + # Apologies for the non-greedy matcher here. + (r'/\*\*.*?\*/', String.Doc), + + # `var` declarations + (r'\bvar\b', Keyword.Declaration, 'var'), + + # `interface` declarations + (r'\binterface\b', Keyword.Declaration, 'interface'), + + # method declarations + (words(_methods, prefix='\\b', suffix='\\b'), + Keyword, 'method'), + + # All other declarations + (words(_declarations, prefix='\\b', suffix='\\b'), + Keyword.Declaration), + + # Keywords + (words(_keywords, prefix='\\b', suffix='\\b'), Keyword), + + # Literals + ('[+-]?0x[_0-9a-fA-F]+', Number.Hex), + (r'[+-]?[_0-9]+\.[_0-9]*([eE][+-]?[_0-9]+)?', Number.Float), + ('[+-]?[_0-9]+', Number.Integer), + ("'", String.Double, 'char'), + ('"', String.Double, 'string'), + + # Quasiliterals + ('`', String.Backtick, 'ql'), + + # Operators + (words(_operators), Operator), + + # Verb operators + (_identifier + '=', Operator.Word), + + # Safe scope constants + (words(_constants, prefix='\\b', suffix='\\b'), + Keyword.Pseudo), + + # Safe scope guards + (words(_guards, prefix='\\b', suffix='\\b'), Keyword.Type), + + # All other safe scope names + (words(_safeScope, prefix='\\b', suffix='\\b'), + Name.Builtin), + + # Identifiers + (_identifier, Name), + + # Punctuation + (r'\(|\)|\{|\}|\[|\]|:|,', Punctuation), + + # Whitespace + (' +', Whitespace), + + # Definite lexer errors + ('=', Error), + ], + 'char': [ + # It is definitely an error to have a char of width == 0. + ("'", Error, 'root'), + (_escape_pattern, String.Escape, 'charEnd'), + ('.', String.Char, 'charEnd'), + ], + 'charEnd': [ + ("'", String.Char, '#pop:2'), + # It is definitely an error to have a char of width > 1. + ('.', Error), + ], + # The state of things coming into an interface. + 'interface': [ + (' +', Whitespace), + (_identifier, Name.Class, '#pop'), + include('root'), + ], + # The state of things coming into a method. + 'method': [ + (' +', Whitespace), + (_identifier, Name.Function, '#pop'), + include('root'), + ], + 'string': [ + ('"', String.Double, 'root'), + (_escape_pattern, String.Escape), + (r'\n', String.Double), + ('.', String.Double), + ], + 'ql': [ + ('`', String.Backtick, 'root'), + (r'\$' + _escape_pattern, String.Escape), + (r'\$\$', String.Escape), + (r'@@', String.Escape), + (r'\$\{', String.Interpol, 'qlNest'), + (r'@\{', String.Interpol, 'qlNest'), + (r'\$' + _identifier, Name), + ('@' + _identifier, Name), + ('.', String.Backtick), + ], + 'qlNest': [ + (r'\}', String.Interpol, '#pop'), + include('root'), + ], + # The state of things immediately following `var`. + 'var': [ + (' +', Whitespace), + (_identifier, Name.Variable, '#pop'), + include('root'), + ], + } diff --git a/pygments/lexers/ncl.py b/pygments/lexers/ncl.py new file mode 100644 index 00000000..3ca5135c --- /dev/null +++ b/pygments/lexers/ncl.py @@ -0,0 +1,894 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.ncl + ~~~~~~~~~~~~~~~~~~~ + + Lexers for NCAR Command Language. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation + +__all__ = ['NCLLexer'] + + +class NCLLexer(RegexLexer): + """ + Lexer for NCL code. + + .. versionadded:: 2.2 + """ + name = 'NCL' + aliases = ['ncl'] + filenames = ['*.ncl'] + mimetypes = ['text/ncl'] + flags = re.MULTILINE + + tokens = { + 'root': [ + (r';.*\n', Comment), + include('strings'), + include('core'), + (r'[a-zA-Z_]\w*', Name), + include('nums'), + (r'[\s]+', Text), + ], + 'core': [ + # Statements + (words(( + 'begin', 'break', 'continue', 'create', 'defaultapp', 'do', + 'else', 'end', 'external', 'exit', 'True', 'False', 'file', 'function', + 'getvalues', 'graphic', 'group', 'if', 'list', 'load', 'local', + 'new', '_Missing', 'Missing', 'noparent', 'procedure', + 'quit', 'QUIT', 'Quit', 'record', 'return', 'setvalues', 'stop', + 'then', 'while'), prefix=r'\b', suffix=r'\s*\b'), + Keyword), + + # Data Types + (words(( + 'ubyte', 'uint', 'uint64', 'ulong', 'string', 'byte', + 'character', 'double', 'float', 'integer', 'int64', 'logical', + 'long', 'short', 'ushort', 'enumeric', 'numeric', 'snumeric'), + prefix=r'\b', suffix=r'\s*\b'), + Keyword.Type), + + # Operators + (r'[\%^*+\-/<>]', Operator), + + # punctuation: + (r'[\[\]():@$!&|.,\\{}]', Punctuation), + (r'[=:]', Punctuation), + + # Intrinsics + (words(( + 'abs', 'acos', 'addfile', 'addfiles', 'all', 'angmom_atm', 'any', + 'area_conserve_remap', 'area_hi2lores', 'area_poly_sphere', + 'asciiread', 'asciiwrite', 'asin', 'atan', 'atan2', 'attsetvalues', + 'avg', 'betainc', 'bin_avg', 'bin_sum', 'bw_bandpass_filter', + 'cancor', 'cbinread', 'cbinwrite', 'cd_calendar', 'cd_inv_calendar', + 'cdfbin_p', 'cdfbin_pr', 'cdfbin_s', 'cdfbin_xn', 'cdfchi_p', + 'cdfchi_x', 'cdfgam_p', 'cdfgam_x', 'cdfnor_p', 'cdfnor_x', + 'cdft_p', 'cdft_t', 'ceil', 'center_finite_diff', + 'center_finite_diff_n', 'cfftb', 'cfftf', 'cfftf_frq_reorder', + 'charactertodouble', 'charactertofloat', 'charactertointeger', + 'charactertolong', 'charactertoshort', 'charactertostring', + 'chartodouble', 'chartofloat', 'chartoint', 'chartointeger', + 'chartolong', 'chartoshort', 'chartostring', 'chiinv', 'clear', + 'color_index_to_rgba', 'conform', 'conform_dims', 'cos', 'cosh', + 'count_unique_values', 'covcorm', 'covcorm_xy', 'craybinnumrec', + 'craybinrecread', 'create_graphic', 'csa1', 'csa1d', 'csa1s', + 'csa1x', 'csa1xd', 'csa1xs', 'csa2', 'csa2d', 'csa2l', 'csa2ld', + 'csa2ls', 'csa2lx', 'csa2lxd', 'csa2lxs', 'csa2s', 'csa2x', + 'csa2xd', 'csa2xs', 'csa3', 'csa3d', 'csa3l', 'csa3ld', 'csa3ls', + 'csa3lx', 'csa3lxd', 'csa3lxs', 'csa3s', 'csa3x', 'csa3xd', + 'csa3xs', 'csc2s', 'csgetp', 'css2c', 'cssetp', 'cssgrid', 'csstri', + 'csvoro', 'cumsum', 'cz2ccm', 'datatondc', 'day_of_week', + 'day_of_year', 'days_in_month', 'default_fillvalue', 'delete', + 'depth_to_pres', 'destroy', 'determinant', 'dewtemp_trh', + 'dgeevx_lapack', 'dim_acumrun_n', 'dim_avg', 'dim_avg_n', + 'dim_avg_wgt', 'dim_avg_wgt_n', 'dim_cumsum', 'dim_cumsum_n', + 'dim_gamfit_n', 'dim_gbits', 'dim_max', 'dim_max_n', 'dim_median', + 'dim_median_n', 'dim_min', 'dim_min_n', 'dim_num', 'dim_num_n', + 'dim_numrun_n', 'dim_pqsort', 'dim_pqsort_n', 'dim_product', + 'dim_product_n', 'dim_rmsd', 'dim_rmsd_n', 'dim_rmvmean', + 'dim_rmvmean_n', 'dim_rmvmed', 'dim_rmvmed_n', 'dim_spi_n', + 'dim_standardize', 'dim_standardize_n', 'dim_stat4', 'dim_stat4_n', + 'dim_stddev', 'dim_stddev_n', 'dim_sum', 'dim_sum_n', 'dim_sum_wgt', + 'dim_sum_wgt_n', 'dim_variance', 'dim_variance_n', 'dimsizes', + 'doubletobyte', 'doubletochar', 'doubletocharacter', + 'doubletofloat', 'doubletoint', 'doubletointeger', 'doubletolong', + 'doubletoshort', 'dpres_hybrid_ccm', 'dpres_plevel', 'draw', + 'draw_color_palette', 'dsgetp', 'dsgrid2', 'dsgrid2d', 'dsgrid2s', + 'dsgrid3', 'dsgrid3d', 'dsgrid3s', 'dspnt2', 'dspnt2d', 'dspnt2s', + 'dspnt3', 'dspnt3d', 'dspnt3s', 'dssetp', 'dtrend', 'dtrend_msg', + 'dtrend_msg_n', 'dtrend_n', 'dtrend_quadratic', + 'dtrend_quadratic_msg_n', 'dv2uvf', 'dv2uvg', 'dz_height', + 'echo_off', 'echo_on', 'eof2data', 'eof_varimax', 'eofcor', + 'eofcor_pcmsg', 'eofcor_ts', 'eofcov', 'eofcov_pcmsg', 'eofcov_ts', + 'eofunc', 'eofunc_ts', 'eofunc_varimax', 'equiv_sample_size', 'erf', + 'erfc', 'esacr', 'esacv', 'esccr', 'esccv', 'escorc', 'escorc_n', + 'escovc', 'exit', 'exp', 'exp_tapersh', 'exp_tapersh_wgts', + 'exp_tapershC', 'ezfftb', 'ezfftb_n', 'ezfftf', 'ezfftf_n', + 'f2fosh', 'f2foshv', 'f2fsh', 'f2fshv', 'f2gsh', 'f2gshv', 'fabs', + 'fbindirread', 'fbindirwrite', 'fbinnumrec', 'fbinread', + 'fbinrecread', 'fbinrecwrite', 'fbinwrite', 'fft2db', 'fft2df', + 'fftshift', 'fileattdef', 'filechunkdimdef', 'filedimdef', + 'fileexists', 'filegrpdef', 'filevarattdef', 'filevarchunkdef', + 'filevarcompressleveldef', 'filevardef', 'filevardimsizes', + 'filwgts_lancos', 'filwgts_lanczos', 'filwgts_normal', + 'floattobyte', 'floattochar', 'floattocharacter', 'floattoint', + 'floattointeger', 'floattolong', 'floattoshort', 'floor', + 'fluxEddy', 'fo2fsh', 'fo2fshv', 'fourier_info', 'frame', 'fspan', + 'ftcurv', 'ftcurvd', 'ftcurvi', 'ftcurvp', 'ftcurvpi', 'ftcurvps', + 'ftcurvs', 'ftest', 'ftgetp', 'ftkurv', 'ftkurvd', 'ftkurvp', + 'ftkurvpd', 'ftsetp', 'ftsurf', 'g2fsh', 'g2fshv', 'g2gsh', + 'g2gshv', 'gamma', 'gammainc', 'gaus', 'gaus_lobat', + 'gaus_lobat_wgt', 'gc_aangle', 'gc_clkwise', 'gc_dangle', + 'gc_inout', 'gc_latlon', 'gc_onarc', 'gc_pnt2gc', 'gc_qarea', + 'gc_tarea', 'generate_2d_array', 'get_color_index', + 'get_color_rgba', 'get_cpu_time', 'get_isolines', 'get_ncl_version', + 'get_script_name', 'get_script_prefix_name', 'get_sphere_radius', + 'get_unique_values', 'getbitsone', 'getenv', 'getfiledimsizes', + 'getfilegrpnames', 'getfilepath', 'getfilevaratts', + 'getfilevarchunkdimsizes', 'getfilevardims', 'getfilevardimsizes', + 'getfilevarnames', 'getfilevartypes', 'getvaratts', 'getvardims', + 'gradsf', 'gradsg', 'greg2jul', 'grid2triple', 'hlsrgb', 'hsvrgb', + 'hydro', 'hyi2hyo', 'idsfft', 'igradsf', 'igradsg', 'ilapsf', + 'ilapsg', 'ilapvf', 'ilapvg', 'ind', 'ind_resolve', 'int2p', + 'int2p_n', 'integertobyte', 'integertochar', 'integertocharacter', + 'integertoshort', 'inttobyte', 'inttochar', 'inttoshort', + 'inverse_matrix', 'isatt', 'isbigendian', 'isbyte', 'ischar', + 'iscoord', 'isdefined', 'isdim', 'isdimnamed', 'isdouble', + 'isenumeric', 'isfile', 'isfilepresent', 'isfilevar', + 'isfilevaratt', 'isfilevarcoord', 'isfilevardim', 'isfloat', + 'isfunc', 'isgraphic', 'isint', 'isint64', 'isinteger', + 'isleapyear', 'islogical', 'islong', 'ismissing', 'isnan_ieee', + 'isnumeric', 'ispan', 'isproc', 'isshort', 'issnumeric', 'isstring', + 'isubyte', 'isuint', 'isuint64', 'isulong', 'isunlimited', + 'isunsigned', 'isushort', 'isvar', 'jul2greg', 'kmeans_as136', + 'kolsm2_n', 'kron_product', 'lapsf', 'lapsg', 'lapvf', 'lapvg', + 'latlon2utm', 'lclvl', 'lderuvf', 'lderuvg', 'linint1', 'linint1_n', + 'linint2', 'linint2_points', 'linmsg', 'linmsg_n', 'linrood_latwgt', + 'linrood_wgt', 'list_files', 'list_filevars', 'list_hlus', + 'list_procfuncs', 'list_vars', 'ListAppend', 'ListCount', + 'ListGetType', 'ListIndex', 'ListIndexFromName', 'ListPop', + 'ListPush', 'ListSetType', 'loadscript', 'local_max', 'local_min', + 'log', 'log10', 'longtobyte', 'longtochar', 'longtocharacter', + 'longtoint', 'longtointeger', 'longtoshort', 'lspoly', 'lspoly_n', + 'mask', 'max', 'maxind', 'min', 'minind', 'mixed_layer_depth', + 'mixhum_ptd', 'mixhum_ptrh', 'mjo_cross_coh2pha', + 'mjo_cross_segment', 'moc_globe_atl', 'monthday', 'natgrid', + 'natgridd', 'natgrids', 'ncargpath', 'ncargversion', 'ndctodata', + 'ndtooned', 'new', 'NewList', 'ngezlogo', 'nggcog', 'nggetp', + 'nglogo', 'ngsetp', 'NhlAddAnnotation', 'NhlAddData', + 'NhlAddOverlay', 'NhlAddPrimitive', 'NhlAppGetDefaultParentId', + 'NhlChangeWorkstation', 'NhlClassName', 'NhlClearWorkstation', + 'NhlDataPolygon', 'NhlDataPolyline', 'NhlDataPolymarker', + 'NhlDataToNDC', 'NhlDestroy', 'NhlDraw', 'NhlFrame', 'NhlFreeColor', + 'NhlGetBB', 'NhlGetClassResources', 'NhlGetErrorObjectId', + 'NhlGetNamedColorIndex', 'NhlGetParentId', + 'NhlGetParentWorkstation', 'NhlGetWorkspaceObjectId', + 'NhlIsAllocatedColor', 'NhlIsApp', 'NhlIsDataComm', 'NhlIsDataItem', + 'NhlIsDataSpec', 'NhlIsTransform', 'NhlIsView', 'NhlIsWorkstation', + 'NhlName', 'NhlNDCPolygon', 'NhlNDCPolyline', 'NhlNDCPolymarker', + 'NhlNDCToData', 'NhlNewColor', 'NhlNewDashPattern', 'NhlNewMarker', + 'NhlPalGetDefined', 'NhlRemoveAnnotation', 'NhlRemoveData', + 'NhlRemoveOverlay', 'NhlRemovePrimitive', 'NhlSetColor', + 'NhlSetDashPattern', 'NhlSetMarker', 'NhlUpdateData', + 'NhlUpdateWorkstation', 'nice_mnmxintvl', 'nngetaspectd', + 'nngetaspects', 'nngetp', 'nngetsloped', 'nngetslopes', 'nngetwts', + 'nngetwtsd', 'nnpnt', 'nnpntd', 'nnpntend', 'nnpntendd', + 'nnpntinit', 'nnpntinitd', 'nnpntinits', 'nnpnts', 'nnsetp', 'num', + 'obj_anal_ic', 'omega_ccm', 'onedtond', 'overlay', 'paleo_outline', + 'pdfxy_bin', 'poisson_grid_fill', 'pop_remap', 'potmp_insitu_ocn', + 'prcwater_dp', 'pres2hybrid', 'pres_hybrid_ccm', 'pres_sigma', + 'print', 'print_table', 'printFileVarSummary', 'printVarSummary', + 'product', 'pslec', 'pslhor', 'pslhyp', 'qsort', 'rand', + 'random_chi', 'random_gamma', 'random_normal', 'random_setallseed', + 'random_uniform', 'rcm2points', 'rcm2rgrid', 'rdsstoi', + 'read_colormap_file', 'reg_multlin', 'regcoef', 'regCoef_n', + 'regline', 'relhum', 'replace_ieeenan', 'reshape', 'reshape_ind', + 'rgba_to_color_index', 'rgbhls', 'rgbhsv', 'rgbyiq', 'rgrid2rcm', + 'rhomb_trunc', 'rip_cape_2d', 'rip_cape_3d', 'round', 'rtest', + 'runave', 'runave_n', 'set_default_fillvalue', 'set_sphere_radius', + 'setfileoption', 'sfvp2uvf', 'sfvp2uvg', 'shaec', 'shagc', + 'shgetnp', 'shgetp', 'shgrid', 'shorttobyte', 'shorttochar', + 'shorttocharacter', 'show_ascii', 'shsec', 'shsetp', 'shsgc', + 'shsgc_R42', 'sigma2hybrid', 'simpeq', 'simpne', 'sin', + 'sindex_yrmo', 'sinh', 'sizeof', 'sleep', 'smth9', 'snindex_yrmo', + 'solve_linsys', 'span_color_indexes', 'span_color_rgba', + 'sparse_matrix_mult', 'spcorr', 'spcorr_n', 'specx_anal', + 'specxy_anal', 'spei', 'sprintf', 'sprinti', 'sqrt', 'sqsort', + 'srand', 'stat2', 'stat4', 'stat_medrng', 'stat_trim', + 'status_exit', 'stdatmus_p2tdz', 'stdatmus_z2tdp', 'stddev', + 'str_capital', 'str_concat', 'str_fields_count', 'str_get_cols', + 'str_get_dq', 'str_get_field', 'str_get_nl', 'str_get_sq', + 'str_get_tab', 'str_index_of_substr', 'str_insert', 'str_is_blank', + 'str_join', 'str_left_strip', 'str_lower', 'str_match', + 'str_match_ic', 'str_match_ic_regex', 'str_match_ind', + 'str_match_ind_ic', 'str_match_ind_ic_regex', 'str_match_ind_regex', + 'str_match_regex', 'str_right_strip', 'str_split', + 'str_split_by_length', 'str_split_csv', 'str_squeeze', 'str_strip', + 'str_sub_str', 'str_switch', 'str_upper', 'stringtochar', + 'stringtocharacter', 'stringtodouble', 'stringtofloat', + 'stringtoint', 'stringtointeger', 'stringtolong', 'stringtoshort', + 'strlen', 'student_t', 'sum', 'svd_lapack', 'svdcov', 'svdcov_sv', + 'svdstd', 'svdstd_sv', 'system', 'systemfunc', 'tan', 'tanh', + 'taper', 'taper_n', 'tdclrs', 'tdctri', 'tdcudp', 'tdcurv', + 'tddtri', 'tdez2d', 'tdez3d', 'tdgetp', 'tdgrds', 'tdgrid', + 'tdgtrs', 'tdinit', 'tditri', 'tdlbla', 'tdlblp', 'tdlbls', + 'tdline', 'tdlndp', 'tdlnpa', 'tdlpdp', 'tdmtri', 'tdotri', + 'tdpara', 'tdplch', 'tdprpa', 'tdprpi', 'tdprpt', 'tdsetp', + 'tdsort', 'tdstri', 'tdstrs', 'tdttri', 'thornthwaite', 'tobyte', + 'tochar', 'todouble', 'tofloat', 'toint', 'toint64', 'tointeger', + 'tolong', 'toshort', 'tosigned', 'tostring', 'tostring_with_format', + 'totype', 'toubyte', 'touint', 'touint64', 'toulong', 'tounsigned', + 'toushort', 'trend_manken', 'tri_trunc', 'triple2grid', + 'triple2grid2d', 'trop_wmo', 'ttest', 'typeof', 'undef', + 'unique_string', 'update', 'ushorttoint', 'ut_calendar', + 'ut_inv_calendar', 'utm2latlon', 'uv2dv_cfd', 'uv2dvf', 'uv2dvg', + 'uv2sfvpf', 'uv2sfvpg', 'uv2vr_cfd', 'uv2vrdvf', 'uv2vrdvg', + 'uv2vrf', 'uv2vrg', 'v5d_close', 'v5d_create', 'v5d_setLowLev', + 'v5d_setUnits', 'v5d_write', 'v5d_write_var', 'variance', 'vhaec', + 'vhagc', 'vhsec', 'vhsgc', 'vibeta', 'vinth2p', 'vinth2p_ecmwf', + 'vinth2p_ecmwf_nodes', 'vinth2p_nodes', 'vintp2p_ecmwf', 'vr2uvf', + 'vr2uvg', 'vrdv2uvf', 'vrdv2uvg', 'wavelet', 'wavelet_default', + 'weibull', 'wgt_area_smooth', 'wgt_areaave', 'wgt_areaave2', + 'wgt_arearmse', 'wgt_arearmse2', 'wgt_areasum2', 'wgt_runave', + 'wgt_runave_n', 'wgt_vert_avg_beta', 'wgt_volave', 'wgt_volave_ccm', + 'wgt_volrmse', 'wgt_volrmse_ccm', 'where', 'wk_smooth121', 'wmbarb', + 'wmbarbmap', 'wmdrft', 'wmgetp', 'wmlabs', 'wmsetp', 'wmstnm', + 'wmvect', 'wmvectmap', 'wmvlbl', 'wrf_avo', 'wrf_cape_2d', + 'wrf_cape_3d', 'wrf_dbz', 'wrf_eth', 'wrf_helicity', 'wrf_ij_to_ll', + 'wrf_interp_1d', 'wrf_interp_2d_xy', 'wrf_interp_3d_z', + 'wrf_latlon_to_ij', 'wrf_ll_to_ij', 'wrf_omega', 'wrf_pvo', + 'wrf_rh', 'wrf_slp', 'wrf_smooth_2d', 'wrf_td', 'wrf_tk', + 'wrf_updraft_helicity', 'wrf_uvmet', 'wrf_virtual_temp', + 'wrf_wetbulb', 'wrf_wps_close_int', 'wrf_wps_open_int', + 'wrf_wps_rddata_int', 'wrf_wps_rdhead_int', 'wrf_wps_read_int', + 'wrf_wps_write_int', 'write_matrix', 'write_table', 'yiqrgb', + 'z2geouv', 'zonal_mpsi', 'addfiles_GetVar', 'advect_variable', + 'area_conserve_remap_Wrap', 'area_hi2lores_Wrap', + 'array_append_record', 'assignFillValue', 'byte2flt', + 'byte2flt_hdf', 'calcDayAnomTLL', 'calcMonAnomLLLT', + 'calcMonAnomLLT', 'calcMonAnomTLL', 'calcMonAnomTLLL', + 'calculate_monthly_values', 'cd_convert', 'changeCase', + 'changeCaseChar', 'clmDayTLL', 'clmDayTLLL', 'clmMon2clmDay', + 'clmMonLLLT', 'clmMonLLT', 'clmMonTLL', 'clmMonTLLL', 'closest_val', + 'copy_VarAtts', 'copy_VarCoords', 'copy_VarCoords_1', + 'copy_VarCoords_2', 'copy_VarMeta', 'copyatt', 'crossp3', + 'cshstringtolist', 'cssgrid_Wrap', 'dble2flt', 'decimalPlaces', + 'delete_VarAtts', 'dim_avg_n_Wrap', 'dim_avg_wgt_n_Wrap', + 'dim_avg_wgt_Wrap', 'dim_avg_Wrap', 'dim_cumsum_n_Wrap', + 'dim_cumsum_Wrap', 'dim_max_n_Wrap', 'dim_min_n_Wrap', + 'dim_rmsd_n_Wrap', 'dim_rmsd_Wrap', 'dim_rmvmean_n_Wrap', + 'dim_rmvmean_Wrap', 'dim_rmvmed_n_Wrap', 'dim_rmvmed_Wrap', + 'dim_standardize_n_Wrap', 'dim_standardize_Wrap', + 'dim_stddev_n_Wrap', 'dim_stddev_Wrap', 'dim_sum_n_Wrap', + 'dim_sum_wgt_n_Wrap', 'dim_sum_wgt_Wrap', 'dim_sum_Wrap', + 'dim_variance_n_Wrap', 'dim_variance_Wrap', 'dpres_plevel_Wrap', + 'dtrend_leftdim', 'dv2uvF_Wrap', 'dv2uvG_Wrap', 'eof_north', + 'eofcor_Wrap', 'eofcov_Wrap', 'eofunc_north', 'eofunc_ts_Wrap', + 'eofunc_varimax_reorder', 'eofunc_varimax_Wrap', 'eofunc_Wrap', + 'epsZero', 'f2fosh_Wrap', 'f2foshv_Wrap', 'f2fsh_Wrap', + 'f2fshv_Wrap', 'f2gsh_Wrap', 'f2gshv_Wrap', 'fbindirSwap', + 'fbinseqSwap1', 'fbinseqSwap2', 'flt2dble', 'flt2string', + 'fo2fsh_Wrap', 'fo2fshv_Wrap', 'g2fsh_Wrap', 'g2fshv_Wrap', + 'g2gsh_Wrap', 'g2gshv_Wrap', 'generate_resample_indices', + 'generate_sample_indices', 'generate_unique_indices', + 'genNormalDist', 'get1Dindex', 'get1Dindex_Collapse', + 'get1Dindex_Exclude', 'get_file_suffix', 'GetFillColor', + 'GetFillColorIndex', 'getFillValue', 'getind_latlon2d', + 'getVarDimNames', 'getVarFillValue', 'grib_stime2itime', + 'hyi2hyo_Wrap', 'ilapsF_Wrap', 'ilapsG_Wrap', 'ind_nearest_coord', + 'indStrSubset', 'int2dble', 'int2flt', 'int2p_n_Wrap', 'int2p_Wrap', + 'isMonotonic', 'isStrSubset', 'latGau', 'latGauWgt', 'latGlobeF', + 'latGlobeFo', 'latRegWgt', 'linint1_n_Wrap', 'linint1_Wrap', + 'linint2_points_Wrap', 'linint2_Wrap', 'local_max_1d', + 'local_min_1d', 'lonFlip', 'lonGlobeF', 'lonGlobeFo', 'lonPivot', + 'merge_levels_sfc', 'mod', 'month_to_annual', + 'month_to_annual_weighted', 'month_to_season', 'month_to_season12', + 'month_to_seasonN', 'monthly_total_to_daily_mean', 'nameDim', + 'natgrid_Wrap', 'NewCosWeight', 'niceLatLon2D', 'NormCosWgtGlobe', + 'numAsciiCol', 'numAsciiRow', 'numeric2int', + 'obj_anal_ic_deprecated', 'obj_anal_ic_Wrap', 'omega_ccm_driver', + 'omega_to_w', 'oneDtostring', 'pack_values', 'pattern_cor', 'pdfx', + 'pdfxy', 'pdfxy_conform', 'pot_temp', 'pot_vort_hybrid', + 'pot_vort_isobaric', 'pres2hybrid_Wrap', 'print_clock', + 'printMinMax', 'quadroots', 'rcm2points_Wrap', 'rcm2rgrid_Wrap', + 'readAsciiHead', 'readAsciiTable', 'reg_multlin_stats', + 'region_ind', 'regline_stats', 'relhum_ttd', 'replaceSingleChar', + 'RGBtoCmap', 'rgrid2rcm_Wrap', 'rho_mwjf', 'rm_single_dims', + 'rmAnnCycle1D', 'rmInsufData', 'rmMonAnnCycLLLT', 'rmMonAnnCycLLT', + 'rmMonAnnCycTLL', 'runave_n_Wrap', 'runave_Wrap', 'short2flt', + 'short2flt_hdf', 'shsgc_R42_Wrap', 'sign_f90', 'sign_matlab', + 'smth9_Wrap', 'smthClmDayTLL', 'smthClmDayTLLL', 'SqrtCosWeight', + 'stat_dispersion', 'static_stability', 'stdMonLLLT', 'stdMonLLT', + 'stdMonTLL', 'stdMonTLLL', 'symMinMaxPlt', 'table_attach_columns', + 'table_attach_rows', 'time_to_newtime', 'transpose', + 'triple2grid_Wrap', 'ut_convert', 'uv2dvF_Wrap', 'uv2dvG_Wrap', + 'uv2vrF_Wrap', 'uv2vrG_Wrap', 'vr2uvF_Wrap', 'vr2uvG_Wrap', + 'w_to_omega', 'wallClockElapseTime', 'wave_number_spc', + 'wgt_areaave_Wrap', 'wgt_runave_leftdim', 'wgt_runave_n_Wrap', + 'wgt_runave_Wrap', 'wgt_vertical_n', 'wind_component', + 'wind_direction', 'yyyyddd_to_yyyymmdd', 'yyyymm_time', + 'yyyymm_to_yyyyfrac', 'yyyymmdd_time', 'yyyymmdd_to_yyyyddd', + 'yyyymmdd_to_yyyyfrac', 'yyyymmddhh_time', 'yyyymmddhh_to_yyyyfrac', + 'zonal_mpsi_Wrap', 'zonalAve', 'calendar_decode2', 'cd_string', + 'kf_filter', 'run_cor', 'time_axis_labels', 'ut_string', + 'wrf_contour', 'wrf_map', 'wrf_map_overlay', 'wrf_map_overlays', + 'wrf_map_resources', 'wrf_map_zoom', 'wrf_overlay', 'wrf_overlays', + 'wrf_user_getvar', 'wrf_user_ij_to_ll', 'wrf_user_intrp2d', + 'wrf_user_intrp3d', 'wrf_user_latlon_to_ij', 'wrf_user_list_times', + 'wrf_user_ll_to_ij', 'wrf_user_unstagger', 'wrf_user_vert_interp', + 'wrf_vector', 'gsn_add_annotation', 'gsn_add_polygon', + 'gsn_add_polyline', 'gsn_add_polymarker', + 'gsn_add_shapefile_polygons', 'gsn_add_shapefile_polylines', + 'gsn_add_shapefile_polymarkers', 'gsn_add_text', 'gsn_attach_plots', + 'gsn_blank_plot', 'gsn_contour', 'gsn_contour_map', + 'gsn_contour_shade', 'gsn_coordinates', 'gsn_create_labelbar', + 'gsn_create_legend', 'gsn_create_text', + 'gsn_csm_attach_zonal_means', 'gsn_csm_blank_plot', + 'gsn_csm_contour', 'gsn_csm_contour_map', 'gsn_csm_contour_map_ce', + 'gsn_csm_contour_map_overlay', 'gsn_csm_contour_map_polar', + 'gsn_csm_hov', 'gsn_csm_lat_time', 'gsn_csm_map', 'gsn_csm_map_ce', + 'gsn_csm_map_polar', 'gsn_csm_pres_hgt', + 'gsn_csm_pres_hgt_streamline', 'gsn_csm_pres_hgt_vector', + 'gsn_csm_streamline', 'gsn_csm_streamline_contour_map', + 'gsn_csm_streamline_contour_map_ce', + 'gsn_csm_streamline_contour_map_polar', 'gsn_csm_streamline_map', + 'gsn_csm_streamline_map_ce', 'gsn_csm_streamline_map_polar', + 'gsn_csm_streamline_scalar', 'gsn_csm_streamline_scalar_map', + 'gsn_csm_streamline_scalar_map_ce', + 'gsn_csm_streamline_scalar_map_polar', 'gsn_csm_time_lat', + 'gsn_csm_vector', 'gsn_csm_vector_map', 'gsn_csm_vector_map_ce', + 'gsn_csm_vector_map_polar', 'gsn_csm_vector_scalar', + 'gsn_csm_vector_scalar_map', 'gsn_csm_vector_scalar_map_ce', + 'gsn_csm_vector_scalar_map_polar', 'gsn_csm_x2y', 'gsn_csm_x2y2', + 'gsn_csm_xy', 'gsn_csm_xy2', 'gsn_csm_xy3', 'gsn_csm_y', + 'gsn_define_colormap', 'gsn_draw_colormap', 'gsn_draw_named_colors', + 'gsn_histogram', 'gsn_labelbar_ndc', 'gsn_legend_ndc', 'gsn_map', + 'gsn_merge_colormaps', 'gsn_open_wks', 'gsn_panel', 'gsn_polygon', + 'gsn_polygon_ndc', 'gsn_polyline', 'gsn_polyline_ndc', + 'gsn_polymarker', 'gsn_polymarker_ndc', 'gsn_retrieve_colormap', + 'gsn_reverse_colormap', 'gsn_streamline', 'gsn_streamline_map', + 'gsn_streamline_scalar', 'gsn_streamline_scalar_map', 'gsn_table', + 'gsn_text', 'gsn_text_ndc', 'gsn_vector', 'gsn_vector_map', + 'gsn_vector_scalar', 'gsn_vector_scalar_map', 'gsn_xy', 'gsn_y', + 'hsv2rgb', 'maximize_output', 'namedcolor2rgb', 'namedcolor2rgba', + 'reset_device_coordinates', 'span_named_colors'), prefix=r'\b'), + Name.Builtin), + + # Resources + (words(( + 'amDataXF', 'amDataYF', 'amJust', 'amOn', 'amOrthogonalPosF', + 'amParallelPosF', 'amResizeNotify', 'amSide', 'amTrackData', + 'amViewId', 'amZone', 'appDefaultParent', 'appFileSuffix', + 'appResources', 'appSysDir', 'appUsrDir', 'caCopyArrays', + 'caXArray', 'caXCast', 'caXMaxV', 'caXMinV', 'caXMissingV', + 'caYArray', 'caYCast', 'caYMaxV', 'caYMinV', 'caYMissingV', + 'cnCellFillEdgeColor', 'cnCellFillMissingValEdgeColor', + 'cnConpackParams', 'cnConstFEnableFill', 'cnConstFLabelAngleF', + 'cnConstFLabelBackgroundColor', 'cnConstFLabelConstantSpacingF', + 'cnConstFLabelFont', 'cnConstFLabelFontAspectF', + 'cnConstFLabelFontColor', 'cnConstFLabelFontHeightF', + 'cnConstFLabelFontQuality', 'cnConstFLabelFontThicknessF', + 'cnConstFLabelFormat', 'cnConstFLabelFuncCode', 'cnConstFLabelJust', + 'cnConstFLabelOn', 'cnConstFLabelOrthogonalPosF', + 'cnConstFLabelParallelPosF', 'cnConstFLabelPerimColor', + 'cnConstFLabelPerimOn', 'cnConstFLabelPerimSpaceF', + 'cnConstFLabelPerimThicknessF', 'cnConstFLabelSide', + 'cnConstFLabelString', 'cnConstFLabelTextDirection', + 'cnConstFLabelZone', 'cnConstFUseInfoLabelRes', + 'cnExplicitLabelBarLabelsOn', 'cnExplicitLegendLabelsOn', + 'cnExplicitLineLabelsOn', 'cnFillBackgroundColor', 'cnFillColor', + 'cnFillColors', 'cnFillDotSizeF', 'cnFillDrawOrder', 'cnFillMode', + 'cnFillOn', 'cnFillOpacityF', 'cnFillPalette', 'cnFillPattern', + 'cnFillPatterns', 'cnFillScaleF', 'cnFillScales', 'cnFixFillBleed', + 'cnGridBoundFillColor', 'cnGridBoundFillPattern', + 'cnGridBoundFillScaleF', 'cnGridBoundPerimColor', + 'cnGridBoundPerimDashPattern', 'cnGridBoundPerimOn', + 'cnGridBoundPerimThicknessF', 'cnHighLabelAngleF', + 'cnHighLabelBackgroundColor', 'cnHighLabelConstantSpacingF', + 'cnHighLabelCount', 'cnHighLabelFont', 'cnHighLabelFontAspectF', + 'cnHighLabelFontColor', 'cnHighLabelFontHeightF', + 'cnHighLabelFontQuality', 'cnHighLabelFontThicknessF', + 'cnHighLabelFormat', 'cnHighLabelFuncCode', 'cnHighLabelPerimColor', + 'cnHighLabelPerimOn', 'cnHighLabelPerimSpaceF', + 'cnHighLabelPerimThicknessF', 'cnHighLabelString', 'cnHighLabelsOn', + 'cnHighLowLabelOverlapMode', 'cnHighUseLineLabelRes', + 'cnInfoLabelAngleF', 'cnInfoLabelBackgroundColor', + 'cnInfoLabelConstantSpacingF', 'cnInfoLabelFont', + 'cnInfoLabelFontAspectF', 'cnInfoLabelFontColor', + 'cnInfoLabelFontHeightF', 'cnInfoLabelFontQuality', + 'cnInfoLabelFontThicknessF', 'cnInfoLabelFormat', + 'cnInfoLabelFuncCode', 'cnInfoLabelJust', 'cnInfoLabelOn', + 'cnInfoLabelOrthogonalPosF', 'cnInfoLabelParallelPosF', + 'cnInfoLabelPerimColor', 'cnInfoLabelPerimOn', + 'cnInfoLabelPerimSpaceF', 'cnInfoLabelPerimThicknessF', + 'cnInfoLabelSide', 'cnInfoLabelString', 'cnInfoLabelTextDirection', + 'cnInfoLabelZone', 'cnLabelBarEndLabelsOn', 'cnLabelBarEndStyle', + 'cnLabelDrawOrder', 'cnLabelMasking', 'cnLabelScaleFactorF', + 'cnLabelScaleValueF', 'cnLabelScalingMode', 'cnLegendLevelFlags', + 'cnLevelCount', 'cnLevelFlag', 'cnLevelFlags', 'cnLevelSelectionMode', + 'cnLevelSpacingF', 'cnLevels', 'cnLineColor', 'cnLineColors', + 'cnLineDashPattern', 'cnLineDashPatterns', 'cnLineDashSegLenF', + 'cnLineDrawOrder', 'cnLineLabelAngleF', 'cnLineLabelBackgroundColor', + 'cnLineLabelConstantSpacingF', 'cnLineLabelCount', + 'cnLineLabelDensityF', 'cnLineLabelFont', 'cnLineLabelFontAspectF', + 'cnLineLabelFontColor', 'cnLineLabelFontColors', + 'cnLineLabelFontHeightF', 'cnLineLabelFontQuality', + 'cnLineLabelFontThicknessF', 'cnLineLabelFormat', + 'cnLineLabelFuncCode', 'cnLineLabelInterval', 'cnLineLabelPerimColor', + 'cnLineLabelPerimOn', 'cnLineLabelPerimSpaceF', + 'cnLineLabelPerimThicknessF', 'cnLineLabelPlacementMode', + 'cnLineLabelStrings', 'cnLineLabelsOn', 'cnLinePalette', + 'cnLineThicknessF', 'cnLineThicknesses', 'cnLinesOn', + 'cnLowLabelAngleF', 'cnLowLabelBackgroundColor', + 'cnLowLabelConstantSpacingF', 'cnLowLabelCount', 'cnLowLabelFont', + 'cnLowLabelFontAspectF', 'cnLowLabelFontColor', + 'cnLowLabelFontHeightF', 'cnLowLabelFontQuality', + 'cnLowLabelFontThicknessF', 'cnLowLabelFormat', 'cnLowLabelFuncCode', + 'cnLowLabelPerimColor', 'cnLowLabelPerimOn', 'cnLowLabelPerimSpaceF', + 'cnLowLabelPerimThicknessF', 'cnLowLabelString', 'cnLowLabelsOn', + 'cnLowUseHighLabelRes', 'cnMaxDataValueFormat', 'cnMaxLevelCount', + 'cnMaxLevelValF', 'cnMaxPointDistanceF', 'cnMinLevelValF', + 'cnMissingValFillColor', 'cnMissingValFillPattern', + 'cnMissingValFillScaleF', 'cnMissingValPerimColor', + 'cnMissingValPerimDashPattern', 'cnMissingValPerimGridBoundOn', + 'cnMissingValPerimOn', 'cnMissingValPerimThicknessF', + 'cnMonoFillColor', 'cnMonoFillPattern', 'cnMonoFillScale', + 'cnMonoLevelFlag', 'cnMonoLineColor', 'cnMonoLineDashPattern', + 'cnMonoLineLabelFontColor', 'cnMonoLineThickness', 'cnNoDataLabelOn', + 'cnNoDataLabelString', 'cnOutOfRangeFillColor', + 'cnOutOfRangeFillPattern', 'cnOutOfRangeFillScaleF', + 'cnOutOfRangePerimColor', 'cnOutOfRangePerimDashPattern', + 'cnOutOfRangePerimOn', 'cnOutOfRangePerimThicknessF', + 'cnRasterCellSizeF', 'cnRasterMinCellSizeF', 'cnRasterModeOn', + 'cnRasterSampleFactorF', 'cnRasterSmoothingOn', 'cnScalarFieldData', + 'cnSmoothingDistanceF', 'cnSmoothingOn', 'cnSmoothingTensionF', + 'cnSpanFillPalette', 'cnSpanLinePalette', 'ctCopyTables', + 'ctXElementSize', 'ctXMaxV', 'ctXMinV', 'ctXMissingV', 'ctXTable', + 'ctXTableLengths', 'ctXTableType', 'ctYElementSize', 'ctYMaxV', + 'ctYMinV', 'ctYMissingV', 'ctYTable', 'ctYTableLengths', + 'ctYTableType', 'dcDelayCompute', 'errBuffer', + 'errFileName', 'errFilePtr', 'errLevel', 'errPrint', 'errUnitNumber', + 'gsClipOn', 'gsColors', 'gsEdgeColor', 'gsEdgeDashPattern', + 'gsEdgeDashSegLenF', 'gsEdgeThicknessF', 'gsEdgesOn', + 'gsFillBackgroundColor', 'gsFillColor', 'gsFillDotSizeF', + 'gsFillIndex', 'gsFillLineThicknessF', 'gsFillOpacityF', + 'gsFillScaleF', 'gsFont', 'gsFontAspectF', 'gsFontColor', + 'gsFontHeightF', 'gsFontOpacityF', 'gsFontQuality', + 'gsFontThicknessF', 'gsLineColor', 'gsLineDashPattern', + 'gsLineDashSegLenF', 'gsLineLabelConstantSpacingF', 'gsLineLabelFont', + 'gsLineLabelFontAspectF', 'gsLineLabelFontColor', + 'gsLineLabelFontHeightF', 'gsLineLabelFontQuality', + 'gsLineLabelFontThicknessF', 'gsLineLabelFuncCode', + 'gsLineLabelString', 'gsLineOpacityF', 'gsLineThicknessF', + 'gsMarkerColor', 'gsMarkerIndex', 'gsMarkerOpacityF', 'gsMarkerSizeF', + 'gsMarkerThicknessF', 'gsSegments', 'gsTextAngleF', + 'gsTextConstantSpacingF', 'gsTextDirection', 'gsTextFuncCode', + 'gsTextJustification', 'gsnAboveYRefLineBarColors', + 'gsnAboveYRefLineBarFillScales', 'gsnAboveYRefLineBarPatterns', + 'gsnAboveYRefLineColor', 'gsnAddCyclic', 'gsnAttachBorderOn', + 'gsnAttachPlotsXAxis', 'gsnBelowYRefLineBarColors', + 'gsnBelowYRefLineBarFillScales', 'gsnBelowYRefLineBarPatterns', + 'gsnBelowYRefLineColor', 'gsnBoxMargin', 'gsnCenterString', + 'gsnCenterStringFontColor', 'gsnCenterStringFontHeightF', + 'gsnCenterStringFuncCode', 'gsnCenterStringOrthogonalPosF', + 'gsnCenterStringParallelPosF', 'gsnContourLineThicknessesScale', + 'gsnContourNegLineDashPattern', 'gsnContourPosLineDashPattern', + 'gsnContourZeroLineThicknessF', 'gsnDebugWriteFileName', 'gsnDraw', + 'gsnFrame', 'gsnHistogramBarWidthPercent', 'gsnHistogramBinIntervals', + 'gsnHistogramBinMissing', 'gsnHistogramBinWidth', + 'gsnHistogramClassIntervals', 'gsnHistogramCompare', + 'gsnHistogramComputePercentages', + 'gsnHistogramComputePercentagesNoMissing', + 'gsnHistogramDiscreteBinValues', 'gsnHistogramDiscreteClassValues', + 'gsnHistogramHorizontal', 'gsnHistogramMinMaxBinsOn', + 'gsnHistogramNumberOfBins', 'gsnHistogramPercentSign', + 'gsnHistogramSelectNiceIntervals', 'gsnLeftString', + 'gsnLeftStringFontColor', 'gsnLeftStringFontHeightF', + 'gsnLeftStringFuncCode', 'gsnLeftStringOrthogonalPosF', + 'gsnLeftStringParallelPosF', 'gsnMajorLatSpacing', + 'gsnMajorLonSpacing', 'gsnMaskLambertConformal', + 'gsnMaskLambertConformalOutlineOn', 'gsnMaximize', + 'gsnMinorLatSpacing', 'gsnMinorLonSpacing', 'gsnPanelBottom', + 'gsnPanelCenter', 'gsnPanelDebug', 'gsnPanelFigureStrings', + 'gsnPanelFigureStringsBackgroundFillColor', + 'gsnPanelFigureStringsFontHeightF', 'gsnPanelFigureStringsJust', + 'gsnPanelFigureStringsPerimOn', 'gsnPanelLabelBar', 'gsnPanelLeft', + 'gsnPanelMainFont', 'gsnPanelMainFontColor', + 'gsnPanelMainFontHeightF', 'gsnPanelMainString', 'gsnPanelRight', + 'gsnPanelRowSpec', 'gsnPanelScalePlotIndex', 'gsnPanelTop', + 'gsnPanelXF', 'gsnPanelXWhiteSpacePercent', 'gsnPanelYF', + 'gsnPanelYWhiteSpacePercent', 'gsnPaperHeight', 'gsnPaperMargin', + 'gsnPaperOrientation', 'gsnPaperWidth', 'gsnPolar', + 'gsnPolarLabelDistance', 'gsnPolarLabelFont', + 'gsnPolarLabelFontHeightF', 'gsnPolarLabelSpacing', 'gsnPolarTime', + 'gsnPolarUT', 'gsnRightString', 'gsnRightStringFontColor', + 'gsnRightStringFontHeightF', 'gsnRightStringFuncCode', + 'gsnRightStringOrthogonalPosF', 'gsnRightStringParallelPosF', + 'gsnScalarContour', 'gsnScale', 'gsnShape', 'gsnSpreadColorEnd', + 'gsnSpreadColorStart', 'gsnSpreadColors', 'gsnStringFont', + 'gsnStringFontColor', 'gsnStringFontHeightF', 'gsnStringFuncCode', + 'gsnTickMarksOn', 'gsnXAxisIrregular2Linear', 'gsnXAxisIrregular2Log', + 'gsnXRefLine', 'gsnXRefLineColor', 'gsnXRefLineDashPattern', + 'gsnXRefLineThicknessF', 'gsnXYAboveFillColors', 'gsnXYBarChart', + 'gsnXYBarChartBarWidth', 'gsnXYBarChartColors', + 'gsnXYBarChartColors2', 'gsnXYBarChartFillDotSizeF', + 'gsnXYBarChartFillLineThicknessF', 'gsnXYBarChartFillOpacityF', + 'gsnXYBarChartFillScaleF', 'gsnXYBarChartOutlineOnly', + 'gsnXYBarChartOutlineThicknessF', 'gsnXYBarChartPatterns', + 'gsnXYBarChartPatterns2', 'gsnXYBelowFillColors', 'gsnXYFillColors', + 'gsnXYFillOpacities', 'gsnXYLeftFillColors', 'gsnXYRightFillColors', + 'gsnYAxisIrregular2Linear', 'gsnYAxisIrregular2Log', 'gsnYRefLine', + 'gsnYRefLineColor', 'gsnYRefLineColors', 'gsnYRefLineDashPattern', + 'gsnYRefLineDashPatterns', 'gsnYRefLineThicknessF', + 'gsnYRefLineThicknesses', 'gsnZonalMean', 'gsnZonalMeanXMaxF', + 'gsnZonalMeanXMinF', 'gsnZonalMeanYRefLine', 'lbAutoManage', + 'lbBottomMarginF', 'lbBoxCount', 'lbBoxEndCapStyle', 'lbBoxFractions', + 'lbBoxLineColor', 'lbBoxLineDashPattern', 'lbBoxLineDashSegLenF', + 'lbBoxLineThicknessF', 'lbBoxLinesOn', 'lbBoxMajorExtentF', + 'lbBoxMinorExtentF', 'lbBoxSeparatorLinesOn', 'lbBoxSizing', + 'lbFillBackground', 'lbFillColor', 'lbFillColors', 'lbFillDotSizeF', + 'lbFillLineThicknessF', 'lbFillPattern', 'lbFillPatterns', + 'lbFillScaleF', 'lbFillScales', 'lbJustification', 'lbLabelAlignment', + 'lbLabelAngleF', 'lbLabelAutoStride', 'lbLabelBarOn', + 'lbLabelConstantSpacingF', 'lbLabelDirection', 'lbLabelFont', + 'lbLabelFontAspectF', 'lbLabelFontColor', 'lbLabelFontHeightF', + 'lbLabelFontQuality', 'lbLabelFontThicknessF', 'lbLabelFuncCode', + 'lbLabelJust', 'lbLabelOffsetF', 'lbLabelPosition', 'lbLabelStride', + 'lbLabelStrings', 'lbLabelsOn', 'lbLeftMarginF', 'lbMaxLabelLenF', + 'lbMinLabelSpacingF', 'lbMonoFillColor', 'lbMonoFillPattern', + 'lbMonoFillScale', 'lbOrientation', 'lbPerimColor', + 'lbPerimDashPattern', 'lbPerimDashSegLenF', 'lbPerimFill', + 'lbPerimFillColor', 'lbPerimOn', 'lbPerimThicknessF', + 'lbRasterFillOn', 'lbRightMarginF', 'lbTitleAngleF', + 'lbTitleConstantSpacingF', 'lbTitleDirection', 'lbTitleExtentF', + 'lbTitleFont', 'lbTitleFontAspectF', 'lbTitleFontColor', + 'lbTitleFontHeightF', 'lbTitleFontQuality', 'lbTitleFontThicknessF', + 'lbTitleFuncCode', 'lbTitleJust', 'lbTitleOffsetF', 'lbTitleOn', + 'lbTitlePosition', 'lbTitleString', 'lbTopMarginF', 'lgAutoManage', + 'lgBottomMarginF', 'lgBoxBackground', 'lgBoxLineColor', + 'lgBoxLineDashPattern', 'lgBoxLineDashSegLenF', 'lgBoxLineThicknessF', + 'lgBoxLinesOn', 'lgBoxMajorExtentF', 'lgBoxMinorExtentF', + 'lgDashIndex', 'lgDashIndexes', 'lgItemCount', 'lgItemOrder', + 'lgItemPlacement', 'lgItemPositions', 'lgItemType', 'lgItemTypes', + 'lgJustification', 'lgLabelAlignment', 'lgLabelAngleF', + 'lgLabelAutoStride', 'lgLabelConstantSpacingF', 'lgLabelDirection', + 'lgLabelFont', 'lgLabelFontAspectF', 'lgLabelFontColor', + 'lgLabelFontHeightF', 'lgLabelFontQuality', 'lgLabelFontThicknessF', + 'lgLabelFuncCode', 'lgLabelJust', 'lgLabelOffsetF', 'lgLabelPosition', + 'lgLabelStride', 'lgLabelStrings', 'lgLabelsOn', 'lgLeftMarginF', + 'lgLegendOn', 'lgLineColor', 'lgLineColors', 'lgLineDashSegLenF', + 'lgLineDashSegLens', 'lgLineLabelConstantSpacingF', 'lgLineLabelFont', + 'lgLineLabelFontAspectF', 'lgLineLabelFontColor', + 'lgLineLabelFontColors', 'lgLineLabelFontHeightF', + 'lgLineLabelFontHeights', 'lgLineLabelFontQuality', + 'lgLineLabelFontThicknessF', 'lgLineLabelFuncCode', + 'lgLineLabelStrings', 'lgLineLabelsOn', 'lgLineThicknessF', + 'lgLineThicknesses', 'lgMarkerColor', 'lgMarkerColors', + 'lgMarkerIndex', 'lgMarkerIndexes', 'lgMarkerSizeF', 'lgMarkerSizes', + 'lgMarkerThicknessF', 'lgMarkerThicknesses', 'lgMonoDashIndex', + 'lgMonoItemType', 'lgMonoLineColor', 'lgMonoLineDashSegLen', + 'lgMonoLineLabelFontColor', 'lgMonoLineLabelFontHeight', + 'lgMonoLineThickness', 'lgMonoMarkerColor', 'lgMonoMarkerIndex', + 'lgMonoMarkerSize', 'lgMonoMarkerThickness', 'lgOrientation', + 'lgPerimColor', 'lgPerimDashPattern', 'lgPerimDashSegLenF', + 'lgPerimFill', 'lgPerimFillColor', 'lgPerimOn', 'lgPerimThicknessF', + 'lgRightMarginF', 'lgTitleAngleF', 'lgTitleConstantSpacingF', + 'lgTitleDirection', 'lgTitleExtentF', 'lgTitleFont', + 'lgTitleFontAspectF', 'lgTitleFontColor', 'lgTitleFontHeightF', + 'lgTitleFontQuality', 'lgTitleFontThicknessF', 'lgTitleFuncCode', + 'lgTitleJust', 'lgTitleOffsetF', 'lgTitleOn', 'lgTitlePosition', + 'lgTitleString', 'lgTopMarginF', 'mpAreaGroupCount', + 'mpAreaMaskingOn', 'mpAreaNames', 'mpAreaTypes', 'mpBottomAngleF', + 'mpBottomMapPosF', 'mpBottomNDCF', 'mpBottomNPCF', + 'mpBottomPointLatF', 'mpBottomPointLonF', 'mpBottomWindowF', + 'mpCenterLatF', 'mpCenterLonF', 'mpCenterRotF', 'mpCountyLineColor', + 'mpCountyLineDashPattern', 'mpCountyLineDashSegLenF', + 'mpCountyLineThicknessF', 'mpDataBaseVersion', 'mpDataResolution', + 'mpDataSetName', 'mpDefaultFillColor', 'mpDefaultFillPattern', + 'mpDefaultFillScaleF', 'mpDynamicAreaGroups', 'mpEllipticalBoundary', + 'mpFillAreaSpecifiers', 'mpFillBoundarySets', 'mpFillColor', + 'mpFillColors', 'mpFillColors-default', 'mpFillDotSizeF', + 'mpFillDrawOrder', 'mpFillOn', 'mpFillPatternBackground', + 'mpFillPattern', 'mpFillPatterns', 'mpFillPatterns-default', + 'mpFillScaleF', 'mpFillScales', 'mpFillScales-default', + 'mpFixedAreaGroups', 'mpGeophysicalLineColor', + 'mpGeophysicalLineDashPattern', 'mpGeophysicalLineDashSegLenF', + 'mpGeophysicalLineThicknessF', 'mpGreatCircleLinesOn', + 'mpGridAndLimbDrawOrder', 'mpGridAndLimbOn', 'mpGridLatSpacingF', + 'mpGridLineColor', 'mpGridLineDashPattern', 'mpGridLineDashSegLenF', + 'mpGridLineThicknessF', 'mpGridLonSpacingF', 'mpGridMaskMode', + 'mpGridMaxLatF', 'mpGridPolarLonSpacingF', 'mpGridSpacingF', + 'mpInlandWaterFillColor', 'mpInlandWaterFillPattern', + 'mpInlandWaterFillScaleF', 'mpLabelDrawOrder', 'mpLabelFontColor', + 'mpLabelFontHeightF', 'mpLabelsOn', 'mpLambertMeridianF', + 'mpLambertParallel1F', 'mpLambertParallel2F', 'mpLandFillColor', + 'mpLandFillPattern', 'mpLandFillScaleF', 'mpLeftAngleF', + 'mpLeftCornerLatF', 'mpLeftCornerLonF', 'mpLeftMapPosF', + 'mpLeftNDCF', 'mpLeftNPCF', 'mpLeftPointLatF', + 'mpLeftPointLonF', 'mpLeftWindowF', 'mpLimbLineColor', + 'mpLimbLineDashPattern', 'mpLimbLineDashSegLenF', + 'mpLimbLineThicknessF', 'mpLimitMode', 'mpMaskAreaSpecifiers', + 'mpMaskOutlineSpecifiers', 'mpMaxLatF', 'mpMaxLonF', + 'mpMinLatF', 'mpMinLonF', 'mpMonoFillColor', 'mpMonoFillPattern', + 'mpMonoFillScale', 'mpNationalLineColor', 'mpNationalLineDashPattern', + 'mpNationalLineThicknessF', 'mpOceanFillColor', 'mpOceanFillPattern', + 'mpOceanFillScaleF', 'mpOutlineBoundarySets', 'mpOutlineDrawOrder', + 'mpOutlineMaskingOn', 'mpOutlineOn', 'mpOutlineSpecifiers', + 'mpPerimDrawOrder', 'mpPerimLineColor', 'mpPerimLineDashPattern', + 'mpPerimLineDashSegLenF', 'mpPerimLineThicknessF', 'mpPerimOn', + 'mpPolyMode', 'mpProjection', 'mpProvincialLineColor', + 'mpProvincialLineDashPattern', 'mpProvincialLineDashSegLenF', + 'mpProvincialLineThicknessF', 'mpRelativeCenterLat', + 'mpRelativeCenterLon', 'mpRightAngleF', 'mpRightCornerLatF', + 'mpRightCornerLonF', 'mpRightMapPosF', 'mpRightNDCF', + 'mpRightNPCF', 'mpRightPointLatF', 'mpRightPointLonF', + 'mpRightWindowF', 'mpSatelliteAngle1F', 'mpSatelliteAngle2F', + 'mpSatelliteDistF', 'mpShapeMode', 'mpSpecifiedFillColors', + 'mpSpecifiedFillDirectIndexing', 'mpSpecifiedFillPatterns', + 'mpSpecifiedFillPriority', 'mpSpecifiedFillScales', + 'mpTopAngleF', 'mpTopMapPosF', 'mpTopNDCF', 'mpTopNPCF', + 'mpTopPointLatF', 'mpTopPointLonF', 'mpTopWindowF', + 'mpUSStateLineColor', 'mpUSStateLineDashPattern', + 'mpUSStateLineDashSegLenF', 'mpUSStateLineThicknessF', + 'pmAnnoManagers', 'pmAnnoViews', 'pmLabelBarDisplayMode', + 'pmLabelBarHeightF', 'pmLabelBarKeepAspect', 'pmLabelBarOrthogonalPosF', + 'pmLabelBarParallelPosF', 'pmLabelBarSide', 'pmLabelBarWidthF', + 'pmLabelBarZone', 'pmLegendDisplayMode', 'pmLegendHeightF', + 'pmLegendKeepAspect', 'pmLegendOrthogonalPosF', + 'pmLegendParallelPosF', 'pmLegendSide', 'pmLegendWidthF', + 'pmLegendZone', 'pmOverlaySequenceIds', 'pmTickMarkDisplayMode', + 'pmTickMarkZone', 'pmTitleDisplayMode', 'pmTitleZone', + 'prGraphicStyle', 'prPolyType', 'prXArray', 'prYArray', + 'sfCopyData', 'sfDataArray', 'sfDataMaxV', 'sfDataMinV', + 'sfElementNodes', 'sfExchangeDimensions', 'sfFirstNodeIndex', + 'sfMissingValueV', 'sfXArray', 'sfXCActualEndF', 'sfXCActualStartF', + 'sfXCEndIndex', 'sfXCEndSubsetV', 'sfXCEndV', 'sfXCStartIndex', + 'sfXCStartSubsetV', 'sfXCStartV', 'sfXCStride', 'sfXCellBounds', + 'sfYArray', 'sfYCActualEndF', 'sfYCActualStartF', 'sfYCEndIndex', + 'sfYCEndSubsetV', 'sfYCEndV', 'sfYCStartIndex', 'sfYCStartSubsetV', + 'sfYCStartV', 'sfYCStride', 'sfYCellBounds', 'stArrowLengthF', + 'stArrowStride', 'stCrossoverCheckCount', + 'stExplicitLabelBarLabelsOn', 'stLabelBarEndLabelsOn', + 'stLabelFormat', 'stLengthCheckCount', 'stLevelColors', + 'stLevelCount', 'stLevelPalette', 'stLevelSelectionMode', + 'stLevelSpacingF', 'stLevels', 'stLineColor', 'stLineOpacityF', + 'stLineStartStride', 'stLineThicknessF', 'stMapDirection', + 'stMaxLevelCount', 'stMaxLevelValF', 'stMinArrowSpacingF', + 'stMinDistanceF', 'stMinLevelValF', 'stMinLineSpacingF', + 'stMinStepFactorF', 'stMonoLineColor', 'stNoDataLabelOn', + 'stNoDataLabelString', 'stScalarFieldData', 'stScalarMissingValColor', + 'stSpanLevelPalette', 'stStepSizeF', 'stStreamlineDrawOrder', + 'stUseScalarArray', 'stVectorFieldData', 'stZeroFLabelAngleF', + 'stZeroFLabelBackgroundColor', 'stZeroFLabelConstantSpacingF', + 'stZeroFLabelFont', 'stZeroFLabelFontAspectF', + 'stZeroFLabelFontColor', 'stZeroFLabelFontHeightF', + 'stZeroFLabelFontQuality', 'stZeroFLabelFontThicknessF', + 'stZeroFLabelFuncCode', 'stZeroFLabelJust', 'stZeroFLabelOn', + 'stZeroFLabelOrthogonalPosF', 'stZeroFLabelParallelPosF', + 'stZeroFLabelPerimColor', 'stZeroFLabelPerimOn', + 'stZeroFLabelPerimSpaceF', 'stZeroFLabelPerimThicknessF', + 'stZeroFLabelSide', 'stZeroFLabelString', 'stZeroFLabelTextDirection', + 'stZeroFLabelZone', 'tfDoNDCOverlay', 'tfPlotManagerOn', + 'tfPolyDrawList', 'tfPolyDrawOrder', 'tiDeltaF', 'tiMainAngleF', + 'tiMainConstantSpacingF', 'tiMainDirection', 'tiMainFont', + 'tiMainFontAspectF', 'tiMainFontColor', 'tiMainFontHeightF', + 'tiMainFontQuality', 'tiMainFontThicknessF', 'tiMainFuncCode', + 'tiMainJust', 'tiMainOffsetXF', 'tiMainOffsetYF', 'tiMainOn', + 'tiMainPosition', 'tiMainSide', 'tiMainString', 'tiUseMainAttributes', + 'tiXAxisAngleF', 'tiXAxisConstantSpacingF', 'tiXAxisDirection', + 'tiXAxisFont', 'tiXAxisFontAspectF', 'tiXAxisFontColor', + 'tiXAxisFontHeightF', 'tiXAxisFontQuality', 'tiXAxisFontThicknessF', + 'tiXAxisFuncCode', 'tiXAxisJust', 'tiXAxisOffsetXF', + 'tiXAxisOffsetYF', 'tiXAxisOn', 'tiXAxisPosition', 'tiXAxisSide', + 'tiXAxisString', 'tiYAxisAngleF', 'tiYAxisConstantSpacingF', + 'tiYAxisDirection', 'tiYAxisFont', 'tiYAxisFontAspectF', + 'tiYAxisFontColor', 'tiYAxisFontHeightF', 'tiYAxisFontQuality', + 'tiYAxisFontThicknessF', 'tiYAxisFuncCode', 'tiYAxisJust', + 'tiYAxisOffsetXF', 'tiYAxisOffsetYF', 'tiYAxisOn', 'tiYAxisPosition', + 'tiYAxisSide', 'tiYAxisString', 'tmBorderLineColor', + 'tmBorderThicknessF', 'tmEqualizeXYSizes', 'tmLabelAutoStride', + 'tmSciNoteCutoff', 'tmXBAutoPrecision', 'tmXBBorderOn', + 'tmXBDataLeftF', 'tmXBDataRightF', 'tmXBFormat', 'tmXBIrrTensionF', + 'tmXBIrregularPoints', 'tmXBLabelAngleF', 'tmXBLabelConstantSpacingF', + 'tmXBLabelDeltaF', 'tmXBLabelDirection', 'tmXBLabelFont', + 'tmXBLabelFontAspectF', 'tmXBLabelFontColor', 'tmXBLabelFontHeightF', + 'tmXBLabelFontQuality', 'tmXBLabelFontThicknessF', + 'tmXBLabelFuncCode', 'tmXBLabelJust', 'tmXBLabelStride', 'tmXBLabels', + 'tmXBLabelsOn', 'tmXBMajorLengthF', 'tmXBMajorLineColor', + 'tmXBMajorOutwardLengthF', 'tmXBMajorThicknessF', 'tmXBMaxLabelLenF', + 'tmXBMaxTicks', 'tmXBMinLabelSpacingF', 'tmXBMinorLengthF', + 'tmXBMinorLineColor', 'tmXBMinorOn', 'tmXBMinorOutwardLengthF', + 'tmXBMinorPerMajor', 'tmXBMinorThicknessF', 'tmXBMinorValues', + 'tmXBMode', 'tmXBOn', 'tmXBPrecision', 'tmXBStyle', 'tmXBTickEndF', + 'tmXBTickSpacingF', 'tmXBTickStartF', 'tmXBValues', 'tmXMajorGrid', + 'tmXMajorGridLineColor', 'tmXMajorGridLineDashPattern', + 'tmXMajorGridThicknessF', 'tmXMinorGrid', 'tmXMinorGridLineColor', + 'tmXMinorGridLineDashPattern', 'tmXMinorGridThicknessF', + 'tmXTAutoPrecision', 'tmXTBorderOn', 'tmXTDataLeftF', + 'tmXTDataRightF', 'tmXTFormat', 'tmXTIrrTensionF', + 'tmXTIrregularPoints', 'tmXTLabelAngleF', 'tmXTLabelConstantSpacingF', + 'tmXTLabelDeltaF', 'tmXTLabelDirection', 'tmXTLabelFont', + 'tmXTLabelFontAspectF', 'tmXTLabelFontColor', 'tmXTLabelFontHeightF', + 'tmXTLabelFontQuality', 'tmXTLabelFontThicknessF', + 'tmXTLabelFuncCode', 'tmXTLabelJust', 'tmXTLabelStride', 'tmXTLabels', + 'tmXTLabelsOn', 'tmXTMajorLengthF', 'tmXTMajorLineColor', + 'tmXTMajorOutwardLengthF', 'tmXTMajorThicknessF', 'tmXTMaxLabelLenF', + 'tmXTMaxTicks', 'tmXTMinLabelSpacingF', 'tmXTMinorLengthF', + 'tmXTMinorLineColor', 'tmXTMinorOn', 'tmXTMinorOutwardLengthF', + 'tmXTMinorPerMajor', 'tmXTMinorThicknessF', 'tmXTMinorValues', + 'tmXTMode', 'tmXTOn', 'tmXTPrecision', 'tmXTStyle', 'tmXTTickEndF', + 'tmXTTickSpacingF', 'tmXTTickStartF', 'tmXTValues', 'tmXUseBottom', + 'tmYLAutoPrecision', 'tmYLBorderOn', 'tmYLDataBottomF', + 'tmYLDataTopF', 'tmYLFormat', 'tmYLIrrTensionF', + 'tmYLIrregularPoints', 'tmYLLabelAngleF', 'tmYLLabelConstantSpacingF', + 'tmYLLabelDeltaF', 'tmYLLabelDirection', 'tmYLLabelFont', + 'tmYLLabelFontAspectF', 'tmYLLabelFontColor', 'tmYLLabelFontHeightF', + 'tmYLLabelFontQuality', 'tmYLLabelFontThicknessF', + 'tmYLLabelFuncCode', 'tmYLLabelJust', 'tmYLLabelStride', 'tmYLLabels', + 'tmYLLabelsOn', 'tmYLMajorLengthF', 'tmYLMajorLineColor', + 'tmYLMajorOutwardLengthF', 'tmYLMajorThicknessF', 'tmYLMaxLabelLenF', + 'tmYLMaxTicks', 'tmYLMinLabelSpacingF', 'tmYLMinorLengthF', + 'tmYLMinorLineColor', 'tmYLMinorOn', 'tmYLMinorOutwardLengthF', + 'tmYLMinorPerMajor', 'tmYLMinorThicknessF', 'tmYLMinorValues', + 'tmYLMode', 'tmYLOn', 'tmYLPrecision', 'tmYLStyle', 'tmYLTickEndF', + 'tmYLTickSpacingF', 'tmYLTickStartF', 'tmYLValues', 'tmYMajorGrid', + 'tmYMajorGridLineColor', 'tmYMajorGridLineDashPattern', + 'tmYMajorGridThicknessF', 'tmYMinorGrid', 'tmYMinorGridLineColor', + 'tmYMinorGridLineDashPattern', 'tmYMinorGridThicknessF', + 'tmYRAutoPrecision', 'tmYRBorderOn', 'tmYRDataBottomF', + 'tmYRDataTopF', 'tmYRFormat', 'tmYRIrrTensionF', + 'tmYRIrregularPoints', 'tmYRLabelAngleF', 'tmYRLabelConstantSpacingF', + 'tmYRLabelDeltaF', 'tmYRLabelDirection', 'tmYRLabelFont', + 'tmYRLabelFontAspectF', 'tmYRLabelFontColor', 'tmYRLabelFontHeightF', + 'tmYRLabelFontQuality', 'tmYRLabelFontThicknessF', + 'tmYRLabelFuncCode', 'tmYRLabelJust', 'tmYRLabelStride', 'tmYRLabels', + 'tmYRLabelsOn', 'tmYRMajorLengthF', 'tmYRMajorLineColor', + 'tmYRMajorOutwardLengthF', 'tmYRMajorThicknessF', 'tmYRMaxLabelLenF', + 'tmYRMaxTicks', 'tmYRMinLabelSpacingF', 'tmYRMinorLengthF', + 'tmYRMinorLineColor', 'tmYRMinorOn', 'tmYRMinorOutwardLengthF', + 'tmYRMinorPerMajor', 'tmYRMinorThicknessF', 'tmYRMinorValues', + 'tmYRMode', 'tmYROn', 'tmYRPrecision', 'tmYRStyle', 'tmYRTickEndF', + 'tmYRTickSpacingF', 'tmYRTickStartF', 'tmYRValues', 'tmYUseLeft', + 'trGridType', 'trLineInterpolationOn', + 'trXAxisType', 'trXCoordPoints', 'trXInterPoints', 'trXLog', + 'trXMaxF', 'trXMinF', 'trXReverse', 'trXSamples', 'trXTensionF', + 'trYAxisType', 'trYCoordPoints', 'trYInterPoints', 'trYLog', + 'trYMaxF', 'trYMinF', 'trYReverse', 'trYSamples', 'trYTensionF', + 'txAngleF', 'txBackgroundFillColor', 'txConstantSpacingF', 'txDirection', + 'txFont', 'HLU-Fonts', 'txFontAspectF', 'txFontColor', + 'txFontHeightF', 'txFontOpacityF', 'txFontQuality', + 'txFontThicknessF', 'txFuncCode', 'txJust', 'txPerimColor', + 'txPerimDashLengthF', 'txPerimDashPattern', 'txPerimOn', + 'txPerimSpaceF', 'txPerimThicknessF', 'txPosXF', 'txPosYF', + 'txString', 'vcExplicitLabelBarLabelsOn', 'vcFillArrowEdgeColor', + 'vcFillArrowEdgeThicknessF', 'vcFillArrowFillColor', + 'vcFillArrowHeadInteriorXF', 'vcFillArrowHeadMinFracXF', + 'vcFillArrowHeadMinFracYF', 'vcFillArrowHeadXF', 'vcFillArrowHeadYF', + 'vcFillArrowMinFracWidthF', 'vcFillArrowWidthF', 'vcFillArrowsOn', + 'vcFillOverEdge', 'vcGlyphOpacityF', 'vcGlyphStyle', + 'vcLabelBarEndLabelsOn', 'vcLabelFontColor', 'vcLabelFontHeightF', + 'vcLabelsOn', 'vcLabelsUseVectorColor', 'vcLevelColors', + 'vcLevelCount', 'vcLevelPalette', 'vcLevelSelectionMode', + 'vcLevelSpacingF', 'vcLevels', 'vcLineArrowColor', + 'vcLineArrowHeadMaxSizeF', 'vcLineArrowHeadMinSizeF', + 'vcLineArrowThicknessF', 'vcMagnitudeFormat', + 'vcMagnitudeScaleFactorF', 'vcMagnitudeScaleValueF', + 'vcMagnitudeScalingMode', 'vcMapDirection', 'vcMaxLevelCount', + 'vcMaxLevelValF', 'vcMaxMagnitudeF', 'vcMinAnnoAngleF', + 'vcMinAnnoArrowAngleF', 'vcMinAnnoArrowEdgeColor', + 'vcMinAnnoArrowFillColor', 'vcMinAnnoArrowLineColor', + 'vcMinAnnoArrowMinOffsetF', 'vcMinAnnoArrowSpaceF', + 'vcMinAnnoArrowUseVecColor', 'vcMinAnnoBackgroundColor', + 'vcMinAnnoConstantSpacingF', 'vcMinAnnoExplicitMagnitudeF', + 'vcMinAnnoFont', 'vcMinAnnoFontAspectF', 'vcMinAnnoFontColor', + 'vcMinAnnoFontHeightF', 'vcMinAnnoFontQuality', + 'vcMinAnnoFontThicknessF', 'vcMinAnnoFuncCode', 'vcMinAnnoJust', + 'vcMinAnnoOn', 'vcMinAnnoOrientation', 'vcMinAnnoOrthogonalPosF', + 'vcMinAnnoParallelPosF', 'vcMinAnnoPerimColor', 'vcMinAnnoPerimOn', + 'vcMinAnnoPerimSpaceF', 'vcMinAnnoPerimThicknessF', 'vcMinAnnoSide', + 'vcMinAnnoString1', 'vcMinAnnoString1On', 'vcMinAnnoString2', + 'vcMinAnnoString2On', 'vcMinAnnoTextDirection', 'vcMinAnnoZone', + 'vcMinDistanceF', 'vcMinFracLengthF', 'vcMinLevelValF', + 'vcMinMagnitudeF', 'vcMonoFillArrowEdgeColor', + 'vcMonoFillArrowFillColor', 'vcMonoLineArrowColor', + 'vcMonoWindBarbColor', 'vcNoDataLabelOn', 'vcNoDataLabelString', + 'vcPositionMode', 'vcRefAnnoAngleF', 'vcRefAnnoArrowAngleF', + 'vcRefAnnoArrowEdgeColor', 'vcRefAnnoArrowFillColor', + 'vcRefAnnoArrowLineColor', 'vcRefAnnoArrowMinOffsetF', + 'vcRefAnnoArrowSpaceF', 'vcRefAnnoArrowUseVecColor', + 'vcRefAnnoBackgroundColor', 'vcRefAnnoConstantSpacingF', + 'vcRefAnnoExplicitMagnitudeF', 'vcRefAnnoFont', + 'vcRefAnnoFontAspectF', 'vcRefAnnoFontColor', 'vcRefAnnoFontHeightF', + 'vcRefAnnoFontQuality', 'vcRefAnnoFontThicknessF', + 'vcRefAnnoFuncCode', 'vcRefAnnoJust', 'vcRefAnnoOn', + 'vcRefAnnoOrientation', 'vcRefAnnoOrthogonalPosF', + 'vcRefAnnoParallelPosF', 'vcRefAnnoPerimColor', 'vcRefAnnoPerimOn', + 'vcRefAnnoPerimSpaceF', 'vcRefAnnoPerimThicknessF', 'vcRefAnnoSide', + 'vcRefAnnoString1', 'vcRefAnnoString1On', 'vcRefAnnoString2', + 'vcRefAnnoString2On', 'vcRefAnnoTextDirection', 'vcRefAnnoZone', + 'vcRefLengthF', 'vcRefMagnitudeF', 'vcScalarFieldData', + 'vcScalarMissingValColor', 'vcScalarValueFormat', + 'vcScalarValueScaleFactorF', 'vcScalarValueScaleValueF', + 'vcScalarValueScalingMode', 'vcSpanLevelPalette', 'vcUseRefAnnoRes', + 'vcUseScalarArray', 'vcVectorDrawOrder', 'vcVectorFieldData', + 'vcWindBarbCalmCircleSizeF', 'vcWindBarbColor', + 'vcWindBarbLineThicknessF', 'vcWindBarbScaleFactorF', + 'vcWindBarbTickAngleF', 'vcWindBarbTickLengthF', + 'vcWindBarbTickSpacingF', 'vcZeroFLabelAngleF', + 'vcZeroFLabelBackgroundColor', 'vcZeroFLabelConstantSpacingF', + 'vcZeroFLabelFont', 'vcZeroFLabelFontAspectF', + 'vcZeroFLabelFontColor', 'vcZeroFLabelFontHeightF', + 'vcZeroFLabelFontQuality', 'vcZeroFLabelFontThicknessF', + 'vcZeroFLabelFuncCode', 'vcZeroFLabelJust', 'vcZeroFLabelOn', + 'vcZeroFLabelOrthogonalPosF', 'vcZeroFLabelParallelPosF', + 'vcZeroFLabelPerimColor', 'vcZeroFLabelPerimOn', + 'vcZeroFLabelPerimSpaceF', 'vcZeroFLabelPerimThicknessF', + 'vcZeroFLabelSide', 'vcZeroFLabelString', 'vcZeroFLabelTextDirection', + 'vcZeroFLabelZone', 'vfCopyData', 'vfDataArray', + 'vfExchangeDimensions', 'vfExchangeUVData', 'vfMagMaxV', 'vfMagMinV', + 'vfMissingUValueV', 'vfMissingVValueV', 'vfPolarData', + 'vfSingleMissingValue', 'vfUDataArray', 'vfUMaxV', 'vfUMinV', + 'vfVDataArray', 'vfVMaxV', 'vfVMinV', 'vfXArray', 'vfXCActualEndF', + 'vfXCActualStartF', 'vfXCEndIndex', 'vfXCEndSubsetV', 'vfXCEndV', + 'vfXCStartIndex', 'vfXCStartSubsetV', 'vfXCStartV', 'vfXCStride', + 'vfYArray', 'vfYCActualEndF', 'vfYCActualStartF', 'vfYCEndIndex', + 'vfYCEndSubsetV', 'vfYCEndV', 'vfYCStartIndex', 'vfYCStartSubsetV', + 'vfYCStartV', 'vfYCStride', 'vpAnnoManagerId', 'vpClipOn', + 'vpHeightF', 'vpKeepAspect', 'vpOn', 'vpUseSegments', 'vpWidthF', + 'vpXF', 'vpYF', 'wkAntiAlias', 'wkBackgroundColor', 'wkBackgroundOpacityF', + 'wkColorMapLen', 'wkColorMap', 'wkColorModel', 'wkDashTableLength', + 'wkDefGraphicStyleId', 'wkDeviceLowerX', 'wkDeviceLowerY', + 'wkDeviceUpperX', 'wkDeviceUpperY', 'wkFileName', 'wkFillTableLength', + 'wkForegroundColor', 'wkFormat', 'wkFullBackground', 'wkGksWorkId', + 'wkHeight', 'wkMarkerTableLength', 'wkMetaName', 'wkOrientation', + 'wkPDFFileName', 'wkPDFFormat', 'wkPDFResolution', 'wkPSFileName', + 'wkPSFormat', 'wkPSResolution', 'wkPaperHeightF', 'wkPaperSize', + 'wkPaperWidthF', 'wkPause', 'wkTopLevelViews', 'wkViews', + 'wkVisualType', 'wkWidth', 'wkWindowId', 'wkXColorMode', 'wsCurrentSize', + 'wsMaximumSize', 'wsThresholdSize', 'xyComputeXMax', + 'xyComputeXMin', 'xyComputeYMax', 'xyComputeYMin', 'xyCoordData', + 'xyCoordDataSpec', 'xyCurveDrawOrder', 'xyDashPattern', + 'xyDashPatterns', 'xyExplicitLabels', 'xyExplicitLegendLabels', + 'xyLabelMode', 'xyLineColor', 'xyLineColors', 'xyLineDashSegLenF', + 'xyLineLabelConstantSpacingF', 'xyLineLabelFont', + 'xyLineLabelFontAspectF', 'xyLineLabelFontColor', + 'xyLineLabelFontColors', 'xyLineLabelFontHeightF', + 'xyLineLabelFontQuality', 'xyLineLabelFontThicknessF', + 'xyLineLabelFuncCode', 'xyLineThicknessF', 'xyLineThicknesses', + 'xyMarkLineMode', 'xyMarkLineModes', 'xyMarker', 'xyMarkerColor', + 'xyMarkerColors', 'xyMarkerSizeF', 'xyMarkerSizes', + 'xyMarkerThicknessF', 'xyMarkerThicknesses', 'xyMarkers', + 'xyMonoDashPattern', 'xyMonoLineColor', 'xyMonoLineLabelFontColor', + 'xyMonoLineThickness', 'xyMonoMarkLineMode', 'xyMonoMarker', + 'xyMonoMarkerColor', 'xyMonoMarkerSize', 'xyMonoMarkerThickness', + 'xyXIrrTensionF', 'xyXIrregularPoints', 'xyXStyle', 'xyYIrrTensionF', + 'xyYIrregularPoints', 'xyYStyle'), prefix=r'\b'), + Name.Builtin), + + # Booleans + (r'\.(True|False)\.', Name.Builtin), + # Comparing Operators + (r'\.(eq|ne|lt|le|gt|ge|not|and|or|xor)\.', Operator.Word), + ], + + 'strings': [ + (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double), + ], + + 'nums': [ + (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer), + (r'[+-]?\d*\.\d+(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float), + (r'[+-]?\d+\.\d*(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float), + ], + } diff --git a/pygments/lexers/nimrod.py b/pygments/lexers/nimrod.py index 00b849a6..d438c1bf 100644 --- a/pygments/lexers/nimrod.py +++ b/pygments/lexers/nimrod.py @@ -3,9 +3,9 @@ pygments.lexers.nimrod ~~~~~~~~~~~~~~~~~~~~~~ - Lexer for the Nimrod language. + Lexer for the Nim language (formerly known as Nimrod). - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -20,15 +20,15 @@ __all__ = ['NimrodLexer'] class NimrodLexer(RegexLexer): """ - For `Nimrod <http://nimrod-code.org/>`_ source code. + For `Nim <http://nim-lang.org/>`_ source code. .. versionadded:: 1.5 """ name = 'Nimrod' - aliases = ['nimrod', 'nim'] + aliases = ['nim', 'nimrod'] filenames = ['*.nim', '*.nimrod'] - mimetypes = ['text/x-nimrod'] + mimetypes = ['text/x-nim'] flags = re.MULTILINE | re.IGNORECASE | re.UNICODE @@ -43,13 +43,13 @@ class NimrodLexer(RegexLexer): return "|".join(newWords) keywords = [ - 'addr', 'and', 'as', 'asm', 'atomic', 'bind', 'block', 'break', - 'case', 'cast', 'const', 'continue', 'converter', 'discard', - 'distinct', 'div', 'elif', 'else', 'end', 'enum', 'except', 'finally', - 'for', 'generic', 'if', 'implies', 'in', 'yield', - 'is', 'isnot', 'iterator', 'lambda', 'let', 'macro', 'method', - 'mod', 'not', 'notin', 'object', 'of', 'or', 'out', 'proc', - 'ptr', 'raise', 'ref', 'return', 'shl', 'shr', 'template', 'try', + 'addr', 'and', 'as', 'asm', 'atomic', 'bind', 'block', 'break', 'case', + 'cast', 'concept', 'const', 'continue', 'converter', 'defer', 'discard', + 'distinct', 'div', 'do', 'elif', 'else', 'end', 'enum', 'except', + 'export', 'finally', 'for', 'func', 'if', 'in', 'yield', 'interface', + 'is', 'isnot', 'iterator', 'let', 'macro', 'method', 'mixin', 'mod', + 'not', 'notin', 'object', 'of', 'or', 'out', 'proc', 'ptr', 'raise', + 'ref', 'return', 'shared', 'shl', 'shr', 'static', 'template', 'try', 'tuple', 'type', 'when', 'while', 'with', 'without', 'xor' ] diff --git a/pygments/lexers/nit.py b/pygments/lexers/nit.py index ab59c4e5..21116499 100644 --- a/pygments/lexers/nit.py +++ b/pygments/lexers/nit.py @@ -5,7 +5,7 @@ Lexer for the Nit language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/nix.py b/pygments/lexers/nix.py index 57f08623..e148c919 100644 --- a/pygments/lexers/nix.py +++ b/pygments/lexers/nix.py @@ -5,7 +5,7 @@ Lexers for the NixOS Nix language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/oberon.py b/pygments/lexers/oberon.py index db18259d..3b5fb3e4 100644 --- a/pygments/lexers/oberon.py +++ b/pygments/lexers/oberon.py @@ -5,7 +5,7 @@ Lexers for Oberon family languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -47,11 +47,11 @@ class ComponentPascalLexer(RegexLexer): (r'\s+', Text), # whitespace ], 'comments': [ - (r'\(\*([^\$].*?)\*\)', Comment.Multiline), + (r'\(\*([^$].*?)\*\)', Comment.Multiline), # TODO: nested comments (* (* ... *) ... (* ... *) *) not supported! ], 'punctuation': [ - (r'[\(\)\[\]\{\},.:;\|]', Punctuation), + (r'[()\[\]{},.:;|]', Punctuation), ], 'numliterals': [ (r'[0-9A-F]+X\b', Number.Hex), # char code @@ -83,7 +83,7 @@ class ComponentPascalLexer(RegexLexer): (r'\$', Operator), ], 'identifiers': [ - (r'([a-zA-Z_\$][\w\$]*)', Name), + (r'([a-zA-Z_$][\w$]*)', Name), ], 'builtins': [ (words(( diff --git a/pygments/lexers/objective.py b/pygments/lexers/objective.py index fc8e5d17..7807255e 100644 --- a/pygments/lexers/objective.py +++ b/pygments/lexers/objective.py @@ -5,7 +5,7 @@ Lexers for Objective-C family languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -298,7 +298,7 @@ class SwiftLexer(RegexLexer): (r'\s+', Text), (r'//', Comment.Single, 'comment-single'), (r'/\*', Comment.Multiline, 'comment-multi'), - (r'#(if|elseif|else|endif)\b', Comment.Preproc, 'preproc'), + (r'#(if|elseif|else|endif|available)\b', Comment.Preproc, 'preproc'), # Keywords include('keywords'), @@ -413,23 +413,26 @@ class SwiftLexer(RegexLexer): ], 'keywords': [ (words(( - 'break', 'case', 'continue', 'default', 'do', 'else', - 'fallthrough', 'for', 'if', 'in', 'return', 'switch', 'where', - 'while'), suffix=r'\b'), + 'as', 'break', 'case', 'catch', 'continue', 'default', 'defer', + 'do', 'else', 'fallthrough', 'for', 'guard', 'if', 'in', 'is', + 'repeat', 'return', '#selector', 'switch', 'throw', 'try', + 'where', 'while'), suffix=r'\b'), Keyword), (r'@availability\([^)]+\)', Keyword.Reserved), (words(( 'associativity', 'convenience', 'dynamic', 'didSet', 'final', - 'get', 'infix', 'inout', 'lazy', 'left', 'mutating', 'none', - 'nonmutating', 'optional', 'override', 'postfix', 'precedence', - 'prefix', 'Protocol', 'required', 'right', 'set', 'Type', - 'unowned', 'weak', 'willSet', '@availability', '@autoclosure', - '@noreturn', '@NSApplicationMain', '@NSCopying', '@NSManaged', - '@objc', '@UIApplicationMain', '@IBAction', '@IBDesignable', + 'get', 'indirect', 'infix', 'inout', 'lazy', 'left', 'mutating', + 'none', 'nonmutating', 'optional', 'override', 'postfix', + 'precedence', 'prefix', 'Protocol', 'required', 'rethrows', + 'right', 'set', 'throws', 'Type', 'unowned', 'weak', 'willSet', + '@availability', '@autoclosure', '@noreturn', + '@NSApplicationMain', '@NSCopying', '@NSManaged', '@objc', + '@UIApplicationMain', '@IBAction', '@IBDesignable', '@IBInspectable', '@IBOutlet'), suffix=r'\b'), Keyword.Reserved), (r'(as|dynamicType|false|is|nil|self|Self|super|true|__COLUMN__' - r'|__FILE__|__FUNCTION__|__LINE__|_)\b', Keyword.Constant), + r'|__FILE__|__FUNCTION__|__LINE__|_' + r'|#(?:file|line|column|function))\b', Keyword.Constant), (r'import\b', Keyword.Declaration, 'module'), (r'(class|enum|extension|struct|protocol)(\s+)([a-zA-Z_]\w*)', bygroups(Keyword.Declaration, Text, Name.Class)), diff --git a/pygments/lexers/ooc.py b/pygments/lexers/ooc.py index b4e8c6db..957b72f1 100644 --- a/pygments/lexers/ooc.py +++ b/pygments/lexers/ooc.py @@ -5,7 +5,7 @@ Lexers for the Ooc language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py index afd0fda5..bfce4c3c 100644 --- a/pygments/lexers/other.py +++ b/pygments/lexers/other.py @@ -5,7 +5,7 @@ Just export lexer classes previously contained in this module. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -36,5 +36,6 @@ from pygments.lexers.urbi import UrbiscriptLexer from pygments.lexers.smalltalk import SmalltalkLexer, NewspeakLexer from pygments.lexers.installers import NSISLexer, RPMSpecLexer from pygments.lexers.textedit import AwkLexer +from pygments.lexers.smv import NuSMVLexer __all__ = [] diff --git a/pygments/lexers/parasail.py b/pygments/lexers/parasail.py index 878f7d26..53088023 100644 --- a/pygments/lexers/parasail.py +++ b/pygments/lexers/parasail.py @@ -5,7 +5,7 @@ Lexer for ParaSail. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -60,7 +60,7 @@ class ParaSailLexer(RegexLexer): (r'[a-zA-Z]\w*', Name), # Operators and Punctuation (r'(<==|==>|<=>|\*\*=|<\|=|<<=|>>=|==|!=|=\?|<=|>=|' - r'\*\*|<<|>>|=>|:=|\+=|-=|\*=|\||\|=|/=|\+|-|\*|/|' + r'\*\*|<<|>>|=>|:=|\+=|-=|\*=|\|=|\||/=|\+|-|\*|/|' r'\.\.|<\.\.|\.\.<|<\.\.<)', Operator), (r'(<|>|\[|\]|\(|\)|\||:|;|,|.|\{|\}|->)', diff --git a/pygments/lexers/parsers.py b/pygments/lexers/parsers.py index e1b74dee..1f3c9b4d 100644 --- a/pygments/lexers/parsers.py +++ b/pygments/lexers/parsers.py @@ -5,7 +5,7 @@ Lexers for parser generators. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/pascal.py b/pygments/lexers/pascal.py index ce991a77..9aa1ac8f 100644 --- a/pygments/lexers/pascal.py +++ b/pygments/lexers/pascal.py @@ -5,7 +5,7 @@ Lexers for Pascal family languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -44,7 +44,7 @@ class DelphiLexer(Lexer): """ name = 'Delphi' aliases = ['delphi', 'pas', 'pascal', 'objectpascal'] - filenames = ['*.pas'] + filenames = ['*.pas', '*.dpr'] mimetypes = ['text/x-pascal'] TURBO_PASCAL_KEYWORDS = ( diff --git a/pygments/lexers/pawn.py b/pygments/lexers/pawn.py index f32fdbed..f462a883 100644 --- a/pygments/lexers/pawn.py +++ b/pygments/lexers/pawn.py @@ -5,7 +5,7 @@ Lexers for the Pawn languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/perl.py b/pygments/lexers/perl.py index b78963d0..4d5ab3b3 100644 --- a/pygments/lexers/perl.py +++ b/pygments/lexers/perl.py @@ -5,7 +5,7 @@ Lexers for Perl and related languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -52,7 +52,7 @@ class PerlLexer(RegexLexer): (words(( 'case', 'continue', 'do', 'else', 'elsif', 'for', 'foreach', 'if', 'last', 'my', 'next', 'our', 'redo', 'reset', 'then', - 'unless', 'until', 'while', 'use', 'print', 'new', 'BEGIN', + 'unless', 'until', 'while', 'print', 'new', 'BEGIN', 'CHECK', 'INIT', 'END', 'return'), suffix=r'\b'), Keyword), (r'(format)(\s+)(\w+)(\s*)(=)(\s*\n)', @@ -94,10 +94,10 @@ class PerlLexer(RegexLexer): 'getservbyport', 'getservent', 'getsockname', 'getsockopt', 'glob', 'gmtime', 'goto', 'grep', 'hex', 'import', 'index', 'int', 'ioctl', 'join', 'keys', 'kill', 'last', 'lc', 'lcfirst', 'length', 'link', 'listen', 'local', 'localtime', 'log', 'lstat', - 'map', 'mkdir', 'msgctl', 'msgget', 'msgrcv', 'msgsnd', 'my', 'next', 'no', 'oct', 'open', - 'opendir', 'ord', 'our', 'pack', 'package', 'pipe', 'pop', 'pos', 'printf', + 'map', 'mkdir', 'msgctl', 'msgget', 'msgrcv', 'msgsnd', 'my', 'next', 'oct', 'open', + 'opendir', 'ord', 'our', 'pack', 'pipe', 'pop', 'pos', 'printf', 'prototype', 'push', 'quotemeta', 'rand', 'read', 'readdir', - 'readline', 'readlink', 'readpipe', 'recv', 'redo', 'ref', 'rename', 'require', + 'readline', 'readlink', 'readpipe', 'recv', 'redo', 'ref', 'rename', 'reverse', 'rewinddir', 'rindex', 'rmdir', 'scalar', 'seek', 'seekdir', 'select', 'semctl', 'semget', 'semop', 'send', 'setgrent', 'sethostent', 'setnetent', 'setpgrp', 'setpriority', 'setprotoent', 'setpwent', 'setservent', @@ -109,7 +109,8 @@ class PerlLexer(RegexLexer): 'utime', 'values', 'vec', 'wait', 'waitpid', 'wantarray', 'warn', 'write'), suffix=r'\b'), Name.Builtin), (r'((__(DATA|DIE|WARN)__)|(STD(IN|OUT|ERR)))\b', Name.Builtin.Pseudo), - (r'<<([\'"]?)([a-zA-Z_]\w*)\1;?\n.*?\n\2\n', String), + (r'(<<)([\'"]?)([a-zA-Z_]\w*)(\2;?\n.*?\n)(\3)(\n)', + bygroups(String, String, String.Delimiter, String, String.Delimiter, Text)), (r'__END__', Comment.Preproc, 'end-part'), (r'\$\^[ADEFHILMOPSTWX]', Name.Variable.Global), (r"\$[\\\"\[\]'&`+*.,;=%~?@$!<>(^|/-](?!\w)", Name.Variable.Global), @@ -130,8 +131,14 @@ class PerlLexer(RegexLexer): (r'(q|qq|qw|qr|qx)\[', String.Other, 'sb-string'), (r'(q|qq|qw|qr|qx)\<', String.Other, 'lt-string'), (r'(q|qq|qw|qr|qx)([\W_])(.|\n)*?\2', String.Other), - (r'package\s+', Keyword, 'modulename'), - (r'sub\s+', Keyword, 'funcname'), + (r'(package)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)', + bygroups(Keyword, Text, Name.Namespace)), + (r'(use|require|no)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)', + bygroups(Keyword, Text, Name.Namespace)), + (r'(sub)(\s+)', bygroups(Keyword, Text), 'funcname'), + (words(( + 'no', 'package', 'require', 'use'), suffix=r'\b'), + Keyword), (r'(\[\]|\*\*|::|<<|>>|>=|<=>|<=|={3}|!=|=~|' r'!~|&&?|\|\||\.{1,3})', Operator), (r'[-+/*%=<>&^|!\\~]=?', Operator), @@ -151,14 +158,12 @@ class PerlLexer(RegexLexer): (r'[\w:]+', Name.Variable, '#pop'), ], 'name': [ - (r'\w+::', Name.Namespace), + (r'[a-zA-Z_]\w*(::[a-zA-Z_]\w*)*(::)?(?=\s*->)', Name.Namespace, '#pop'), + (r'[a-zA-Z_]\w*(::[a-zA-Z_]\w*)*::', Name.Namespace, '#pop'), (r'[\w:]+', Name, '#pop'), (r'[A-Z_]+(?=\W)', Name.Constant, '#pop'), (r'(?=\W)', Text, '#pop'), ], - 'modulename': [ - (r'[a-zA-Z_]\w*', Name.Namespace, '#pop') - ], 'funcname': [ (r'[a-zA-Z_]\w*[!?]?', Name.Function), (r'\s+', Text), diff --git a/pygments/lexers/php.py b/pygments/lexers/php.py index 75b662cb..f618b5fd 100644 --- a/pygments/lexers/php.py +++ b/pygments/lexers/php.py @@ -5,13 +5,14 @@ Lexers for PHP and related languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re -from pygments.lexer import RegexLexer, include, bygroups, default, using, this +from pygments.lexer import RegexLexer, include, bygroups, default, using, \ + this, words from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Other from pygments.util import get_bool_opt, get_list_opt, iteritems @@ -137,7 +138,9 @@ class PhpLexer(RegexLexer): ], 'php': [ (r'\?>', Comment.Preproc, '#pop'), - (r'<<<([\'"]?)(' + _ident_inner + r')\1\n.*?\n\s*\2;?\n', String), + (r'(<<<)([\'"]?)(' + _ident_inner + r')(\2\n.*?\n\s*)(\3)(;?)(\n)', + bygroups(String, String, String.Delimiter, String, String.Delimiter, + Punctuation, Text)), (r'\s+', Text), (r'#.*?\n', Comment.Single), (r'//.*?\n', Comment.Single), @@ -162,13 +165,14 @@ class PhpLexer(RegexLexer): r'FALSE|print|for|require|continue|foreach|require_once|' r'declare|return|default|static|do|switch|die|stdClass|' r'echo|else|TRUE|elseif|var|empty|if|xor|enddeclare|include|' - r'virtual|endfor|include_once|while|endforeach|global|__FILE__|' - r'endif|list|__LINE__|endswitch|new|__sleep|endwhile|not|' - r'array|__wakeup|E_ALL|NULL|final|php_user_filter|interface|' + r'virtual|endfor|include_once|while|endforeach|global|' + r'endif|list|endswitch|new|endwhile|not|' + r'array|E_ALL|NULL|final|php_user_filter|interface|' r'implements|public|private|protected|abstract|clone|try|' r'catch|throw|this|use|namespace|trait|yield|' r'finally)\b', Keyword), (r'(true|false|null)\b', Keyword.Constant), + include('magicconstants'), (r'\$\{\$+' + _ident_inner + '\}', Name.Variable), (r'\$+' + _ident_inner, Name.Variable), (_ident_inner, Name.Other), @@ -182,11 +186,29 @@ class PhpLexer(RegexLexer): (r'`([^`\\]*(?:\\.[^`\\]*)*)`', String.Backtick), (r'"', String.Double, 'string'), ], + 'magicfuncs': [ + # source: http://php.net/manual/en/language.oop5.magic.php + (words(( + '__construct', '__destruct', '__call', '__callStatic', '__get', '__set', + '__isset', '__unset', '__sleep', '__wakeup', '__toString', '__invoke', + '__set_state', '__clone', '__debugInfo',), suffix=r'\b'), + Name.Function.Magic), + ], + 'magicconstants': [ + # source: http://php.net/manual/en/language.constants.predefined.php + (words(( + '__LINE__', '__FILE__', '__DIR__', '__FUNCTION__', '__CLASS__', + '__TRAIT__', '__METHOD__', '__NAMESPACE__',), + suffix=r'\b'), + Name.Constant), + ], 'classname': [ (_ident_inner, Name.Class, '#pop') ], 'functionname': [ - (_ident_inner, Name.Function, '#pop') + include('magicfuncs'), + (_ident_inner, Name.Function, '#pop'), + default('#pop') ], 'string': [ (r'"', String.Double, '#pop'), @@ -202,7 +224,7 @@ class PhpLexer(RegexLexer): String.Interpol)), (r'(\$\{)(\S+)(\})', bygroups(String.Interpol, Name.Variable, String.Interpol)), - (r'[${\\]+', String.Double) + (r'[${\\]', String.Double) ], } diff --git a/pygments/lexers/praat.py b/pygments/lexers/praat.py index 776c38b8..1a38a9e8 100644 --- a/pygments/lexers/praat.py +++ b/pygments/lexers/praat.py @@ -5,7 +5,7 @@ Lexer for Praat - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -27,21 +27,21 @@ class PraatLexer(RegexLexer): aliases = ['praat'] filenames = ['*.praat', '*.proc', '*.psc'] - keywords = [ + keywords = ( 'if', 'then', 'else', 'elsif', 'elif', 'endif', 'fi', 'for', 'from', 'to', 'endfor', 'endproc', 'while', 'endwhile', 'repeat', 'until', 'select', 'plus', 'minus', 'demo', 'assert', 'stopwatch', 'nocheck', 'nowarn', 'noprogress', 'editor', 'endeditor', 'clearinfo', - ] + ) - functions_string = [ + functions_string = ( 'backslashTrigraphsToUnicode', 'chooseDirectory', 'chooseReadFile', 'chooseWriteFile', 'date', 'demoKey', 'do', 'environment', 'extractLine', 'extractWord', 'fixed', 'info', 'left', 'mid', 'percent', 'readFile', 'replace', 'replace_regex', 'right', 'selected', 'string', 'unicodeToBackslashTrigraphs', - ] + ) - functions_numeric = [ + functions_numeric = ( 'abs', 'appendFile', 'appendFileLine', 'appendInfo', 'appendInfoLine', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'barkToHertz', 'beginPause', 'beginSendPraat', 'besselI', 'besselK', 'beta', 'beta2', @@ -67,13 +67,13 @@ class PraatLexer(RegexLexer): 'sincpi', 'sinh', 'soundPressureToPhon', 'sqrt', 'startsWith', 'studentP', 'studentQ', 'tan', 'tanh', 'variableExists', 'word', 'writeFile', 'writeFileLine', 'writeInfo', 'writeInfoLine', - ] + ) - functions_array = [ + functions_array = ( 'linear', 'randomGauss', 'randomInteger', 'randomUniform', 'zero', - ] + ) - objects = [ + objects = ( 'Activation', 'AffineTransform', 'AmplitudeTier', 'Art', 'Artword', 'Autosegment', 'BarkFilter', 'BarkSpectrogram', 'CCA', 'Categories', 'Cepstrogram', 'Cepstrum', 'Cepstrumc', 'ChebyshevSeries', 'ClassificationTable', @@ -100,17 +100,17 @@ class PraatLexer(RegexLexer): 'Strings', 'StringsIndex', 'Table', 'TableOfReal', 'TextGrid', 'TextInterval', 'TextPoint', 'TextTier', 'Tier', 'Transition', 'VocalTract', 'VocalTractTier', 'Weight', 'WordList', - ] + ) - variables_numeric = [ + variables_numeric = ( 'macintosh', 'windows', 'unix', 'praatVersion', 'pi', 'e', 'undefined', - ] + ) - variables_string = [ + variables_string = ( 'praatVersion', 'tab', 'shellDirectory', 'homeDirectory', 'preferencesDirectory', 'newline', 'temporaryDirectory', 'defaultDirectory', - ] + ) tokens = { 'root': [ @@ -151,7 +151,7 @@ class PraatLexer(RegexLexer): (r"'(?=.*')", String.Interpol, 'string_interpolated'), (r'\.{3}', Keyword, ('#pop', 'old_arguments')), (r':', Keyword, ('#pop', 'comma_list')), - (r'[\s\n]', Text, '#pop'), + (r'\s', Text, '#pop'), ], 'procedure_call': [ (r'\s+', Text), @@ -168,8 +168,8 @@ class PraatLexer(RegexLexer): ], 'function_call': [ (words(functions_string, suffix=r'\$(?=\s*[:(])'), Name.Function, 'function'), - (words(functions_array, suffix=r'#(?=\s*[:(])'), Name.Function, 'function'), - (words(functions_numeric, suffix=r'(?=\s*[:(])'), Name.Function, 'function'), + (words(functions_array, suffix=r'#(?=\s*[:(])'), Name.Function, 'function'), + (words(functions_numeric, suffix=r'(?=\s*[:(])'), Name.Function, 'function'), ], 'function': [ (r'\s+', Text), @@ -190,6 +190,7 @@ class PraatLexer(RegexLexer): include('operator'), include('number'), + (r'[()]', Text), (r',', Punctuation), ], 'old_arguments': [ @@ -203,6 +204,7 @@ class PraatLexer(RegexLexer): (r'[^\n]', Text), ], 'number': [ + (r'\n', Text, '#pop'), (r'\b\d+(\.\d*)?([eE][-+]?\d+)?%?', Number), ], 'object_attributes': [ @@ -230,16 +232,16 @@ class PraatLexer(RegexLexer): bygroups(Name.Builtin, Name.Builtin, String.Interpol), ('object_attributes', 'string_interpolated')), - (r'\.?_?[a-z][a-zA-Z0-9_.]*(\$|#)?', Text), + (r'\.?_?[a-z][\w.]*(\$|#)?', Text), (r'[\[\]]', Punctuation, 'comma_list'), (r"'(?=.*')", String.Interpol, 'string_interpolated'), ], 'operator': [ - (r'([+\/*<>=!-]=?|[&*|][&*|]?|\^|<>)', Operator), - (r'\b(and|or|not|div|mod)\b', Operator.Word), + (r'([+\/*<>=!-]=?|[&*|][&*|]?|\^|<>)', Operator), + (r'(?<![\w.])(and|or|not|div|mod)(?![\w.])', Operator.Word), ], 'string_interpolated': [ - (r'\.?[_a-z][a-zA-Z0-9_.]*[\$#]?(?:\[[a-zA-Z0-9,]+\])?(:[0-9]+)?', + (r'\.?[_a-z][\w.]*[$#]?(?:\[[a-zA-Z0-9,]+\])?(:[0-9]+)?', String.Interpol), (r"'", String.Interpol, '#pop'), ], @@ -267,9 +269,6 @@ class PraatLexer(RegexLexer): bygroups(Keyword, Text), 'number'), (r'(option|button)([ \t]+)', - bygroups(Keyword, Text), 'number'), - - (r'(option|button)([ \t]+)', bygroups(Keyword, Text), 'string_unquoted'), (r'(sentence|text)([ \t]+\S+)', diff --git a/pygments/lexers/prolog.py b/pygments/lexers/prolog.py index 7d32d7f6..90f9529c 100644 --- a/pygments/lexers/prolog.py +++ b/pygments/lexers/prolog.py @@ -5,7 +5,7 @@ Lexers for Prolog and Prolog-like languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py index 9433f7fd..390eafe8 100644 --- a/pygments/lexers/python.py +++ b/pygments/lexers/python.py @@ -5,7 +5,7 @@ Lexers for Python and related languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -39,7 +39,7 @@ class PythonLexer(RegexLexer): return [ # the old style '%s' % (...) string formatting (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' - '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol), + '[hlL]?[E-GXc-giorsux%]', String.Interpol), # backslashes, quotes and formatting signs must be parsed one at a time (r'[^\\\'"%\n]+', ttype), (r'[\'"\\]', ttype), @@ -51,8 +51,10 @@ class PythonLexer(RegexLexer): tokens = { 'root': [ (r'\n', Text), - (r'^(\s*)([rRuU]{,2}"""(?:.|\n)*?""")', bygroups(Text, String.Doc)), - (r"^(\s*)([rRuU]{,2}'''(?:.|\n)*?''')", bygroups(Text, String.Doc)), + (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")', + bygroups(Text, String.Affix, String.Doc)), + (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')", + bygroups(Text, String.Affix, String.Doc)), (r'[^\S\n]+', Text), (r'\A#!.+$', Comment.Hashbang), (r'#.*$', Comment.Single), @@ -69,15 +71,25 @@ class PythonLexer(RegexLexer): (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), 'import'), include('builtins'), + include('magicfuncs'), + include('magicvars'), include('backtick'), - ('(?:[rR]|[uU][rR]|[rR][uU])"""', String.Double, 'tdqs'), - ("(?:[rR]|[uU][rR]|[rR][uU])'''", String.Single, 'tsqs'), - ('(?:[rR]|[uU][rR]|[rR][uU])"', String.Double, 'dqs'), - ("(?:[rR]|[uU][rR]|[rR][uU])'", String.Single, 'sqs'), - ('[uU]?"""', String.Double, combined('stringescape', 'tdqs')), - ("[uU]?'''", String.Single, combined('stringescape', 'tsqs')), - ('[uU]?"', String.Double, combined('stringescape', 'dqs')), - ("[uU]?'", String.Single, combined('stringescape', 'sqs')), + ('([rR]|[uUbB][rR]|[rR][uUbB])(""")', + bygroups(String.Affix, String.Double), 'tdqs'), + ("([rR]|[uUbB][rR]|[rR][uUbB])(''')", + bygroups(String.Affix, String.Single), 'tsqs'), + ('([rR]|[uUbB][rR]|[rR][uUbB])(")', + bygroups(String.Affix, String.Double), 'dqs'), + ("([rR]|[uUbB][rR]|[rR][uUbB])(')", + bygroups(String.Affix, String.Single), 'sqs'), + ('([uUbB]?)(""")', bygroups(String.Affix, String.Double), + combined('stringescape', 'tdqs')), + ("([uUbB]?)(''')", bygroups(String.Affix, String.Single), + combined('stringescape', 'tsqs')), + ('([uUbB]?)(")', bygroups(String.Affix, String.Double), + combined('stringescape', 'dqs')), + ("([uUbB]?)(')", bygroups(String.Affix, String.Single), + combined('stringescape', 'sqs')), include('name'), include('numbers'), ], @@ -104,7 +116,7 @@ class PythonLexer(RegexLexer): 'unichr', 'unicode', 'vars', 'xrange', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'), Name.Builtin), - (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True' + (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|cls' r')\b', Name.Builtin.Pseudo), (words(( 'ArithmeticError', 'AssertionError', 'AttributeError', @@ -123,6 +135,37 @@ class PythonLexer(RegexLexer): 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'), Name.Exception), ], + 'magicfuncs': [ + (words(( + '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__', + '__complex__', '__contains__', '__del__', '__delattr__', '__delete__', + '__delitem__', '__delslice__', '__div__', '__divmod__', '__enter__', + '__eq__', '__exit__', '__float__', '__floordiv__', '__ge__', '__get__', + '__getattr__', '__getattribute__', '__getitem__', '__getslice__', '__gt__', + '__hash__', '__hex__', '__iadd__', '__iand__', '__idiv__', '__ifloordiv__', + '__ilshift__', '__imod__', '__imul__', '__index__', '__init__', + '__instancecheck__', '__int__', '__invert__', '__iop__', '__ior__', + '__ipow__', '__irshift__', '__isub__', '__iter__', '__itruediv__', + '__ixor__', '__le__', '__len__', '__long__', '__lshift__', '__lt__', + '__missing__', '__mod__', '__mul__', '__ne__', '__neg__', '__new__', + '__nonzero__', '__oct__', '__op__', '__or__', '__pos__', '__pow__', + '__radd__', '__rand__', '__rcmp__', '__rdiv__', '__rdivmod__', '__repr__', + '__reversed__', '__rfloordiv__', '__rlshift__', '__rmod__', '__rmul__', + '__rop__', '__ror__', '__rpow__', '__rrshift__', '__rshift__', '__rsub__', + '__rtruediv__', '__rxor__', '__set__', '__setattr__', '__setitem__', + '__setslice__', '__str__', '__sub__', '__subclasscheck__', '__truediv__', + '__unicode__', '__xor__'), suffix=r'\b'), + Name.Function.Magic), + ], + 'magicvars': [ + (words(( + '__bases__', '__class__', '__closure__', '__code__', '__defaults__', + '__dict__', '__doc__', '__file__', '__func__', '__globals__', + '__metaclass__', '__module__', '__mro__', '__name__', '__self__', + '__slots__', '__weakref__'), + suffix=r'\b'), + Name.Variable.Magic), + ], 'numbers': [ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float), (r'\d+[eE][+-]?[0-9]+j?', Number.Float), @@ -140,7 +183,9 @@ class PythonLexer(RegexLexer): ('[a-zA-Z_]\w*', Name), ], 'funcname': [ - ('[a-zA-Z_]\w*', Name.Function, '#pop') + include('magicfuncs'), + ('[a-zA-Z_]\w*', Name.Function, '#pop'), + default('#pop'), ], 'classname': [ ('[a-zA-Z_]\w*', Name.Class, '#pop') @@ -217,16 +262,16 @@ class Python3Lexer(RegexLexer): return [ # the old style '%s' % (...) string formatting (still valid in Py3) (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' - '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol), + '[hlL]?[E-GXc-giorsux%]', String.Interpol), # the new style '{}'.format(...) string formatting (r'\{' - '((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name - '(\![sra])?' # conversion - '(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[bcdeEfFgGnosxX%]?)?' + '((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name + '(\![sra])?' # conversion + '(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?' '\}', String.Interpol), # backslashes, quotes and formatting signs must be parsed one at a time - (r'[^\\\'"%\{\n]+', ttype), + (r'[^\\\'"%{\n]+', ttype), (r'[\'"\\]', ttype), # unhandled string formatting sign (r'%|(\{{1,2})', ttype) @@ -258,7 +303,7 @@ class Python3Lexer(RegexLexer): 'sum', 'super', 'tuple', 'type', 'vars', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'), Name.Builtin), - (r'(?<!\.)(self|Ellipsis|NotImplemented)\b', Name.Builtin.Pseudo), + (r'(?<!\.)(self|Ellipsis|NotImplemented|cls)\b', Name.Builtin.Pseudo), (words(( 'ArithmeticError', 'AssertionError', 'AttributeError', 'BaseException', 'BufferError', 'BytesWarning', 'DeprecationWarning', @@ -283,8 +328,41 @@ class Python3Lexer(RegexLexer): prefix=r'(?<!\.)', suffix=r'\b'), Name.Exception), ] + tokens['magicfuncs'] = [ + (words(( + '__abs__', '__add__', '__aenter__', '__aexit__', '__aiter__', '__and__', + '__anext__', '__await__', '__bool__', '__bytes__', '__call__', + '__complex__', '__contains__', '__del__', '__delattr__', '__delete__', + '__delitem__', '__dir__', '__divmod__', '__enter__', '__eq__', '__exit__', + '__float__', '__floordiv__', '__format__', '__ge__', '__get__', + '__getattr__', '__getattribute__', '__getitem__', '__gt__', '__hash__', + '__iadd__', '__iand__', '__ifloordiv__', '__ilshift__', '__imatmul__', + '__imod__', '__import__', '__imul__', '__index__', '__init__', + '__instancecheck__', '__int__', '__invert__', '__ior__', '__ipow__', + '__irshift__', '__isub__', '__iter__', '__itruediv__', '__ixor__', + '__le__', '__len__', '__length_hint__', '__lshift__', '__lt__', + '__matmul__', '__missing__', '__mod__', '__mul__', '__ne__', '__neg__', + '__new__', '__next__', '__or__', '__pos__', '__pow__', '__prepare__', + '__radd__', '__rand__', '__rdivmod__', '__repr__', '__reversed__', + '__rfloordiv__', '__rlshift__', '__rmatmul__', '__rmod__', '__rmul__', + '__ror__', '__round__', '__rpow__', '__rrshift__', '__rshift__', + '__rsub__', '__rtruediv__', '__rxor__', '__set__', '__setattr__', + '__setitem__', '__str__', '__sub__', '__subclasscheck__', '__truediv__', + '__xor__'), suffix=r'\b'), + Name.Function.Magic), + ] + tokens['magicvars'] = [ + (words(( + '__annotations__', '__bases__', '__class__', '__closure__', '__code__', + '__defaults__', '__dict__', '__doc__', '__file__', '__func__', + '__globals__', '__kwdefaults__', '__module__', '__mro__', '__name__', + '__objclass__', '__qualname__', '__self__', '__slots__', '__weakref__'), + suffix=r'\b'), + Name.Variable.Magic), + ] tokens['numbers'] = [ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), + (r'\d+[eE][+-]?[0-9]+j?', Number.Float), (r'0[oO][0-7]+', Number.Oct), (r'0[bB][01]+', Number.Bin), (r'0[xX][a-fA-F0-9]+', Number.Hex), @@ -633,7 +711,7 @@ class CythonLexer(RegexLexer): ], 'strings': [ (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' - '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol), + '[hlL]?[E-GXc-giorsux%]', String.Interpol), (r'[^\\\'"%\n]+', String), # quotes, percents and backslashes must be parsed one at a time (r'[\'"\\]', String), @@ -704,18 +782,20 @@ class DgLexer(RegexLexer): (words(( 'bool', 'bytearray', 'bytes', 'classmethod', 'complex', 'dict', 'dict\'', 'float', 'frozenset', 'int', 'list', 'list\'', 'memoryview', 'object', - 'property', 'range', 'set', 'set\'', 'slice', 'staticmethod', 'str', 'super', - 'tuple', 'tuple\'', 'type'), prefix=r'(?<!\.)', suffix=r'(?![\'\w])'), + 'property', 'range', 'set', 'set\'', 'slice', 'staticmethod', 'str', + 'super', 'tuple', 'tuple\'', 'type'), + prefix=r'(?<!\.)', suffix=r'(?![\'\w])'), Name.Builtin), (words(( '__import__', 'abs', 'all', 'any', 'bin', 'bind', 'chr', 'cmp', 'compile', 'complex', 'delattr', 'dir', 'divmod', 'drop', 'dropwhile', 'enumerate', - 'eval', 'exhaust', 'filter', 'flip', 'foldl1?', 'format', 'fst', 'getattr', - 'globals', 'hasattr', 'hash', 'head', 'hex', 'id', 'init', 'input', - 'isinstance', 'issubclass', 'iter', 'iterate', 'last', 'len', 'locals', - 'map', 'max', 'min', 'next', 'oct', 'open', 'ord', 'pow', 'print', 'repr', - 'reversed', 'round', 'setattr', 'scanl1?', 'snd', 'sorted', 'sum', 'tail', - 'take', 'takewhile', 'vars', 'zip'), prefix=r'(?<!\.)', suffix=r'(?![\'\w])'), + 'eval', 'exhaust', 'filter', 'flip', 'foldl1?', 'format', 'fst', + 'getattr', 'globals', 'hasattr', 'hash', 'head', 'hex', 'id', 'init', + 'input', 'isinstance', 'issubclass', 'iter', 'iterate', 'last', 'len', + 'locals', 'map', 'max', 'min', 'next', 'oct', 'open', 'ord', 'pow', + 'print', 'repr', 'reversed', 'round', 'setattr', 'scanl1?', 'snd', + 'sorted', 'sum', 'tail', 'take', 'takewhile', 'vars', 'zip'), + prefix=r'(?<!\.)', suffix=r'(?![\'\w])'), Name.Builtin), (r"(?<!\.)(self|Ellipsis|NotImplemented|None|True|False)(?!['\w])", Name.Builtin.Pseudo), @@ -741,7 +821,7 @@ class DgLexer(RegexLexer): ], 'string': [ (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' - '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol), + '[hlL]?[E-GXc-giorsux%]', String.Interpol), (r'[^\\\'"%\n]+', String), # quotes, percents and backslashes must be parsed one at a time (r'[\'"\\]', String), diff --git a/pygments/lexers/qvt.py b/pygments/lexers/qvt.py index 5bc61310..f496d600 100644 --- a/pygments/lexers/qvt.py +++ b/pygments/lexers/qvt.py @@ -5,11 +5,12 @@ Lexer for QVT Operational language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -from pygments.lexer import RegexLexer, bygroups, include, combined +from pygments.lexer import RegexLexer, bygroups, include, combined, default, \ + words from pygments.token import Text, Comment, Operator, Keyword, Punctuation, \ Name, String, Number @@ -50,23 +51,26 @@ class QVToLexer(RegexLexer): bygroups(Comment, Comment, Comment.Preproc, Comment)), # Uncomment the following if you want to distinguish between # '/*' and '/**', à la javadoc - #(r'/[*]{2}(.|\n)*?[*]/', Comment.Multiline), + # (r'/[*]{2}(.|\n)*?[*]/', Comment.Multiline), (r'/[*](.|\n)*?[*]/', Comment.Multiline), (r'\\\n', Text), (r'(and|not|or|xor|##?)\b', Operator.Word), - (r'([:]{1-2}=|[-+]=)\b', Operator.Word), - (r'(@|<<|>>)\b', Keyword), # stereotypes - (r'!=|<>|=|==|!->|->|>=|<=|[.]{3}|[+/*%=<>&|.~]', Operator), + (r'(:{1,2}=|[-+]=)\b', Operator.Word), + (r'(@|<<|>>)\b', Keyword), # stereotypes + (r'!=|<>|==|=|!->|->|>=|<=|[.]{3}|[+/*%=<>&|.~]', Operator), (r'[]{}:(),;[]', Punctuation), (r'(true|false|unlimited|null)\b', Keyword.Constant), (r'(this|self|result)\b', Name.Builtin.Pseudo), (r'(var)\b', Keyword.Declaration), (r'(from|import)\b', Keyword.Namespace, 'fromimport'), - (r'(metamodel|class|exception|primitive|enum|transformation|library)(\s+)([a-zA-Z0-9_]+)', + (r'(metamodel|class|exception|primitive|enum|transformation|' + r'library)(\s+)(\w+)', bygroups(Keyword.Word, Text, Name.Class)), - (r'(exception)(\s+)([a-zA-Z0-9_]+)', bygroups(Keyword.Word, Text, Name.Exception)), + (r'(exception)(\s+)(\w+)', + bygroups(Keyword.Word, Text, Name.Exception)), (r'(main)\b', Name.Function), - (r'(mapping|helper|query)(\s+)', bygroups(Keyword.Declaration, Text), 'operation'), + (r'(mapping|helper|query)(\s+)', + bygroups(Keyword.Declaration, Text), 'operation'), (r'(assert)(\s+)\b', bygroups(Keyword, Text), 'assert'), (r'(Bag|Collection|Dict|OrderedSet|Sequence|Set|Tuple|List)\b', Keyword.Type), @@ -75,46 +79,45 @@ class QVToLexer(RegexLexer): ("'", String, combined('stringescape', 'sqs')), include('name'), include('numbers'), - # (r'([a-zA-Z_][a-zA-Z0-9_]*)(::)([a-zA-Z_][a-zA-Z0-9_]*)', + # (r'([a-zA-Z_]\w*)(::)([a-zA-Z_]\w*)', # bygroups(Text, Text, Text)), - ], + ], 'fromimport': [ (r'(?:[ \t]|\\\n)+', Text), - (r'[a-zA-Z_][a-zA-Z0-9_.]*', Name.Namespace), - (r'', Text, '#pop'), - ], + (r'[a-zA-Z_][\w.]*', Name.Namespace), + default('#pop'), + ], 'operation': [ (r'::', Text), - (r'(.*::)([a-zA-Z_][a-zA-Z0-9_]*)[ \t]*(\()', bygroups(Text,Name.Function, Text), '#pop') - ], + (r'(.*::)([a-zA-Z_]\w*)([ \t]*)(\()', + bygroups(Text, Name.Function, Text, Punctuation), '#pop') + ], 'assert': [ (r'(warning|error|fatal)\b', Keyword, '#pop'), - (r'', Text, '#pop') # all else: go back - ], + default('#pop'), # all else: go back + ], 'keywords': [ - (r'(abstract|access|any|assert|' - r'blackbox|break|case|collect|collectNested|' - r'collectOne|collectselect|collectselectOne|composes|' - r'compute|configuration|constructor|continue|datatype|' - r'default|derived|disjuncts|do|elif|else|end|' - r'endif|except|exists|extends|' - r'forAll|forEach|forOne|from|if|' - r'implies|in|inherits|init|inout|' - r'intermediate|invresolve|invresolveIn|invresolveone|' - r'invresolveoneIn|isUnique|iterate|late|let|' - r'literal|log|map|merges|' - r'modeltype|new|object|one|' - r'ordered|out|package|population|' - r'property|raise|readonly|references|refines|' - r'reject|resolve|resolveIn|resolveone|resolveoneIn|' - r'return|select|selectOne|sortedBy|static|switch|' - r'tag|then|try|typedef|' - r'unlimited|uses|when|where|while|with|' - r'xcollect|xmap|xselect)\b', Keyword), + (words(( + 'abstract', 'access', 'any', 'assert', 'blackbox', 'break', + 'case', 'collect', 'collectNested', 'collectOne', 'collectselect', + 'collectselectOne', 'composes', 'compute', 'configuration', + 'constructor', 'continue', 'datatype', 'default', 'derived', + 'disjuncts', 'do', 'elif', 'else', 'end', 'endif', 'except', + 'exists', 'extends', 'forAll', 'forEach', 'forOne', 'from', 'if', + 'implies', 'in', 'inherits', 'init', 'inout', 'intermediate', + 'invresolve', 'invresolveIn', 'invresolveone', 'invresolveoneIn', + 'isUnique', 'iterate', 'late', 'let', 'literal', 'log', 'map', + 'merges', 'modeltype', 'new', 'object', 'one', 'ordered', 'out', + 'package', 'population', 'property', 'raise', 'readonly', + 'references', 'refines', 'reject', 'resolve', 'resolveIn', + 'resolveone', 'resolveoneIn', 'return', 'select', 'selectOne', + 'sortedBy', 'static', 'switch', 'tag', 'then', 'try', 'typedef', + 'unlimited', 'uses', 'when', 'where', 'while', 'with', 'xcollect', + 'xmap', 'xselect'), suffix=r'\b'), Keyword), ], # There is no need to distinguish between String.Single and @@ -127,18 +130,18 @@ class QVToLexer(RegexLexer): 'stringescape': [ (r'\\([\\btnfr"\']|u[0-3][0-7]{2}|u[0-7]{1,2})', String.Escape) ], - 'dqs': [ # double-quoted string + 'dqs': [ # double-quoted string (r'"', String, '#pop'), (r'\\\\|\\"', String.Escape), include('strings') ], - 'sqs': [ # single-quoted string + 'sqs': [ # single-quoted string (r"'", String, '#pop'), (r"\\\\|\\'", String.Escape), include('strings') ], 'name': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), ], # numbers: excerpt taken from the python lexer 'numbers': [ @@ -146,5 +149,4 @@ class QVToLexer(RegexLexer): (r'\d+[eE][+-]?[0-9]+', Number.Float), (r'\d+', Number.Integer) ], - } - + } diff --git a/pygments/lexers/r.py b/pygments/lexers/r.py index 1a47ca26..dce61969 100644 --- a/pygments/lexers/r.py +++ b/pygments/lexers/r.py @@ -5,7 +5,7 @@ Lexers for the R/S languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/rdf.py b/pygments/lexers/rdf.py index 103b4ad0..d0f8778a 100644 --- a/pygments/lexers/rdf.py +++ b/pygments/lexers/rdf.py @@ -5,7 +5,7 @@ Lexers for semantic web and RDF query languages and markup. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -42,8 +42,7 @@ class SparqlLexer(RegexLexer): u'\u2c00-\u2fef' u'\u3001-\ud7ff' u'\uf900-\ufdcf' - u'\ufdf0-\ufffd' - u'\U00010000-\U000effff') + u'\ufdf0-\ufffd') PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_') @@ -56,7 +55,7 @@ class SparqlLexer(RegexLexer): HEX_GRP = '0-9A-Fa-f' - PN_LOCAL_ESC_CHARS_GRP = r' _~.\-!$&""()*+,;=/?#@%' + PN_LOCAL_ESC_CHARS_GRP = r' _~.\-!$&"()*+,;=/?#@%' # terminal productions :: @@ -191,7 +190,7 @@ class TurtleLexer(RegexLexer): flags = re.IGNORECASE patterns = { - 'PNAME_NS': r'((?:[a-zA-Z][\w-]*)?\:)', # Simplified character range + 'PNAME_NS': r'((?:[a-z][\w-]*)?\:)', # Simplified character range 'IRIREF': r'(<[^<>"{}|^`\\\x00-\x20]*>)' } @@ -258,8 +257,7 @@ class TurtleLexer(RegexLexer): (r'.', String, '#pop'), ], 'end-of-string': [ - - (r'(@)([a-zA-Z]+(:?-[a-zA-Z0-9]+)*)', + (r'(@)([a-z]+(:?-[a-z0-9]+)*)', bygroups(Operator, Generic.Emph), '#pop:2'), (r'(\^\^)%(IRIREF)s' % patterns, bygroups(Operator, Generic.Emph), '#pop:2'), diff --git a/pygments/lexers/rebol.py b/pygments/lexers/rebol.py index b844ad96..f3d00200 100644 --- a/pygments/lexers/rebol.py +++ b/pygments/lexers/rebol.py @@ -5,7 +5,7 @@ Lexers for the REBOL and related languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/resource.py b/pygments/lexers/resource.py index 4647bef8..f7494904 100644 --- a/pygments/lexers/resource.py +++ b/pygments/lexers/resource.py @@ -5,7 +5,7 @@ Lexer for resource definition files. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -81,4 +81,5 @@ class ResourceLexer(RegexLexer): } def analyse_text(text): - return text.startswith('root:table') + if text.startswith('root:table'): + return 1.0 diff --git a/pygments/lexers/rnc.py b/pygments/lexers/rnc.py new file mode 100644 index 00000000..2f2aacdd --- /dev/null +++ b/pygments/lexers/rnc.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.rnc + ~~~~~~~~~~~~~~~~~~~ + + Lexer for Relax-NG Compact syntax + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Punctuation + +__all__ = ['RNCCompactLexer'] + + +class RNCCompactLexer(RegexLexer): + """ + For `RelaxNG-compact <http://relaxng.org>`_ syntax. + + .. versionadded:: 2.2 + """ + + name = 'Relax-NG Compact' + aliases = ['rnc', 'rng-compact'] + filenames = ['*.rnc'] + + tokens = { + 'root': [ + (r'namespace\b', Keyword.Namespace), + (r'(?:default|datatypes)\b', Keyword.Declaration), + (r'##.*$', Comment.Preproc), + (r'#.*$', Comment.Single), + (r'"[^"]*"', String.Double), + # TODO single quoted strings and escape sequences outside of + # double-quoted strings + (r'(?:element|attribute|mixed)\b', Keyword.Declaration, 'variable'), + (r'(text\b|xsd:[^ ]+)', Keyword.Type, 'maybe_xsdattributes'), + (r'[,?&*=|~]|>>', Operator), + (r'[(){}]', Punctuation), + (r'.', Text), + ], + + # a variable has been declared using `element` or `attribute` + 'variable': [ + (r'[^{]+', Name.Variable), + (r'\{', Punctuation, '#pop'), + ], + + # after an xsd:<datatype> declaration there may be attributes + 'maybe_xsdattributes': [ + (r'\{', Punctuation, 'xsdattributes'), + (r'\}', Punctuation, '#pop'), + (r'.', Text), + ], + + # attributes take the form { key1 = value1 key2 = value2 ... } + 'xsdattributes': [ + (r'[^ =}]', Name.Attribute), + (r'=', Operator), + (r'"[^"]*"', String.Double), + (r'\}', Punctuation, '#pop'), + (r'.', Text), + ], + } diff --git a/pygments/lexers/roboconf.py b/pygments/lexers/roboconf.py index 59755a68..8c7df83d 100644 --- a/pygments/lexers/roboconf.py +++ b/pygments/lexers/roboconf.py @@ -5,7 +5,7 @@ Lexers for Roboconf DSL. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/robotframework.py b/pygments/lexers/robotframework.py index eab06efe..e868127b 100644 --- a/pygments/lexers/robotframework.py +++ b/pygments/lexers/robotframework.py @@ -5,7 +5,7 @@ Lexer for Robot Framework. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/ruby.py b/pygments/lexers/ruby.py index e81d6ecf..fe750f1a 100644 --- a/pygments/lexers/ruby.py +++ b/pygments/lexers/ruby.py @@ -5,7 +5,7 @@ Lexers for Ruby and related languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -47,9 +47,9 @@ class RubyLexer(ExtendedRegexLexer): start = match.start(1) yield start, Operator, match.group(1) # <<-? - yield match.start(2), String.Heredoc, match.group(2) # quote ", ', ` - yield match.start(3), Name.Constant, match.group(3) # heredoc name - yield match.start(4), String.Heredoc, match.group(4) # quote again + yield match.start(2), String.Heredoc, match.group(2) # quote ", ', ` + yield match.start(3), String.Delimiter, match.group(3) # heredoc name + yield match.start(4), String.Heredoc, match.group(4) # quote again heredocstack = ctx.__dict__.setdefault('heredocstack', []) outermost = not bool(heredocstack) @@ -74,7 +74,7 @@ class RubyLexer(ExtendedRegexLexer): if check == hdname: for amatch in lines: yield amatch.start(), String.Heredoc, amatch.group() - yield match.start(), Name.Constant, match.group() + yield match.start(), String.Delimiter, match.group() ctx.pos = match.end() break else: diff --git a/pygments/lexers/rust.py b/pygments/lexers/rust.py index 5d1162b8..6914f54d 100644 --- a/pygments/lexers/rust.py +++ b/pygments/lexers/rust.py @@ -5,7 +5,7 @@ Lexers for the Rust language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -18,7 +18,7 @@ __all__ = ['RustLexer'] class RustLexer(RegexLexer): """ - Lexer for the Rust programming language (version 1.0). + Lexer for the Rust programming language (version 1.10). .. versionadded:: 1.6 """ @@ -27,6 +27,35 @@ class RustLexer(RegexLexer): aliases = ['rust'] mimetypes = ['text/rust'] + keyword_types = ( + words(('u8', 'u16', 'u32', 'u64', 'i8', 'i16', 'i32', 'i64', + 'usize', 'isize', 'f32', 'f64', 'str', 'bool'), + suffix=r'\b'), + Keyword.Type) + + builtin_types = (words(( + # Reexported core operators + 'Copy', 'Send', 'Sized', 'Sync', + 'Drop', 'Fn', 'FnMut', 'FnOnce', + + # Reexported types and traits + 'Box', + 'ToOwned', + 'Clone', + 'PartialEq', 'PartialOrd', 'Eq', 'Ord', + 'AsRef', 'AsMut', 'Into', 'From', + 'Default', + 'Iterator', 'Extend', 'IntoIterator', + 'DoubleEndedIterator', 'ExactSizeIterator', + 'Option', + 'Some', 'None', + 'Result', + 'Ok', 'Err', + 'SliceConcatExt', + 'String', 'ToString', + 'Vec'), suffix=r'\b'), + Name.Builtin) + tokens = { 'root': [ # rust allows a file to start with a shebang, but if the first line @@ -49,50 +78,32 @@ class RustLexer(RegexLexer): (r"""\$([a-zA-Z_]\w*|\(,?|\),?|,?)""", Comment.Preproc), # Keywords (words(( - 'as', 'box', 'crate', 'do', 'else', 'enum', 'extern', # break and continue are in labels - 'fn', 'for', 'if', 'impl', 'in', 'loop', 'match', 'mut', 'priv', - 'proc', 'pub', 'ref', 'return', 'static', 'struct', - 'trait', 'true', 'type', 'unsafe', 'while'), suffix=r'\b'), + 'as', 'box', 'const', 'crate', 'else', 'extern', + 'for', 'if', 'impl', 'in', 'loop', 'match', 'move', + 'mut', 'pub', 'ref', 'return', 'static', 'super', + 'trait', 'unsafe', 'use', 'where', 'while'), suffix=r'\b'), Keyword), - (words(('alignof', 'be', 'const', 'offsetof', 'pure', 'sizeof', - 'typeof', 'once', 'unsized', 'yield'), suffix=r'\b'), + (words(('abstract', 'alignof', 'become', 'do', 'final', 'macro', + 'offsetof', 'override', 'priv', 'proc', 'pure', 'sizeof', + 'typeof', 'unsized', 'virtual', 'yield'), suffix=r'\b'), Keyword.Reserved), - (r'(mod|use)\b', Keyword.Namespace), (r'(true|false)\b', Keyword.Constant), + (r'mod\b', Keyword, 'modname'), (r'let\b', Keyword.Declaration), - (words(('u8', 'u16', 'u32', 'u64', 'i8', 'i16', 'i32', 'i64', 'usize', - 'isize', 'f32', 'f64', 'str', 'bool'), suffix=r'\b'), - Keyword.Type), + (r'fn\b', Keyword, 'funcname'), + (r'(struct|enum|type|union)\b', Keyword, 'typename'), + (r'(default)(\s+)(type|fn)\b', bygroups(Keyword, Text, Keyword)), + keyword_types, (r'self\b', Name.Builtin.Pseudo), # Prelude (taken from Rust’s src/libstd/prelude.rs) - (words(( - # Reexported core operators - 'Copy', 'Send', 'Sized', 'Sync', - 'Drop', 'Fn', 'FnMut', 'FnOnce', - - # Reexported functions - 'drop', - - # Reexported types and traits - 'Box', - 'ToOwned', - 'Clone', - 'PartialEq', 'PartialOrd', 'Eq', 'Ord', - 'AsRef', 'AsMut', 'Into', 'From', - 'Default', - 'Iterator', 'Extend', 'IntoIterator', - 'DoubleEndedIterator', 'ExactSizeIterator', - 'Option', - 'Some', 'None', - 'Result', - 'Ok', 'Err', - 'SliceConcatExt', - 'String', 'ToString', - 'Vec', - ), suffix=r'\b'), - Name.Builtin), + builtin_types, + # Path seperators, so types don't catch them. + (r'::\b', Text), + # Types in positions. + (r'(?::|->)', Text, 'typename'), # Labels - (r'(break|continue)(\s*)(\'[A-Za-z_]\w*)?', bygroups(Keyword, Text.Whitespace, Name.Label)), + (r'(break|continue)(\s*)(\'[A-Za-z_]\w*)?', + bygroups(Keyword, Text.Whitespace, Name.Label)), # Character Literal (r"""'(\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0""" r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""", @@ -108,7 +119,8 @@ class RustLexer(RegexLexer): (r'0[xX][0-9a-fA-F_]+', Number.Hex, 'number_lit'), # Decimal Literal (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|' - r'\.[0-9_]*(?!\.)|[eE][+\-]?[0-9_]+)', Number.Float, 'number_lit'), + r'\.[0-9_]*(?!\.)|[eE][+\-]?[0-9_]+)', Number.Float, + 'number_lit'), (r'[0-9][0-9_]*', Number.Integer, 'number_lit'), # String Literal (r'b"', String, 'bytestring'), @@ -148,6 +160,24 @@ class RustLexer(RegexLexer): (r'\*/', String.Doc, '#pop'), (r'[*/]', String.Doc), ], + 'modname': [ + (r'\s+', Text), + (r'[a-zA-Z_]\w*', Name.Namespace, '#pop'), + default('#pop'), + ], + 'funcname': [ + (r'\s+', Text), + (r'[a-zA-Z_]\w*', Name.Function, '#pop'), + default('#pop'), + ], + 'typename': [ + (r'\s+', Text), + (r'&', Keyword.Pseudo), + builtin_types, + keyword_types, + (r'[a-zA-Z_]\w*', Name.Class, '#pop'), + default('#pop'), + ], 'number_lit': [ (r'[ui](8|16|32|64|size)', Keyword, '#pop'), (r'f(32|64)', Keyword, '#pop'), diff --git a/pygments/lexers/sas.py b/pygments/lexers/sas.py new file mode 100644 index 00000000..3747ed9a --- /dev/null +++ b/pygments/lexers/sas.py @@ -0,0 +1,228 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.sas + ~~~~~~~~~~~~~~~~~~~ + + Lexer for SAS. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re +from pygments.lexer import RegexLexer, include, words +from pygments.token import Comment, Keyword, Name, Number, String, Text, \ + Other, Generic + +__all__ = ['SASLexer'] + + +class SASLexer(RegexLexer): + """ + For `SAS <http://www.sas.com/>`_ files. + + .. versionadded:: 2.2 + """ + # Syntax from syntax/sas.vim by James Kidd <james.kidd@covance.com> + + name = 'SAS' + aliases = ['sas'] + filenames = ['*.SAS', '*.sas'] + mimetypes = ['text/x-sas', 'text/sas', 'application/x-sas'] + flags = re.IGNORECASE | re.MULTILINE + + builtins_macros = ( + "bquote", "nrbquote", "cmpres", "qcmpres", "compstor", "datatyp", + "display", "do", "else", "end", "eval", "global", "goto", "if", + "index", "input", "keydef", "label", "left", "length", "let", + "local", "lowcase", "macro", "mend", "nrquote", + "nrstr", "put", "qleft", "qlowcase", "qscan", + "qsubstr", "qsysfunc", "qtrim", "quote", "qupcase", "scan", + "str", "substr", "superq", "syscall", "sysevalf", "sysexec", + "sysfunc", "sysget", "syslput", "sysprod", "sysrc", "sysrput", + "then", "to", "trim", "unquote", "until", "upcase", "verify", + "while", "window" + ) + + builtins_conditionals = ( + "do", "if", "then", "else", "end", "until", "while" + ) + + builtins_statements = ( + "abort", "array", "attrib", "by", "call", "cards", "cards4", + "catname", "continue", "datalines", "datalines4", "delete", "delim", + "delimiter", "display", "dm", "drop", "endsas", "error", "file", + "filename", "footnote", "format", "goto", "in", "infile", "informat", + "input", "keep", "label", "leave", "length", "libname", "link", + "list", "lostcard", "merge", "missing", "modify", "options", "output", + "out", "page", "put", "redirect", "remove", "rename", "replace", + "retain", "return", "select", "set", "skip", "startsas", "stop", + "title", "update", "waitsas", "where", "window", "x", "systask" + ) + + builtins_sql = ( + "add", "and", "alter", "as", "cascade", "check", "create", + "delete", "describe", "distinct", "drop", "foreign", "from", + "group", "having", "index", "insert", "into", "in", "key", "like", + "message", "modify", "msgtype", "not", "null", "on", "or", + "order", "primary", "references", "reset", "restrict", "select", + "set", "table", "unique", "update", "validate", "view", "where" + ) + + builtins_functions = ( + "abs", "addr", "airy", "arcos", "arsin", "atan", "attrc", + "attrn", "band", "betainv", "blshift", "bnot", "bor", + "brshift", "bxor", "byte", "cdf", "ceil", "cexist", "cinv", + "close", "cnonct", "collate", "compbl", "compound", + "compress", "cos", "cosh", "css", "curobs", "cv", "daccdb", + "daccdbsl", "daccsl", "daccsyd", "dacctab", "dairy", "date", + "datejul", "datepart", "datetime", "day", "dclose", "depdb", + "depdbsl", "depsl", "depsyd", + "deptab", "dequote", "dhms", "dif", "digamma", + "dim", "dinfo", "dnum", "dopen", "doptname", "doptnum", + "dread", "dropnote", "dsname", "erf", "erfc", "exist", "exp", + "fappend", "fclose", "fcol", "fdelete", "fetch", "fetchobs", + "fexist", "fget", "fileexist", "filename", "fileref", + "finfo", "finv", "fipname", "fipnamel", "fipstate", "floor", + "fnonct", "fnote", "fopen", "foptname", "foptnum", "fpoint", + "fpos", "fput", "fread", "frewind", "frlen", "fsep", "fuzz", + "fwrite", "gaminv", "gamma", "getoption", "getvarc", "getvarn", + "hbound", "hms", "hosthelp", "hour", "ibessel", "index", + "indexc", "indexw", "input", "inputc", "inputn", "int", + "intck", "intnx", "intrr", "irr", "jbessel", "juldate", + "kurtosis", "lag", "lbound", "left", "length", "lgamma", + "libname", "libref", "log", "log10", "log2", "logpdf", "logpmf", + "logsdf", "lowcase", "max", "mdy", "mean", "min", "minute", + "mod", "month", "mopen", "mort", "n", "netpv", "nmiss", + "normal", "note", "npv", "open", "ordinal", "pathname", + "pdf", "peek", "peekc", "pmf", "point", "poisson", "poke", + "probbeta", "probbnml", "probchi", "probf", "probgam", + "probhypr", "probit", "probnegb", "probnorm", "probt", + "put", "putc", "putn", "qtr", "quote", "ranbin", "rancau", + "ranexp", "rangam", "range", "rank", "rannor", "ranpoi", + "rantbl", "rantri", "ranuni", "repeat", "resolve", "reverse", + "rewind", "right", "round", "saving", "scan", "sdf", "second", + "sign", "sin", "sinh", "skewness", "soundex", "spedis", + "sqrt", "std", "stderr", "stfips", "stname", "stnamel", + "substr", "sum", "symget", "sysget", "sysmsg", "sysprod", + "sysrc", "system", "tan", "tanh", "time", "timepart", "tinv", + "tnonct", "today", "translate", "tranwrd", "trigamma", + "trim", "trimn", "trunc", "uniform", "upcase", "uss", "var", + "varfmt", "varinfmt", "varlabel", "varlen", "varname", + "varnum", "varray", "varrayx", "vartype", "verify", "vformat", + "vformatd", "vformatdx", "vformatn", "vformatnx", "vformatw", + "vformatwx", "vformatx", "vinarray", "vinarrayx", "vinformat", + "vinformatd", "vinformatdx", "vinformatn", "vinformatnx", + "vinformatw", "vinformatwx", "vinformatx", "vlabel", + "vlabelx", "vlength", "vlengthx", "vname", "vnamex", "vtype", + "vtypex", "weekday", "year", "yyq", "zipfips", "zipname", + "zipnamel", "zipstate" + ) + + tokens = { + 'root': [ + include('comments'), + include('proc-data'), + include('cards-datalines'), + include('logs'), + include('general'), + (r'.', Text), + ], + # SAS is multi-line regardless, but * is ended by ; + 'comments': [ + (r'^\s*\*.*?;', Comment), + (r'/\*.*?\*/', Comment), + (r'^\s*\*(.|\n)*?;', Comment.Multiline), + (r'/[*](.|\n)*?[*]/', Comment.Multiline), + ], + # Special highlight for proc, data, quit, run + 'proc-data': [ + (r'(^|;)\s*(proc \w+|data|run|quit)[\s;]', + Keyword.Reserved), + ], + # Special highlight cards and datalines + 'cards-datalines': [ + (r'^\s*(datalines|cards)\s*;\s*$', Keyword, 'data'), + ], + 'data': [ + (r'(.|\n)*^\s*;\s*$', Other, '#pop'), + ], + # Special highlight for put NOTE|ERROR|WARNING (order matters) + 'logs': [ + (r'\n?^\s*%?put ', Keyword, 'log-messages'), + ], + 'log-messages': [ + (r'NOTE(:|-).*', Generic, '#pop'), + (r'WARNING(:|-).*', Generic.Emph, '#pop'), + (r'ERROR(:|-).*', Generic.Error, '#pop'), + include('general'), + ], + 'general': [ + include('keywords'), + include('vars-strings'), + include('special'), + include('numbers'), + ], + # Keywords, statements, functions, macros + 'keywords': [ + (words(builtins_statements, + prefix = r'\b', + suffix = r'\b'), + Keyword), + (words(builtins_sql, + prefix = r'\b', + suffix = r'\b'), + Keyword), + (words(builtins_conditionals, + prefix = r'\b', + suffix = r'\b'), + Keyword), + (words(builtins_macros, + prefix = r'%', + suffix = r'\b'), + Name.Builtin), + (words(builtins_functions, + prefix = r'\b', + suffix = r'\('), + Name.Builtin), + ], + # Strings and user-defined variables and macros (order matters) + 'vars-strings': [ + (r'&[a-z_]\w{0,31}\.?', Name.Variable), + (r'%[a-z_]\w{0,31}', Name.Function), + (r'\'', String, 'string_squote'), + (r'"', String, 'string_dquote'), + ], + 'string_squote': [ + ('\'', String, '#pop'), + (r'\\\\|\\"|\\\n', String.Escape), + # AFAIK, macro variables are not evaluated in single quotes + # (r'&', Name.Variable, 'validvar'), + (r'[^$\'\\]+', String), + (r'[$\'\\]', String), + ], + 'string_dquote': [ + (r'"', String, '#pop'), + (r'\\\\|\\"|\\\n', String.Escape), + (r'&', Name.Variable, 'validvar'), + (r'[^$&"\\]+', String), + (r'[$"\\]', String), + ], + 'validvar': [ + (r'[a-z_]\w{0,31}\.?', Name.Variable, '#pop'), + ], + # SAS numbers and special variables + 'numbers': [ + (r'\b[+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+|\.)(E[+-]?[0-9]+)?i?\b', + Number), + ], + 'special': [ + (r'(null|missing|_all_|_automatic_|_character_|_n_|' + r'_infile_|_name_|_null_|_numeric_|_user_|_webout_)', + Keyword.Constant), + ], + # 'operators': [ + # (r'(-|=|<=|>=|<|>|<>|&|!=|' + # r'\||\*|\+|\^|/|!|~|~=)', Operator) + # ], + } diff --git a/pygments/lexers/scripting.py b/pygments/lexers/scripting.py index 4dd9594b..b3af606e 100644 --- a/pygments/lexers/scripting.py +++ b/pygments/lexers/scripting.py @@ -5,7 +5,7 @@ Lexer for scripting and embedded languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -50,36 +50,47 @@ class LuaLexer(RegexLexer): filenames = ['*.lua', '*.wlua'] mimetypes = ['text/x-lua', 'application/x-lua'] + _comment_multiline = r'(?:--\[(?P<level>=*)\[[\w\W]*?\](?P=level)\])' + _comment_single = r'(?:--.*$)' + _space = r'(?:\s+)' + _s = r'(?:%s|%s|%s)' % (_comment_multiline, _comment_single, _space) + _name = r'(?:[^\W\d]\w*)' + tokens = { 'root': [ - # lua allows a file to start with a shebang - (r'#!(.*?)$', Comment.Preproc), + # Lua allows a file to start with a shebang. + (r'#!.*', Comment.Preproc), default('base'), ], + 'ws': [ + (_comment_multiline, Comment.Multiline), + (_comment_single, Comment.Single), + (_space, Text), + ], 'base': [ - (r'(?s)--\[(=*)\[.*?\]\1\]', Comment.Multiline), - ('--.*$', Comment.Single), + include('ws'), + (r'(?i)0x[\da-f]*(\.[\da-f]*)?(p[+-]?\d+)?', Number.Hex), (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number.Float), (r'(?i)\d+e[+-]?\d+', Number.Float), - ('(?i)0x[0-9a-f]*', Number.Hex), (r'\d+', Number.Integer), - (r'\n', Text), - (r'[^\S\n]', Text), # multiline strings (r'(?s)\[(=*)\[.*?\]\1\]', String), - (r'(==|~=|<=|>=|\.\.\.|\.\.|[=+\-*/%^<>#])', Operator), + (r'::', Punctuation, 'label'), + (r'\.{3}', Punctuation), + (r'[=<>|~&+\-*/%#^]+|\.\.', Operator), (r'[\[\]{}().,:;]', Punctuation), (r'(and|or|not)\b', Operator.Word), ('(break|do|else|elseif|end|for|if|in|repeat|return|then|until|' - r'while)\b', Keyword), + r'while)\b', Keyword.Reserved), + (r'goto\b', Keyword.Reserved, 'goto'), (r'(local)\b', Keyword.Declaration), (r'(true|false|nil)\b', Keyword.Constant), - (r'(function)\b', Keyword, 'funcname'), + (r'(function)\b', Keyword.Reserved, 'funcname'), (r'[A-Za-z_]\w*(\.[A-Za-z_]\w*)?', Name), @@ -88,31 +99,38 @@ class LuaLexer(RegexLexer): ], 'funcname': [ - (r'\s+', Text), - ('(?:([A-Za-z_]\w*)(\.))?([A-Za-z_]\w*)', - bygroups(Name.Class, Punctuation, Name.Function), '#pop'), + include('ws'), + (r'[.:]', Punctuation), + (r'%s(?=%s*[.:])' % (_name, _s), Name.Class), + (_name, Name.Function, '#pop'), # inline function ('\(', Punctuation, '#pop'), ], - # if I understand correctly, every character is valid in a lua string, - # so this state is only for later corrections - 'string': [ - ('.', String) + 'goto': [ + include('ws'), + (_name, Name.Label, '#pop'), + ], + + 'label': [ + include('ws'), + (r'::', Punctuation, '#pop'), + (_name, Name.Label), ], 'stringescape': [ - (r'''\\([abfnrtv\\"']|\d{1,3})''', String.Escape) + (r'\\([abfnrtv\\"\']|[\r\n]{1,2}|z\s*|x[0-9a-fA-F]{2}|\d{1,3}|' + r'u\{[0-9a-fA-F]+\})', String.Escape), ], 'sqs': [ - ("'", String, '#pop'), - include('string') + (r"'", String.Single, '#pop'), + (r"[^\\']+", String.Single), ], 'dqs': [ - ('"', String, '#pop'), - include('string') + (r'"', String.Double, '#pop'), + (r'[^\\"]+', String.Double), ] } @@ -1020,11 +1038,11 @@ class EasytrieveLexer(RegexLexer): (r"'(''|[^'])*'", String), (r'\s+', Whitespace), # Everything else just belongs to a name - (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name) + (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name), ], 'after_declaration': [ (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name.Function), - ('', Whitespace, '#pop') + default('#pop'), ], 'after_macro_argument': [ (r'\*.*\n', Comment.Single, '#pop'), @@ -1032,7 +1050,7 @@ class EasytrieveLexer(RegexLexer): (_OPERATORS_PATTERN, Operator, '#pop'), (r"'(''|[^'])*'", String, '#pop'), # Everything else just belongs to a name - (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name) + (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name), ], } _COMMENT_LINE_REGEX = re.compile(r'^\s*\*') @@ -1122,7 +1140,8 @@ class EasytrieveLexer(RegexLexer): class JclLexer(RegexLexer): """ - `Job Control Language (JCL) <http://publibz.boulder.ibm.com/cgi-bin/bookmgr_OS390/BOOKS/IEA2B570/CCONTENTS>`_ + `Job Control Language (JCL) + <http://publibz.boulder.ibm.com/cgi-bin/bookmgr_OS390/BOOKS/IEA2B570/CCONTENTS>`_ is a scripting language used on mainframe platforms to instruct the system on how to run a batch job or start a subsystem. It is somewhat comparable to MS DOS batch and Unix shell scripts. @@ -1145,10 +1164,10 @@ class JclLexer(RegexLexer): ], 'statement': [ (r'\s*\n', Whitespace, '#pop'), - (r'([a-z][a-z_0-9]*)(\s+)(exec|job)(\s*)', + (r'([a-z]\w*)(\s+)(exec|job)(\s*)', bygroups(Name.Label, Whitespace, Keyword.Reserved, Whitespace), 'option'), - (r'[a-z][a-z_0-9]*', Name.Variable, 'statement_command'), + (r'[a-z]\w*', Name.Variable, 'statement_command'), (r'\s+', Whitespace, 'statement_command'), ], 'statement_command': [ @@ -1167,10 +1186,10 @@ class JclLexer(RegexLexer): (r'\*', Name.Builtin), (r'[\[\](){}<>;,]', Punctuation), (r'[-+*/=&%]', Operator), - (r'[a-z_][a-z_0-9]*', Name), - (r'[0-9]+\.[0-9]*', Number.Float), - (r'\.[0-9]+', Number.Float), - (r'[0-9]+', Number.Integer), + (r'[a-z_]\w*', Name), + (r'\d+\.\d*', Number.Float), + (r'\.\d+', Number.Float), + (r'\d+', Number.Integer), (r"'", String, 'option_string'), (r'[ \t]+', Whitespace, 'option_comment'), (r'\.', Punctuation), diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py index ad2e2d7a..ceb6f14d 100644 --- a/pygments/lexers/shell.py +++ b/pygments/lexers/shell.py @@ -5,7 +5,7 @@ Lexers for various shells. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -27,16 +27,17 @@ line_re = re.compile('.*?\n') class BashLexer(RegexLexer): """ - Lexer for (ba|k|)sh shell scripts. + Lexer for (ba|k|z|)sh shell scripts. .. versionadded:: 0.6 """ name = 'Bash' - aliases = ['bash', 'sh', 'ksh', 'shell'] + aliases = ['bash', 'sh', 'ksh', 'zsh', 'shell'] filenames = ['*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', - '*.exheres-0', '*.exlib', - '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'PKGBUILD'] + '*.exheres-0', '*.exlib', '*.zsh', + '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', + 'PKGBUILD'] mimetypes = ['application/x-sh', 'application/x-shellscript'] tokens = { @@ -50,7 +51,7 @@ class BashLexer(RegexLexer): (r'\$\(\(', Keyword, 'math'), (r'\$\(', Keyword, 'paren'), (r'\$\{#?', String.Interpol, 'curly'), - (r'\$[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable), # user variable + (r'\$[a-zA-Z_]\w*', Name.Variable), # user variable (r'\$(?:\d+|[#$?!_*@-])', Name.Variable), # builtin (r'\$', Text), ], @@ -68,7 +69,7 @@ class BashLexer(RegexLexer): (r'\A#!.+\n', Comment.Hashbang), (r'#.*\n', Comment.Single), (r'\\[\w\W]', String.Escape), - (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Text, Operator)), + (r'(\b\w+)(\s*)(\+?=)', bygroups(Name.Variable, Text, Operator)), (r'[\[\]{}()=]', Operator), (r'<<<', Operator), # here-string (r'<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2', String), @@ -83,7 +84,7 @@ class BashLexer(RegexLexer): (r'&', Punctuation), (r'\|', Punctuation), (r'\s+', Text), - (r'\d+(?= |\Z)', Number), + (r'\d+\b', Number), (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text), (r'<', Text), ], @@ -137,11 +138,15 @@ class ShellSessionBaseLexer(Lexer): pos = 0 curcode = '' insertions = [] + backslash_continuation = False for match in line_re.finditer(text): line = match.group() m = re.match(self._ps1rgx, line) - if m: + if backslash_continuation: + curcode += line + backslash_continuation = curcode.endswith('\\\n') + elif m: # To support output lexers (say diff output), the output # needs to be broken by prompts whenever the output lexer # changes. @@ -151,10 +156,12 @@ class ShellSessionBaseLexer(Lexer): insertions.append((len(curcode), [(0, Generic.Prompt, m.group(1))])) curcode += m.group(2) + backslash_continuation = curcode.endswith('\\\n') elif line.startswith(self._ps2): insertions.append((len(curcode), [(0, Generic.Prompt, line[:len(self._ps2)])])) curcode += line[len(self._ps2):] + backslash_continuation = curcode.endswith('\\\n') else: if insertions: toks = innerlexer.get_tokens_unprocessed(curcode) @@ -452,9 +459,9 @@ class BatchLexer(RegexLexer): bygroups(String.Double, using(this, state='string'), Text, Punctuation)), (r'"', String.Double, ('#pop', 'for2', 'string')), - (r"('(?:%s|[\w\W])*?')([%s%s]*)(\))" % (_variable, _nl, _ws), + (r"('(?:%%%%|%s|[\w\W])*?')([%s%s]*)(\))" % (_variable, _nl, _ws), bygroups(using(this, state='sqstring'), Text, Punctuation)), - (r'(`(?:%s|[\w\W])*?`)([%s%s]*)(\))' % (_variable, _nl, _ws), + (r'(`(?:%%%%|%s|[\w\W])*?`)([%s%s]*)(\))' % (_variable, _nl, _ws), bygroups(using(this, state='bqstring'), Text, Punctuation)), include('for2') ], @@ -472,13 +479,16 @@ class BatchLexer(RegexLexer): using(this, state='variable')), '#pop'), (r'(exist%s)(%s%s)' % (_token_terminator, _space, _stoken), bygroups(Keyword, using(this, state='text')), '#pop'), - (r'(%s%s?)(==)(%s?%s)' % (_stoken, _space, _space, _stoken), - bygroups(using(this, state='text'), Operator, - using(this, state='text')), '#pop'), (r'(%s%s)(%s)(%s%s)' % (_number, _space, _opword, _space, _number), bygroups(using(this, state='arithmetic'), Operator.Word, using(this, state='arithmetic')), '#pop'), - (r'(%s%s)(%s)(%s%s)' % (_stoken, _space, _opword, _space, _stoken), + (_stoken, using(this, state='text'), ('#pop', 'if2')), + ], + 'if2': [ + (r'(%s?)(==)(%s?%s)' % (_space, _space, _stoken), + bygroups(using(this, state='text'), Operator, + using(this, state='text')), '#pop'), + (r'(%s)(%s)(%s%s)' % (_space, _opword, _space, _stoken), bygroups(using(this, state='text'), Operator.Word, using(this, state='text')), '#pop') ], diff --git a/pygments/lexers/smalltalk.py b/pygments/lexers/smalltalk.py index ebeb6320..79078b66 100644 --- a/pygments/lexers/smalltalk.py +++ b/pygments/lexers/smalltalk.py @@ -5,7 +5,7 @@ Lexers for Smalltalk and related languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/smv.py b/pygments/lexers/smv.py new file mode 100644 index 00000000..380a3b70 --- /dev/null +++ b/pygments/lexers/smv.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.smv + ~~~~~~~~~~~~~~~~~~~ + + Lexers for the SMV languages. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, words +from pygments.token import Comment, Generic, Keyword, Name, Number, \ + Operator, Punctuation, Text + +__all__ = ['NuSMVLexer'] + + +class NuSMVLexer(RegexLexer): + """ + Lexer for the NuSMV language. + + .. versionadded:: 2.2 + """ + + name = 'NuSMV' + aliases = ['nusmv'] + filenames = ['*.smv'] + mimetypes = [] + + tokens = { + 'root': [ + # Comments + (r'(?s)\/\-\-.*?\-\-/', Comment), + (r'--.*\n', Comment), + + # Reserved + (words(('MODULE', 'DEFINE', 'MDEFINE', 'CONSTANTS', 'VAR', 'IVAR', + 'FROZENVAR', 'INIT', 'TRANS', 'INVAR', 'SPEC', 'CTLSPEC', + 'LTLSPEC', 'PSLSPEC', 'COMPUTE', 'NAME', 'INVARSPEC', + 'FAIRNESS', 'JUSTICE', 'COMPASSION', 'ISA', 'ASSIGN', + 'CONSTRAINT', 'SIMPWFF', 'CTLWFF', 'LTLWFF', 'PSLWFF', + 'COMPWFF', 'IN', 'MIN', 'MAX', 'MIRROR', 'PRED', + 'PREDICATES'), suffix=r'(?![\w$#-])'), + Keyword.Declaration), + (r'process(?![\w$#-])', Keyword), + (words(('array', 'of', 'boolean', 'integer', 'real', 'word'), + suffix=r'(?![\w$#-])'), Keyword.Type), + (words(('case', 'esac'), suffix=r'(?![\w$#-])'), Keyword), + (words(('word1', 'bool', 'signed', 'unsigned', 'extend', 'resize', + 'sizeof', 'uwconst', 'swconst', 'init', 'self', 'count', + 'abs', 'max', 'min'), suffix=r'(?![\w$#-])'), + Name.Builtin), + (words(('EX', 'AX', 'EF', 'AF', 'EG', 'AG', 'E', 'F', 'O', 'G', + 'H', 'X', 'Y', 'Z', 'A', 'U', 'S', 'V', 'T', 'BU', 'EBF', + 'ABF', 'EBG', 'ABG', 'next', 'mod', 'union', 'in', 'xor', + 'xnor'), suffix=r'(?![\w$#-])'), + Operator.Word), + (words(('TRUE', 'FALSE'), suffix=r'(?![\w$#-])'), Keyword.Constant), + + # Names + (r'[a-zA-Z_][\w$#-]*', Name.Variable), + + # Operators + (r':=', Operator), + (r'[-&|+*/<>!=]', Operator), + + # Literals + (r'\-?\d+\b', Number.Integer), + (r'0[su][bB]\d*_[01_]+', Number.Bin), + (r'0[su][oO]\d*_[0-7_]+', Number.Oct), + (r'0[su][dD]\d*_[\d_]+', Number.Dec), + (r'0[su][hH]\d*_[\da-fA-F_]+', Number.Hex), + + # Whitespace, punctuation and the rest + (r'\s+', Text.Whitespace), + (r'[()\[\]{};?:.,]', Punctuation), + ], + } diff --git a/pygments/lexers/snobol.py b/pygments/lexers/snobol.py index e4178f9c..f6e12fd2 100644 --- a/pygments/lexers/snobol.py +++ b/pygments/lexers/snobol.py @@ -5,7 +5,7 @@ Lexers for the SNOBOL language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/special.py b/pygments/lexers/special.py index d3a168e7..6e076b0c 100644 --- a/pygments/lexers/special.py +++ b/pygments/lexers/special.py @@ -5,7 +5,7 @@ Special lexers. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -27,10 +27,13 @@ class TextLexer(Lexer): aliases = ['text'] filenames = ['*.txt'] mimetypes = ['text/plain'] + priority = 0.01 def get_tokens_unprocessed(self, text): yield 0, Text, text + def analyse_text(text): + return TextLexer.priority _ttype_cache = {} diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index a7736f75..7507c0fc 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -34,34 +34,39 @@ The ``tests/examplefiles`` contains a few test files with data to be parsed by these lexers. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups, words -from pygments.token import Punctuation, \ +from pygments.token import Punctuation, Whitespace, Error, \ Text, Comment, Operator, Keyword, Name, String, Number, Generic from pygments.lexers import get_lexer_by_name, ClassNotFound from pygments.util import iteritems from pygments.lexers._postgres_builtins import KEYWORDS, DATATYPES, \ PSEUDO_TYPES, PLPGSQL_KEYWORDS +from pygments.lexers import _tsql_builtins __all__ = ['PostgresLexer', 'PlPgsqlLexer', 'PostgresConsoleLexer', - 'SqlLexer', 'MySqlLexer', 'SqliteConsoleLexer', 'RqlLexer'] + 'SqlLexer', 'TransactSqlLexer', 'MySqlLexer', + 'SqliteConsoleLexer', 'RqlLexer'] line_re = re.compile('.*?\n') language_re = re.compile(r"\s+LANGUAGE\s+'?(\w+)'?", re.IGNORECASE) +do_re = re.compile(r'\bDO\b', re.IGNORECASE) + def language_callback(lexer, match): """Parse the content of a $-string using a lexer - The lexer is chosen looking for a nearby LANGUAGE. + The lexer is chosen looking for a nearby LANGUAGE or assumed as + plpgsql if inside a DO statement and no LANGUAGE has been found. """ l = None m = language_re.match(lexer.text[match.end():match.end()+100]) @@ -72,15 +77,26 @@ def language_callback(lexer, match): lexer.text[max(0, match.start()-100):match.start()])) if m: l = lexer._get_lexer(m[-1].group(1)) - + else: + m = list(do_re.finditer( + lexer.text[max(0, match.start()-25):match.start()])) + if m: + l = lexer._get_lexer('plpgsql') + + # 1 = $, 2 = delimiter, 3 = $ + yield (match.start(1), String, match.group(1)) + yield (match.start(2), String.Delimiter, match.group(2)) + yield (match.start(3), String, match.group(3)) + # 4 = string contents if l: - yield (match.start(1), String, match.group(1)) - for x in l.get_tokens_unprocessed(match.group(2)): + for x in l.get_tokens_unprocessed(match.group(4)): yield x - yield (match.start(3), String, match.group(3)) - else: - yield (match.start(), String, match.group()) + yield (match.start(4), String, match.group(4)) + # 5 = $, 6 = delimiter, 7 = $ + yield (match.start(5), String, match.group(5)) + yield (match.start(6), String.Delimiter, match.group(6)) + yield (match.start(7), String, match.group(7)) class PostgresBase(object): @@ -137,7 +153,7 @@ class PostgresLexer(PostgresBase, RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (r'--.*?\n', Comment.Single), + (r'--.*\n?', Comment.Single), (r'/\*', Comment.Multiline, 'multiline-comments'), (r'(' + '|'.join(s.replace(" ", "\s+") for s in DATATYPES + PSEUDO_TYPES) @@ -148,9 +164,10 @@ class PostgresLexer(PostgresBase, RegexLexer): (r'\$\d+', Name.Variable), (r'([0-9]*\.[0-9]*|[0-9]+)(e[+-]?[0-9]+)?', Number.Float), (r'[0-9]+', Number.Integer), - (r"(E|U&)?'", String.Single, 'string'), - (r'(U&)?"', String.Name, 'quoted-ident'), # quoted identifier - (r'(?s)(\$[^$]*\$)(.*?)(\1)', language_callback), + (r"((?:E|U&)?)(')", bygroups(String.Affix, String.Single), 'string'), + # quoted identifier + (r'((?:U&)?)(")', bygroups(String.Affix, String.Name), 'quoted-ident'), + (r'(?s)(\$)([^$]*)(\$)(.*?)(\$)(\2)(\$)', language_callback), (r'[a-z_]\w*', Name), # psql variable in SQL @@ -363,7 +380,7 @@ class SqlLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (r'--.*?\n', Comment.Single), + (r'--.*\n?', Comment.Single), (r'/\*', Comment.Multiline, 'multiline-comments'), (words(( 'ABORT', 'ABS', 'ABSOLUTE', 'ACCESS', 'ADA', 'ADD', 'ADMIN', 'AFTER', 'AGGREGATE', @@ -464,6 +481,62 @@ class SqlLexer(RegexLexer): } +class TransactSqlLexer(RegexLexer): + """ + Transact-SQL (T-SQL) is Microsoft's and Sybase's proprietary extension to + SQL. + + The list of keywords includes ODBC and keywords reserved for future use.. + """ + + name = 'Transact-SQL' + aliases = ['tsql', 't-sql'] + filenames = ['*.sql'] + mimetypes = ['text/x-tsql'] + + # Use re.UNICODE to allow non ASCII letters in names. + flags = re.IGNORECASE | re.UNICODE + tokens = { + 'root': [ + (r'\s+', Whitespace), + (r'--(?m).*?$\n?', Comment.Single), + (r'/\*', Comment.Multiline, 'multiline-comments'), + (words(_tsql_builtins.OPERATORS), Operator), + (words(_tsql_builtins.OPERATOR_WORDS, suffix=r'\b'), Operator.Word), + (words(_tsql_builtins.TYPES, suffix=r'\b'), Name.Class), + (words(_tsql_builtins.FUNCTIONS, suffix=r'\b'), Name.Function), + (r'(goto)(\s+)(\w+\b)', bygroups(Keyword, Whitespace, Name.Label)), + (words(_tsql_builtins.KEYWORDS, suffix=r'\b'), Keyword), + (r'(\[)([^]]+)(\])', bygroups(Operator, Name, Operator)), + (r'0x[0-9a-f]+', Number.Hex), + # Float variant 1, for example: 1., 1.e2, 1.2e3 + (r'[0-9]+\.[0-9]*(e[+-]?[0-9]+)?', Number.Float), + # Float variant 2, for example: .1, .1e2 + (r'\.[0-9]+(e[+-]?[0-9]+)?', Number.Float), + # Float variant 3, for example: 123e45 + (r'[0-9]+e[+-]?[0-9]+', Number.Float), + (r'[0-9]+', Number.Integer), + (r"'(''|[^'])*'", String.Single), + (r'"(""|[^"])*"', String.Symbol), + (r'[;(),.]', Punctuation), + # Below we use \w even for the first "real" character because + # tokens starting with a digit have already been recognized + # as Number above. + (r'@@\w+', Name.Builtin), + (r'@\w+', Name.Variable), + (r'(\w+)(:)', bygroups(Name.Label, Punctuation)), + (r'#?#?\w+', Name), # names for temp tables and anything else + (r'\?', Name.Variable.Magic), # parameter for prepared statements + ], + 'multiline-comments': [ + (r'/\*', Comment.Multiline, 'multiline-comments'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[^/*]+', Comment.Multiline), + (r'[/*]', Comment.Multiline) + ] + } + + class MySqlLexer(RegexLexer): """ Special lexer for MySQL. @@ -477,7 +550,7 @@ class MySqlLexer(RegexLexer): tokens = { 'root': [ (r'\s+', Text), - (r'(#|--\s+).*?\n', Comment.Single), + (r'(#|--\s+).*\n?', Comment.Single), (r'/\*', Comment.Multiline, 'multiline-comments'), (r'[0-9]+', Number.Integer), (r'[0-9]*\.[0-9]+(e[+-][0-9]+)', Number.Float), diff --git a/pygments/lexers/stata.py b/pygments/lexers/stata.py new file mode 100644 index 00000000..a015a23e --- /dev/null +++ b/pygments/lexers/stata.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.stata + ~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Stata + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, include, words +from pygments.token import Comment, Keyword, Name, Number, \ + String, Text, Operator + +from pygments.lexers._stata_builtins import builtins_base, builtins_functions + +__all__ = ['StataLexer'] + + +class StataLexer(RegexLexer): + """ + For `Stata <http://www.stata.com/>`_ do files. + + .. versionadded:: 2.2 + """ + # Syntax based on + # - http://fmwww.bc.edu/RePEc/bocode/s/synlightlist.ado + # - http://github.com/isagalaev/highlight.js/blob/master/src/languages/stata.js + # - http://github.com/jpitblado/vim-stata/blob/master/syntax/stata.vim + + name = 'Stata' + aliases = ['stata', 'do'] + filenames = ['*.do', '*.ado'] + mimetypes = ['text/x-stata', 'text/stata', 'application/x-stata'] + + tokens = { + 'root': [ + include('comments'), + include('vars-strings'), + include('numbers'), + include('keywords'), + (r'.', Text), + ], + # Global and local macros; regular and special strings + 'vars-strings': [ + (r'\$[\w{]', Name.Variable.Global, 'var_validglobal'), + (r'`\w{0,31}\'', Name.Variable), + (r'"', String, 'string_dquote'), + (r'`"', String, 'string_mquote'), + ], + # For either string type, highlight macros as macros + 'string_dquote': [ + (r'"', String, '#pop'), + (r'\\\\|\\"|\\\n', String.Escape), + (r'\$', Name.Variable.Global, 'var_validglobal'), + (r'`', Name.Variable, 'var_validlocal'), + (r'[^$`"\\]+', String), + (r'[$"\\]', String), + ], + 'string_mquote': [ + (r'"\'', String, '#pop'), + (r'\\\\|\\"|\\\n', String.Escape), + (r'\$', Name.Variable.Global, 'var_validglobal'), + (r'`', Name.Variable, 'var_validlocal'), + (r'[^$`"\\]+', String), + (r'[$"\\]', String), + ], + 'var_validglobal': [ + (r'\{\w{0,32}\}', Name.Variable.Global, '#pop'), + (r'\w{1,32}', Name.Variable.Global, '#pop'), + ], + 'var_validlocal': [ + (r'\w{0,31}\'', Name.Variable, '#pop'), + ], + # * only OK at line start, // OK anywhere + 'comments': [ + (r'^\s*\*.*$', Comment), + (r'//.*', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline), + (r'/[*](.|\n)*?[*]/', Comment.Multiline), + ], + # Built in functions and statements + 'keywords': [ + (words(builtins_functions, prefix = r'\b', suffix = r'\('), + Name.Function), + (words(builtins_base, prefix = r'(^\s*|\s)', suffix = r'\b'), + Keyword), + ], + # http://www.stata.com/help.cgi?operators + 'operators': [ + (r'-|==|<=|>=|<|>|&|!=', Operator), + (r'\*|\+|\^|/|!|~|==|~=', Operator) + ], + # Stata numbers + 'numbers': [ + # decimal number + (r'\b[+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+|\.)([eE][+-]?[0-9]+)?[i]?\b', + Number), + ], + # Stata formats + 'format': [ + (r'%-?\d{1,2}(\.\d{1,2})?[gfe]c?', Name.Variable), + (r'%(21x|16H|16L|8H|8L)', Name.Variable), + (r'%-?(tc|tC|td|tw|tm|tq|th|ty|tg).{0,32}', Name.Variable), + (r'%[-~]?\d{1,4}s', Name.Variable), + ] + } diff --git a/pygments/lexers/supercollider.py b/pygments/lexers/supercollider.py index d3e4c460..40ff0aeb 100644 --- a/pygments/lexers/supercollider.py +++ b/pygments/lexers/supercollider.py @@ -5,13 +5,13 @@ Lexer for SuperCollider - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re -from pygments.lexer import RegexLexer, include, words +from pygments.lexer import RegexLexer, include, words, default from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation @@ -43,7 +43,7 @@ class SuperColliderLexer(RegexLexer): (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' r'([gim]+\b|\B)', String.Regex, '#pop'), (r'(?=/)', Text, ('#pop', 'badregex')), - (r'', Text, '#pop') + default('#pop'), ], 'badregex': [ (r'\n', Text, '#pop') @@ -74,13 +74,13 @@ class SuperColliderLexer(RegexLexer): (words(('true', 'false', 'nil', 'inf'), suffix=r'\b'), Keyword.Constant), (words(( 'Array', 'Boolean', 'Date', 'Error', 'Function', 'Number', - 'Object', 'Packages', 'RegExp', 'String', 'Error', + 'Object', 'Packages', 'RegExp', 'String', 'isFinite', 'isNaN', 'parseFloat', 'parseInt', 'super', 'thisFunctionDef', 'thisFunction', 'thisMethod', 'thisProcess', 'thisThread', 'this'), suffix=r'\b'), Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), - (r'\\?[$a-zA-Z_][a-zA-Z0-9_]*', String.Symbol), + (r'[$a-zA-Z_]\w*', Name.Other), + (r'\\?[$a-zA-Z_]\w*', String.Symbol), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), diff --git a/pygments/lexers/tcl.py b/pygments/lexers/tcl.py index 96feb7a8..1d1be033 100644 --- a/pygments/lexers/tcl.py +++ b/pygments/lexers/tcl.py @@ -5,7 +5,7 @@ Lexers for Tcl and related languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py index 3e55b6ad..83c57db8 100644 --- a/pygments/lexers/templates.py +++ b/pygments/lexers/templates.py @@ -5,7 +5,7 @@ Lexers for various template engines' markup. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -44,7 +44,7 @@ __all__ = ['HtmlPhpLexer', 'XmlPhpLexer', 'CssPhpLexer', 'TeaTemplateLexer', 'LassoHtmlLexer', 'LassoXmlLexer', 'LassoCssLexer', 'LassoJavascriptLexer', 'HandlebarsLexer', 'HandlebarsHtmlLexer', 'YamlJinjaLexer', 'LiquidLexer', - 'TwigLexer', 'TwigHtmlLexer'] + 'TwigLexer', 'TwigHtmlLexer', 'Angular2Lexer', 'Angular2HtmlLexer'] class ErbLexer(Lexer): @@ -1814,8 +1814,9 @@ class HandlebarsLexer(RegexLexer): (r'\}\}', Comment.Preproc, '#pop'), # Handlebars - (r'([#/]*)(each|if|unless|else|with|log|in)', bygroups(Keyword, + (r'([#/]*)(each|if|unless|else|with|log|in(line)?)', bygroups(Keyword, Keyword)), + (r'#\*inline', Keyword), # General {{#block}} (r'([#/])([\w-]+)', bygroups(Name.Function, Name.Function)), @@ -1823,11 +1824,37 @@ class HandlebarsLexer(RegexLexer): # {{opt=something}} (r'([\w-]+)(=)', bygroups(Name.Attribute, Operator)), + # Partials {{> ...}} + (r'(>)(\s*)(@partial-block)', bygroups(Keyword, Text, Keyword)), + (r'(#?>)(\s*)([\w-]+)', bygroups(Keyword, Text, Name.Variable)), + (r'(>)(\s*)(\()', bygroups(Keyword, Text, Punctuation), + 'dynamic-partial'), + + include('generic'), + ], + 'dynamic-partial': [ + (r'\s+', Text), + (r'\)', Punctuation, '#pop'), + + (r'(lookup)(\s+)(\.|this)(\s+)', bygroups(Keyword, Text, + Name.Variable, Text)), + (r'(lookup)(\s+)(\S+)', bygroups(Keyword, Text, + using(this, state='variable'))), + (r'[\w-]+', Name.Function), + + include('generic'), + ], + 'variable': [ + (r'[a-zA-Z][\w-]*', Name.Variable), + (r'\.[\w-]+', Name.Variable), + (r'(this\/|\.\/|(\.\.\/)+)[\w-]+', Name.Variable), + ], + 'generic': [ + include('variable'), + # borrowed from DjangoLexer (r':?"(\\\\|\\"|[^"])*"', String.Double), (r":?'(\\\\|\\'|[^'])*'", String.Single), - (r'[a-zA-Z][\w-]*', Name.Variable), - (r'\.[\w-]+', Name.Variable), (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), ] @@ -2174,3 +2201,83 @@ class TwigHtmlLexer(DelegatingLexer): def __init__(self, **options): super(TwigHtmlLexer, self).__init__(HtmlLexer, TwigLexer, **options) + + +class Angular2Lexer(RegexLexer): + """ + Generic + `angular2 <http://victorsavkin.com/post/119943127151/angular-2-template-syntax>`_ + template lexer. + + Highlights only the Angular template tags (stuff between `{{` and `}}` and + special attributes: '(event)=', '[property]=', '[(twoWayBinding)]='). + Everything else is left for a delegating lexer. + + .. versionadded:: 2.1 + """ + + name = "Angular2" + aliases = ['ng2'] + + tokens = { + 'root': [ + (r'[^{([*#]+', Other), + + # {{meal.name}} + (r'(\{\{)(\s*)', bygroups(Comment.Preproc, Text), 'ngExpression'), + + # (click)="deleteOrder()"; [value]="test"; [(twoWayTest)]="foo.bar" + (r'([([]+)([\w:.-]+)([\])]+)(\s*)(=)(\s*)', + bygroups(Punctuation, Name.Attribute, Punctuation, Text, Operator, Text), + 'attr'), + (r'([([]+)([\w:.-]+)([\])]+)(\s*)', + bygroups(Punctuation, Name.Attribute, Punctuation, Text)), + + # *ngIf="..."; #f="ngForm" + (r'([*#])([\w:.-]+)(\s*)(=)(\s*)', + bygroups(Punctuation, Name.Attribute, Punctuation, Operator), 'attr'), + (r'([*#])([\w:.-]+)(\s*)', + bygroups(Punctuation, Name.Attribute, Punctuation)), + ], + + 'ngExpression': [ + (r'\s+(\|\s+)?', Text), + (r'\}\}', Comment.Preproc, '#pop'), + + # Literals + (r':?(true|false)', String.Boolean), + (r':?"(\\\\|\\"|[^"])*"', String.Double), + (r":?'(\\\\|\\'|[^'])*'", String.Single), + (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" + r"0[xX][0-9a-fA-F]+[Ll]?", Number), + + # Variabletext + (r'[a-zA-Z][\w-]*(\(.*\))?', Name.Variable), + (r'\.[\w-]+(\(.*\))?', Name.Variable), + + # inline If + (r'(\?)(\s*)([^}\s]+)(\s*)(:)(\s*)([^}\s]+)(\s*)', + bygroups(Operator, Text, String, Text, Operator, Text, String, Text)), + ], + 'attr': [ + ('".*?"', String, '#pop'), + ("'.*?'", String, '#pop'), + (r'[^\s>]+', String, '#pop'), + ], + } + + +class Angular2HtmlLexer(DelegatingLexer): + """ + Subclass of the `Angular2Lexer` that highlights unlexed data with the + `HtmlLexer`. + + .. versionadded:: 2.0 + """ + + name = "HTML + Angular2" + aliases = ["html+ng2"] + filenames = ['*.ng2'] + + def __init__(self, **options): + super(Angular2HtmlLexer, self).__init__(HtmlLexer, Angular2Lexer, **options) diff --git a/pygments/lexers/testing.py b/pygments/lexers/testing.py index 0bdebe74..1e0795b1 100644 --- a/pygments/lexers/testing.py +++ b/pygments/lexers/testing.py @@ -5,7 +5,7 @@ Lexers for testing languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -147,7 +147,7 @@ class TAPLexer(RegexLexer): (r'^TAP version \d+\n', Name.Namespace), # Specify a plan with a plan line. - (r'^1..\d+', Keyword.Declaration, 'plan'), + (r'^1\.\.\d+', Keyword.Declaration, 'plan'), # A test failure (r'^(not ok)([^\S\n]*)(\d*)', diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py index 4bec5ec8..9b3b5fea 100644 --- a/pygments/lexers/text.py +++ b/pygments/lexers/text.py @@ -5,7 +5,7 @@ Lexers for non-source code file types. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/textedit.py b/pygments/lexers/textedit.py index 89417216..e8856dbd 100644 --- a/pygments/lexers/textedit.py +++ b/pygments/lexers/textedit.py @@ -5,7 +5,7 @@ Lexers for languages related to text processing. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/textfmts.py b/pygments/lexers/textfmts.py index cab9add5..bb8124ef 100644 --- a/pygments/lexers/textfmts.py +++ b/pygments/lexers/textfmts.py @@ -5,7 +5,7 @@ Lexers for various text formats. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/theorem.py b/pygments/lexers/theorem.py index 60a101cc..e84a398b 100644 --- a/pygments/lexers/theorem.py +++ b/pygments/lexers/theorem.py @@ -5,7 +5,7 @@ Lexers for theorem-proving languages. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -390,20 +390,23 @@ class LeanLexer(RegexLexer): flags = re.MULTILINE | re.UNICODE - keywords1 = ('import', 'abbreviation', 'opaque_hint', 'tactic_hint', 'definition', 'renaming', - 'inline', 'hiding', 'exposing', 'parameter', 'parameters', 'conjecture', - 'hypothesis', 'lemma', 'corollary', 'variable', 'variables', 'print', 'theorem', - 'axiom', 'inductive', 'structure', 'universe', 'alias', 'help', - 'options', 'precedence', 'postfix', 'prefix', 'calc_trans', 'calc_subst', 'calc_refl', - 'infix', 'infixl', 'infixr', 'notation', 'eval', 'check', 'exit', 'coercion', 'end', - 'private', 'using', 'namespace', 'including', 'instance', 'section', 'context', - 'protected', 'expose', 'export', 'set_option', 'add_rewrite', 'extends', - 'open', 'example', 'constant', 'constants', 'print', 'opaque', 'reducible', 'irreducible' + keywords1 = ( + 'import', 'abbreviation', 'opaque_hint', 'tactic_hint', 'definition', + 'renaming', 'inline', 'hiding', 'exposing', 'parameter', 'parameters', + 'conjecture', 'hypothesis', 'lemma', 'corollary', 'variable', 'variables', + 'theorem', 'axiom', 'inductive', 'structure', 'universe', 'alias', + 'help', 'options', 'precedence', 'postfix', 'prefix', 'calc_trans', + 'calc_subst', 'calc_refl', 'infix', 'infixl', 'infixr', 'notation', 'eval', + 'check', 'exit', 'coercion', 'end', 'private', 'using', 'namespace', + 'including', 'instance', 'section', 'context', 'protected', 'expose', + 'export', 'set_option', 'add_rewrite', 'extends', 'open', 'example', + 'constant', 'constants', 'print', 'opaque', 'reducible', 'irreducible', ) keywords2 = ( - 'forall', 'fun', 'Pi', 'obtain', 'from', 'have', 'show', 'assume', 'take', - 'let', 'if', 'else', 'then', 'by', 'in', 'with', 'begin', 'proof', 'qed', 'calc', 'match' + 'forall', 'fun', 'Pi', 'obtain', 'from', 'have', 'show', 'assume', + 'take', 'let', 'if', 'else', 'then', 'by', 'in', 'with', 'begin', + 'proof', 'qed', 'calc', 'match', ) keywords3 = ( @@ -412,15 +415,16 @@ class LeanLexer(RegexLexer): ) operators = ( - '!=', '#', '&', '&&', '*', '+', '-', '/', '@', '!', '`', - '-.', '->', '.', '..', '...', '::', ':>', ';', ';;', '<', - '<-', '=', '==', '>', '_', '`', '|', '||', '~', '=>', '<=', '>=', - '/\\', '\\/', u'∀', u'Π', u'λ', u'↔', u'∧', u'∨', u'≠', u'≤', u'≥', - u'¬', u'⁻¹', u'⬝', u'▸', u'→', u'∃', u'ℕ', u'ℤ', u'≈', u'×', u'⌞', u'⌟', u'≡', - u'⟨', u'⟩' + u'!=', u'#', u'&', u'&&', u'*', u'+', u'-', u'/', u'@', u'!', u'`', + u'-.', u'->', u'.', u'..', u'...', u'::', u':>', u';', u';;', u'<', + u'<-', u'=', u'==', u'>', u'_', u'|', u'||', u'~', u'=>', u'<=', u'>=', + u'/\\', u'\\/', u'∀', u'Π', u'λ', u'↔', u'∧', u'∨', u'≠', u'≤', u'≥', + u'¬', u'⁻¹', u'⬝', u'▸', u'→', u'∃', u'ℕ', u'ℤ', u'≈', u'×', u'⌞', + u'⌟', u'≡', u'⟨', u'⟩', ) - punctuation = ('(', ')', ':', '{', '}', '[', ']', u'⦃', u'⦄', ':=', ',') + punctuation = (u'(', u')', u':', u'{', u'}', u'[', u']', u'⦃', u'⦄', + u':=', u',') tokens = { 'root': [ diff --git a/pygments/lexers/trafficscript.py b/pygments/lexers/trafficscript.py index 03ab6a06..42542280 100644 --- a/pygments/lexers/trafficscript.py +++ b/pygments/lexers/trafficscript.py @@ -5,7 +5,7 @@ Lexer for RiverBed's TrafficScript (RTS) language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/typoscript.py b/pygments/lexers/typoscript.py new file mode 100644 index 00000000..e358af07 --- /dev/null +++ b/pygments/lexers/typoscript.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.typoscript + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for TypoScript + + `TypoScriptLexer` + A TypoScript lexer. + + `TypoScriptCssDataLexer` + Lexer that highlights markers, constants and registers within css. + + `TypoScriptHtmlDataLexer` + Lexer that highlights markers, constants and registers within html tags. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, bygroups, using +from pygments.token import Text, Comment, Name, String, Number, \ + Operator, Punctuation + +__all__ = ['TypoScriptLexer', 'TypoScriptCssDataLexer', 'TypoScriptHtmlDataLexer'] + + +class TypoScriptCssDataLexer(RegexLexer): + """ + Lexer that highlights markers, constants and registers within css blocks. + + .. versionadded:: 2.2 + """ + + name = 'TypoScriptCssData' + aliases = ['typoscriptcssdata'] + + tokens = { + 'root': [ + # marker: ###MARK### + (r'(.*)(###\w+###)(.*)', bygroups(String, Name.Constant, String)), + # constant: {$some.constant} + (r'(\{)(\$)((?:[\w\-]+\.)*)([\w\-]+)(\})', + bygroups(String.Symbol, Operator, Name.Constant, + Name.Constant, String.Symbol)), # constant + # constant: {register:somevalue} + (r'(.*)(\{)([\w\-]+)(\s*:\s*)([\w\-]+)(\})(.*)', + bygroups(String, String.Symbol, Name.Constant, Operator, + Name.Constant, String.Symbol, String)), # constant + # whitespace + (r'\s+', Text), + # comments + (r'/\*(?:(?!\*/).)*\*/', Comment), + (r'(?<!(#|\'|"))(?:#(?!(?:[a-fA-F0-9]{6}|[a-fA-F0-9]{3}))[^\n#]+|//[^\n]*)', + Comment), + # other + (r'[<>,:=.*%+|]', String), + (r'[\w"\-!/&;(){}]+', String), + ] + } + + +class TypoScriptHtmlDataLexer(RegexLexer): + """ + Lexer that highlights markers, constants and registers within html tags. + + .. versionadded:: 2.2 + """ + + name = 'TypoScriptHtmlData' + aliases = ['typoscripthtmldata'] + + tokens = { + 'root': [ + # INCLUDE_TYPOSCRIPT + (r'(INCLUDE_TYPOSCRIPT)', Name.Class), + # Language label or extension resource FILE:... or LLL:... or EXT:... + (r'(EXT|FILE|LLL):[^}\n"]*', String), + # marker: ###MARK### + (r'(.*)(###\w+###)(.*)', bygroups(String, Name.Constant, String)), + # constant: {$some.constant} + (r'(\{)(\$)((?:[\w\-]+\.)*)([\w\-]+)(\})', + bygroups(String.Symbol, Operator, Name.Constant, + Name.Constant, String.Symbol)), # constant + # constant: {register:somevalue} + (r'(.*)(\{)([\w\-]+)(\s*:\s*)([\w\-]+)(\})(.*)', + bygroups(String, String.Symbol, Name.Constant, Operator, + Name.Constant, String.Symbol, String)), # constant + # whitespace + (r'\s+', Text), + # other + (r'[<>,:=.*%+|]', String), + (r'[\w"\-!/&;(){}#]+', String), + ] + } + + +class TypoScriptLexer(RegexLexer): + """ + Lexer for TypoScript code. + + http://docs.typo3.org/typo3cms/TyposcriptReference/ + + .. versionadded:: 2.2 + """ + + name = 'TypoScript' + aliases = ['typoscript'] + filenames = ['*.ts', '*.txt'] + mimetypes = ['text/x-typoscript'] + + flags = re.DOTALL | re.MULTILINE + + # Slightly higher than TypeScript (which is 0). + priority = 0.1 + + tokens = { + 'root': [ + include('comment'), + include('constant'), + include('html'), + include('label'), + include('whitespace'), + include('keywords'), + include('punctuation'), + include('operator'), + include('structure'), + include('literal'), + include('other'), + ], + 'keywords': [ + # Conditions + (r'(\[)(?i)(browser|compatVersion|dayofmonth|dayofweek|dayofyear|' + r'device|ELSE|END|GLOBAL|globalString|globalVar|hostname|hour|IP|' + r'language|loginUser|loginuser|minute|month|page|PIDinRootline|' + r'PIDupinRootline|system|treeLevel|useragent|userFunc|usergroup|' + r'version)([^\]]*)(\])', + bygroups(String.Symbol, Name.Constant, Text, String.Symbol)), + # Functions + (r'(?=[\w\-])(HTMLparser|HTMLparser_tags|addParams|cache|encapsLines|' + r'filelink|if|imageLinkWrap|imgResource|makelinks|numRows|numberFormat|' + r'parseFunc|replacement|round|select|split|stdWrap|strPad|tableStyle|' + r'tags|textStyle|typolink)(?![\w\-])', Name.Function), + # Toplevel objects and _* + (r'(?:(=?\s*<?\s+|^\s*))(cObj|field|config|content|constants|FEData|' + r'file|frameset|includeLibs|lib|page|plugin|register|resources|sitemap|' + r'sitetitle|styles|temp|tt_[^:.\s]*|types|xmlnews|INCLUDE_TYPOSCRIPT|' + r'_CSS_DEFAULT_STYLE|_DEFAULT_PI_VARS|_LOCAL_LANG)(?![\w\-])', + bygroups(Operator, Name.Builtin)), + # Content objects + (r'(?=[\w\-])(CASE|CLEARGIF|COA|COA_INT|COBJ_ARRAY|COLUMNS|CONTENT|' + r'CTABLE|EDITPANEL|FILE|FILES|FLUIDTEMPLATE|FORM|HMENU|HRULER|HTML|' + r'IMAGE|IMGTEXT|IMG_RESOURCE|LOAD_REGISTER|MEDIA|MULTIMEDIA|OTABLE|' + r'PAGE|QTOBJECT|RECORDS|RESTORE_REGISTER|SEARCHRESULT|SVG|SWFOBJECT|' + r'TEMPLATE|TEXT|USER|USER_INT)(?![\w\-])', Name.Class), + # Menu states + (r'(?=[\w\-])(ACTIFSUBRO|ACTIFSUB|ACTRO|ACT|CURIFSUBRO|CURIFSUB|CURRO|' + r'CUR|IFSUBRO|IFSUB|NO|SPC|USERDEF1RO|USERDEF1|USERDEF2RO|USERDEF2|' + r'USRRO|USR)', Name.Class), + # Menu objects + (r'(?=[\w\-])(GMENU_FOLDOUT|GMENU_LAYERS|GMENU|IMGMENUITEM|IMGMENU|' + r'JSMENUITEM|JSMENU|TMENUITEM|TMENU_LAYERS|TMENU)', Name.Class), + # PHP objects + (r'(?=[\w\-])(PHP_SCRIPT(_EXT|_INT)?)', Name.Class), + (r'(?=[\w\-])(userFunc)(?![\w\-])', Name.Function), + ], + 'whitespace': [ + (r'\s+', Text), + ], + 'html': [ + (r'<\S[^\n>]*>', using(TypoScriptHtmlDataLexer)), + (r'&[^;\n]*;', String), + (r'(_CSS_DEFAULT_STYLE)(\s*)(\()(?s)(.*(?=\n\)))', + bygroups(Name.Class, Text, String.Symbol, using(TypoScriptCssDataLexer))), + ], + 'literal': [ + (r'0x[0-9A-Fa-f]+t?', Number.Hex), + # (r'[0-9]*\.[0-9]+([eE][0-9]+)?[fd]?\s*(?:[^=])', Number.Float), + (r'[0-9]+', Number.Integer), + (r'(###\w+###)', Name.Constant), + ], + 'label': [ + # Language label or extension resource FILE:... or LLL:... or EXT:... + (r'(EXT|FILE|LLL):[^}\n"]*', String), + # Path to a resource + (r'(?![^\w\-])([\w\-]+(?:/[\w\-]+)+/?)(\S*\n)', + bygroups(String, String)), + ], + 'punctuation': [ + (r'[,.]', Punctuation), + ], + 'operator': [ + (r'[<>,:=.*%+|]', Operator), + ], + 'structure': [ + # Brackets and braces + (r'[{}()\[\]\\]', String.Symbol), + ], + 'constant': [ + # Constant: {$some.constant} + (r'(\{)(\$)((?:[\w\-]+\.)*)([\w\-]+)(\})', + bygroups(String.Symbol, Operator, Name.Constant, + Name.Constant, String.Symbol)), # constant + # Constant: {register:somevalue} + (r'(\{)([\w\-]+)(\s*:\s*)([\w\-]+)(\})', + bygroups(String.Symbol, Name.Constant, Operator, + Name.Constant, String.Symbol)), # constant + # Hex color: #ff0077 + (r'(#[a-fA-F0-9]{6}\b|#[a-fA-F0-9]{3}\b)', String.Char) + ], + 'comment': [ + (r'(?<!(#|\'|"))(?:#(?!(?:[a-fA-F0-9]{6}|[a-fA-F0-9]{3}))[^\n#]+|//[^\n]*)', + Comment), + (r'/\*(?:(?!\*/).)*\*/', Comment), + (r'(\s*#\s*\n)', Comment), + ], + 'other': [ + (r'[\w"\-!/&;]+', Text), + ], + } + + def analyse_text(text): + if '<INCLUDE_TYPOSCRIPT:' in text: + return 1.0 diff --git a/pygments/lexers/urbi.py b/pygments/lexers/urbi.py index 558a21fb..7aaba90c 100644 --- a/pygments/lexers/urbi.py +++ b/pygments/lexers/urbi.py @@ -5,7 +5,7 @@ Lexers for UrbiScript language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/varnish.py b/pygments/lexers/varnish.py new file mode 100644 index 00000000..44521422 --- /dev/null +++ b/pygments/lexers/varnish.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.varnish + ~~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for Varnish configuration + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, include, bygroups, using, this, \ + inherit, words +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Literal + +__all__ = ['VCLLexer', 'VCLSnippetLexer'] + + +class VCLLexer(RegexLexer): + """ + For Varnish Configuration Language (VCL). + + .. versionadded:: 2.2 + """ + name = 'VCL' + aliases = ['vcl'] + filenames = ['*.vcl'] + mimetypes = ['text/x-vclsrc'] + + def analyse_text(text): + # If the very first line is 'vcl 4.0;' it's pretty much guaranteed + # that this is VCL + if text.startswith('vcl 4.0;'): + return 1.0 + # Skip over comments and blank lines + # This is accurate enough that returning 0.9 is reasonable. + # Almost no VCL files start without some comments. + elif '\nvcl 4\.0;' in text[:1000]: + return 0.9 + + tokens = { + 'probe': [ + include('whitespace'), + include('comments'), + (r'(\.\w+)(\s*=\s*)([^;]*)(;)', + bygroups(Name.Attribute, Operator, using(this), Punctuation)), + (r'\}', Punctuation, '#pop'), + ], + 'acl': [ + include('whitespace'), + include('comments'), + (r'[!/]+', Operator), + (r';', Punctuation), + (r'\d+', Number), + (r'\}', Punctuation, '#pop'), + ], + 'backend': [ + include('whitespace'), + (r'(\.probe)(\s*=\s*)(\w+)(;)', + bygroups(Name.Attribute, Operator, Name.Variable.Global, Punctuation)), + (r'(\.probe)(\s*=\s*)(\{)', + bygroups(Name.Attribute, Operator, Punctuation), 'probe'), + (r'(\.\w+\b)(\s*=\s*)([^;]*)(\s*;)', + bygroups(Name.Attribute, Operator, using(this), Punctuation)), + (r'\{', Punctuation, '#push'), + (r'\}', Punctuation, '#pop'), + ], + 'statements': [ + (r'(\d\.)?\d+[sdwhmy]', Literal.Date), + (r'(\d\.)?\d+ms', Literal.Date), + (r'(vcl_pass|vcl_hash|vcl_hit|vcl_init|vcl_backend_fetch|vcl_pipe|' + r'vcl_backend_response|vcl_synth|vcl_deliver|vcl_backend_error|' + r'vcl_fini|vcl_recv|vcl_purge|vcl_miss)\b', Name.Function), + (r'(pipe|retry|hash|synth|deliver|purge|abandon|lookup|pass|fail|ok|' + r'miss|fetch|restart)\b', Name.Constant), + (r'(beresp|obj|resp|req|req_top|bereq)\.http\.[a-zA-Z_-]+\b', Name.Variable), + (words(( + 'obj.status', 'req.hash_always_miss', 'beresp.backend', 'req.esi_level', + 'req.can_gzip', 'beresp.ttl', 'obj.uncacheable', 'req.ttl', 'obj.hits', + 'client.identity', 'req.hash_ignore_busy', 'obj.reason', 'req.xid', + 'req_top.proto', 'beresp.age', 'obj.proto', 'obj.age', 'local.ip', + 'beresp.uncacheable', 'req.method', 'beresp.backend.ip', 'now', + 'obj.grace', 'req.restarts', 'beresp.keep', 'req.proto', 'resp.proto', + 'bereq.xid', 'bereq.between_bytes_timeout', 'req.esi', + 'bereq.first_byte_timeout', 'bereq.method', 'bereq.connect_timeout', + 'beresp.do_gzip', 'resp.status', 'beresp.do_gunzip', + 'beresp.storage_hint', 'resp.is_streaming', 'beresp.do_stream', + 'req_top.method', 'bereq.backend', 'beresp.backend.name', 'beresp.status', + 'req.url', 'obj.keep', 'obj.ttl', 'beresp.reason', 'bereq.retries', + 'resp.reason', 'bereq.url', 'beresp.do_esi', 'beresp.proto', 'client.ip', + 'bereq.proto', 'server.hostname', 'remote.ip', 'req.backend_hint', + 'server.identity', 'req_top.url', 'beresp.grace', 'beresp.was_304', + 'server.ip', 'bereq.uncacheable'), suffix=r'\b'), + Name.Variable), + (r'[!%&+*\-,/<.}{>=|~]+', Operator), + (r'[();]', Punctuation), + + (r'[,]+', Punctuation), + (words(('hash_data', 'regsub', 'regsuball', 'if', 'else', + 'elsif', 'elif', 'synth', 'synthetic', 'ban', + 'return', 'set', 'unset', 'import', 'include', 'new', + 'rollback', 'call'), suffix=r'\b'), + Keyword), + (r'storage\.\w+\.\w+\b', Name.Variable), + (words(('true', 'false')), Name.Builtin), + (r'\d+\b', Number), + (r'(backend)(\s+\w+)(\s*\{)', + bygroups(Keyword, Name.Variable.Global, Punctuation), 'backend'), + (r'(probe\s)(\s*\w+\s)(\{)', + bygroups(Keyword, Name.Variable.Global, Punctuation), 'probe'), + (r'(acl\s)(\s*\w+\s)(\{)', + bygroups(Keyword, Name.Variable.Global, Punctuation), 'acl'), + (r'(vcl )(4.0)(;)$', + bygroups(Keyword.Reserved, Name.Constant, Punctuation)), + (r'(sub\s+)([a-zA-Z]\w*)(\s*\{)', + bygroups(Keyword, Name.Function, Punctuation)), + (r'([a-zA-Z_]\w*)' + r'(\.)' + r'([a-zA-Z_]\w*)' + r'(\s*\(.*\))', + bygroups(Name.Function, Punctuation, Name.Function, using(this))), + ('[a-zA-Z_]\w*', Name), + ], + 'comment': [ + (r'[^*/]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline), + ], + 'comments': [ + (r'#.*$', Comment), + (r'/\*', Comment.Multiline, 'comment'), + (r'//.*$', Comment), + ], + 'string': [ + (r'"', String, '#pop'), + (r'[^"\n]+', String), # all other characters + ], + 'multistring': [ + (r'[^"}]', String), + (r'"\}', String, '#pop'), + (r'["}]', String), + ], + 'whitespace': [ + (r'L?"', String, 'string'), + (r'\{"', String, 'multistring'), + (r'\n', Text), + (r'\s+', Text), + (r'\\\n', Text), # line continuation + ], + 'root': [ + include('whitespace'), + include('comments'), + include('statements'), + (r'\s+', Text), + ], + } + + +class VCLSnippetLexer(VCLLexer): + """ + For Varnish Configuration Language snippets. + + .. versionadded:: 2.2 + """ + name = 'VCLSnippets' + aliases = ['vclsnippets', 'vclsnippet'] + mimetypes = ['text/x-vclsnippet'] + filenames = [] + + def analyse_text(text): + # override method inherited from VCLLexer + return 0 + + tokens = { + 'snippetspre': [ + (r'\.\.\.+', Comment), + (r'(bereq|req|req_top|resp|beresp|obj|client|server|local|remote|' + r'storage)($|\.\*)', Name.Variable), + ], + 'snippetspost': [ + (r'(backend)\b', Keyword.Reserved), + ], + 'root': [ + include('snippetspre'), + inherit, + include('snippetspost'), + ], + } diff --git a/pygments/lexers/verification.py b/pygments/lexers/verification.py new file mode 100644 index 00000000..5322e17f --- /dev/null +++ b/pygments/lexers/verification.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.verification + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Intermediate Verification Languages (IVLs). + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, include, words +from pygments.token import Comment, Operator, Keyword, Name, Number, \ + Punctuation, Whitespace + +__all__ = ['BoogieLexer', 'SilverLexer'] + + +class BoogieLexer(RegexLexer): + """ + For `Boogie <https://boogie.codeplex.com/>`_ source code. + + .. versionadded:: 2.1 + """ + name = 'Boogie' + aliases = ['boogie'] + filenames = ['*.bpl'] + + tokens = { + 'root': [ + # Whitespace and Comments + (r'\n', Whitespace), + (r'\s+', Whitespace), + (r'//[/!](.*?)\n', Comment.Doc), + (r'//(.*?)\n', Comment.Single), + (r'/\*', Comment.Multiline, 'comment'), + + (words(( + 'axiom', 'break', 'call', 'ensures', 'else', 'exists', 'function', + 'forall', 'if', 'invariant', 'modifies', 'procedure', 'requires', + 'then', 'var', 'while'), + suffix=r'\b'), Keyword), + (words(('const',), suffix=r'\b'), Keyword.Reserved), + + (words(('bool', 'int', 'ref'), suffix=r'\b'), Keyword.Type), + include('numbers'), + (r"(>=|<=|:=|!=|==>|&&|\|\||[+/\-=>*<\[\]])", Operator), + (r"([{}():;,.])", Punctuation), + # Identifier + (r'[a-zA-Z_]\w*', Name), + ], + 'comment': [ + (r'[^*/]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline), + ], + 'numbers': [ + (r'[0-9]+', Number.Integer), + ], + } + + +class SilverLexer(RegexLexer): + """ + For `Silver <https://bitbucket.org/viperproject/silver>`_ source code. + + .. versionadded:: 2.2 + """ + name = 'Silver' + aliases = ['silver'] + filenames = ['*.sil', '*.vpr'] + + tokens = { + 'root': [ + # Whitespace and Comments + (r'\n', Whitespace), + (r'\s+', Whitespace), + (r'//[/!](.*?)\n', Comment.Doc), + (r'//(.*?)\n', Comment.Single), + (r'/\*', Comment.Multiline, 'comment'), + + (words(( + 'result', 'true', 'false', 'null', 'method', 'function', + 'predicate', 'program', 'domain', 'axiom', 'var', 'returns', + 'field', 'define', 'requires', 'ensures', 'invariant', + 'fold', 'unfold', 'inhale', 'exhale', 'new', 'assert', + 'assume', 'goto', 'while', 'if', 'elseif', 'else', 'fresh', + 'constraining', 'Seq', 'Set', 'Multiset', 'union', 'intersection', + 'setminus', 'subset', 'unfolding', 'in', 'old', 'forall', 'exists', + 'acc', 'wildcard', 'write', 'none', 'epsilon', 'perm', 'unique', + 'apply', 'package', 'folding', 'label', 'forperm'), + suffix=r'\b'), Keyword), + (words(('Int', 'Perm', 'Bool', 'Ref'), suffix=r'\b'), Keyword.Type), + include('numbers'), + + (r'[!%&*+=|?:<>/\-\[\]]', Operator), + (r'([{}():;,.])', Punctuation), + # Identifier + (r'[\w$]\w*', Name), + ], + 'comment': [ + (r'[^*/]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline), + ], + 'numbers': [ + (r'[0-9]+', Number.Integer), + ], + } diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py index 59fbf2fc..6e9c4f92 100644 --- a/pygments/lexers/web.py +++ b/pygments/lexers/web.py @@ -5,7 +5,7 @@ Just export previously exported lexers. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/webmisc.py b/pygments/lexers/webmisc.py index 551846c2..712c8246 100644 --- a/pygments/lexers/webmisc.py +++ b/pygments/lexers/webmisc.py @@ -5,7 +5,7 @@ Lexers for misc. web stuff. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -358,8 +358,10 @@ class XQueryLexer(ExtendedRegexLexer): bygroups(Keyword, Text, Keyword), 'itemtype'), (r'(treat)(\s+)(as)\b', bygroups(Keyword, Text, Keyword), 'itemtype'), - (r'(case)(\s+)(' + stringdouble + ')', bygroups(Keyword, Text, String.Double), 'itemtype'), - (r'(case)(\s+)(' + stringsingle + ')', bygroups(Keyword, Text, String.Single), 'itemtype'), + (r'(case)(\s+)(' + stringdouble + ')', + bygroups(Keyword, Text, String.Double), 'itemtype'), + (r'(case)(\s+)(' + stringsingle + ')', + bygroups(Keyword, Text, String.Single), 'itemtype'), (r'(case|as)\b', Keyword, 'itemtype'), (r'(\))(\s*)(as)', bygroups(Punctuation, Text, Keyword), 'itemtype'), @@ -367,7 +369,8 @@ class XQueryLexer(ExtendedRegexLexer): (r'(for|let|previous|next)(\s+)(\$)', bygroups(Keyword, Text, Name.Variable), 'varname'), (r'(for)(\s+)(tumbling|sliding)(\s+)(window)(\s+)(\$)', - bygroups(Keyword, Text, Keyword, Text, Keyword, Text, Name.Variable), 'varname'), + bygroups(Keyword, Text, Keyword, Text, Keyword, Text, Name.Variable), + 'varname'), # (r'\)|\?|\]', Punctuation, '#push'), (r'\)|\?|\]', Punctuation), (r'(empty)(\s+)(greatest|least)', bygroups(Keyword, Text, Keyword)), @@ -417,17 +420,21 @@ class XQueryLexer(ExtendedRegexLexer): (r'preserve|no-preserve', Keyword), (r',', Punctuation), ], - 'annotationname':[ + 'annotationname': [ (r'\(:', Comment, 'comment'), (qname, Name.Decorator), (r'(\()(' + stringdouble + ')', bygroups(Punctuation, String.Double)), (r'(\()(' + stringsingle + ')', bygroups(Punctuation, String.Single)), - (r'(\,)(\s+)(' + stringdouble + ')', bygroups(Punctuation, Text, String.Double)), - (r'(\,)(\s+)(' + stringsingle + ')', bygroups(Punctuation, Text, String.Single)), + (r'(\,)(\s+)(' + stringdouble + ')', + bygroups(Punctuation, Text, String.Double)), + (r'(\,)(\s+)(' + stringsingle + ')', + bygroups(Punctuation, Text, String.Single)), (r'\)', Punctuation), (r'(\s+)(\%)', bygroups(Text, Name.Decorator), 'annotationname'), - (r'(\s+)(variable)(\s+)(\$)', bygroups(Text, Keyword.Declaration, Text, Name.Variable), 'varname'), - (r'(\s+)(function)(\s+)', bygroups(Text, Keyword.Declaration, Text), 'root') + (r'(\s+)(variable)(\s+)(\$)', + bygroups(Text, Keyword.Declaration, Text, Name.Variable), 'varname'), + (r'(\s+)(function)(\s+)', + bygroups(Text, Keyword.Declaration, Text), 'root') ], 'varname': [ (r'\(:', Comment, 'comment'), @@ -473,8 +480,10 @@ class XQueryLexer(ExtendedRegexLexer): bygroups(Keyword, Text, Keyword), 'singletype'), (r'(treat)(\s+)(as)', bygroups(Keyword, Text, Keyword)), (r'(instance)(\s+)(of)', bygroups(Keyword, Text, Keyword)), - (r'(case)(\s+)(' + stringdouble + ')', bygroups(Keyword, Text, String.Double), 'itemtype'), - (r'(case)(\s+)(' + stringsingle + ')', bygroups(Keyword, Text, String.Single), 'itemtype'), + (r'(case)(\s+)(' + stringdouble + ')', + bygroups(Keyword, Text, String.Double), 'itemtype'), + (r'(case)(\s+)(' + stringsingle + ')', + bygroups(Keyword, Text, String.Single), 'itemtype'), (r'case|as', Keyword, 'itemtype'), (r'(\))(\s*)(as)', bygroups(Operator, Text, Keyword), 'itemtype'), (ncname + r':\*', Keyword.Type, 'operator'), diff --git a/pygments/lexers/whiley.py b/pygments/lexers/whiley.py new file mode 100644 index 00000000..0d0e8ab8 --- /dev/null +++ b/pygments/lexers/whiley.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.whiley + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for the Whiley language. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups, words +from pygments.token import Comment, Keyword, Name, Number, Operator, \ + Punctuation, String, Text + +__all__ = ['WhileyLexer'] + + +class WhileyLexer(RegexLexer): + """ + Lexer for the Whiley programming language. + + .. versionadded:: 2.2 + """ + name = 'Whiley' + filenames = ['*.whiley'] + aliases = ['whiley'] + mimetypes = ['text/x-whiley'] + + # See the language specification: + # http://whiley.org/download/WhileyLanguageSpec.pdf + + tokens = { + 'root': [ + # Whitespace + (r'\s+', Text), + + # Comments + (r'//.*', Comment.Single), + # don't parse empty comment as doc comment + (r'/\*\*/', Comment.Multiline), + (r'(?s)/\*\*.*?\*/', String.Doc), + (r'(?s)/\*.*?\*/', Comment.Multiline), + + # Keywords + (words(( + 'if', 'else', 'while', 'for', 'do', 'return', + 'switch', 'case', 'default', 'break', 'continue', + 'requires', 'ensures', 'where', 'assert', 'assume', + 'all', 'no', 'some', 'in', 'is', 'new', + 'throw', 'try', 'catch', 'debug', 'skip', 'fail', + 'finite', 'total'), suffix=r'\b'), Keyword.Reserved), + (words(( + 'function', 'method', 'public', 'private', 'protected', + 'export', 'native'), suffix=r'\b'), Keyword.Declaration), + # "constant" & "type" are not keywords unless used in declarations + (r'(constant|type)(\s+)([a-zA-Z_]\w*)(\s+)(is)\b', + bygroups(Keyword.Declaration, Text, Name, Text, Keyword.Reserved)), + (r'(true|false|null)\b', Keyword.Constant), + (r'(bool|byte|int|real|any|void)\b', Keyword.Type), + # "from" is not a keyword unless used with import + (r'(import)(\s+)(\*)([^\S\n]+)(from)\b', + bygroups(Keyword.Namespace, Text, Punctuation, Text, Keyword.Namespace)), + (r'(import)(\s+)([a-zA-Z_]\w*)([^\S\n]+)(from)\b', + bygroups(Keyword.Namespace, Text, Name, Text, Keyword.Namespace)), + (r'(package|import)\b', Keyword.Namespace), + + # standard library: https://github.com/Whiley/WhileyLibs/ + (words(( + # types defined in whiley.lang.Int + 'i8', 'i16', 'i32', 'i64', + 'u8', 'u16', 'u32', 'u64', + 'uint', 'nat', + + # whiley.lang.Any + 'toString'), suffix=r'\b'), Name.Builtin), + + # byte literal + (r'[01]+b', Number.Bin), + + # decimal literal + (r'[0-9]+\.[0-9]+', Number.Float), + # match "1." but not ranges like "3..5" + (r'[0-9]+\.(?!\.)', Number.Float), + + # integer literal + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'[0-9]+', Number.Integer), + + # character literal + (r"""'[^\\]'""", String.Char), + (r"""(')(\\['"\\btnfr])(')""", + bygroups(String.Char, String.Escape, String.Char)), + + # string literal + (r'"', String, 'string'), + + # operators and punctuation + (r'[{}()\[\],.;]', Punctuation), + (u'[+\\-*/%&|<>^!~@=:?' + # unicode operators + u'\u2200\u2203\u2205\u2282\u2286\u2283\u2287' + u'\u222A\u2229\u2264\u2265\u2208\u2227\u2228' + u']', Operator), + + # identifier + (r'[a-zA-Z_]\w*', Name), + ], + 'string': [ + (r'"', String, '#pop'), + (r'\\[btnfr]', String.Escape), + (r'\\u[0-9a-fA-F]{4}', String.Escape), + (r'\\.', String), + (r'[^\\"]+', String), + ], + } diff --git a/pygments/lexers/x10.py b/pygments/lexers/x10.py index ea75ab71..1c63326d 100644 --- a/pygments/lexers/x10.py +++ b/pygments/lexers/x10.py @@ -5,7 +5,7 @@ Lexers for the X10 programming language. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/modeline.py b/pygments/modeline.py index 2200f1cf..9f8d5dab 100644 --- a/pygments/modeline.py +++ b/pygments/modeline.py @@ -5,7 +5,7 @@ A simple modeline parser (based on pymodeline). - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -35,9 +35,10 @@ def get_filetype_from_buffer(buf, max_lines=5): ret = get_filetype_from_line(l) if ret: return ret - for l in lines[max_lines:-1:-1]: - ret = get_filetype_from_line(l) - if ret: - return ret + for i in range(max_lines, -1, -1): + if i < len(lines): + ret = get_filetype_from_line(lines[i]) + if ret: + return ret return None diff --git a/pygments/plugin.py b/pygments/plugin.py index f9ea0890..7987d646 100644 --- a/pygments/plugin.py +++ b/pygments/plugin.py @@ -32,43 +32,37 @@ yourfilter = yourfilter:YourFilter - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -try: - import pkg_resources -except ImportError: - pkg_resources = None - LEXER_ENTRY_POINT = 'pygments.lexers' FORMATTER_ENTRY_POINT = 'pygments.formatters' STYLE_ENTRY_POINT = 'pygments.styles' FILTER_ENTRY_POINT = 'pygments.filters' +def iter_entry_points(group_name): + try: + import pkg_resources + except ImportError: + return [] + + return pkg_resources.iter_entry_points(group_name) def find_plugin_lexers(): - if pkg_resources is None: - return - for entrypoint in pkg_resources.iter_entry_points(LEXER_ENTRY_POINT): + for entrypoint in iter_entry_points(LEXER_ENTRY_POINT): yield entrypoint.load() def find_plugin_formatters(): - if pkg_resources is None: - return - for entrypoint in pkg_resources.iter_entry_points(FORMATTER_ENTRY_POINT): + for entrypoint in iter_entry_points(FORMATTER_ENTRY_POINT): yield entrypoint.name, entrypoint.load() def find_plugin_styles(): - if pkg_resources is None: - return - for entrypoint in pkg_resources.iter_entry_points(STYLE_ENTRY_POINT): + for entrypoint in iter_entry_points(STYLE_ENTRY_POINT): yield entrypoint.name, entrypoint.load() def find_plugin_filters(): - if pkg_resources is None: - return - for entrypoint in pkg_resources.iter_entry_points(FILTER_ENTRY_POINT): + for entrypoint in iter_entry_points(FILTER_ENTRY_POINT): yield entrypoint.name, entrypoint.load() diff --git a/pygments/regexopt.py b/pygments/regexopt.py index 79903684..dcfae2fd 100644 --- a/pygments/regexopt.py +++ b/pygments/regexopt.py @@ -6,7 +6,7 @@ An algorithm that generates optimized regexes for matching long lists of literal strings. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -54,7 +54,7 @@ def regex_opt_inner(strings, open_paren): return open_paren + regex_opt_inner(rest, '') + '|' \ + make_charset(oneletter) + close_paren # print '-> only 1-character' - return make_charset(oneletter) + return open_paren + make_charset(oneletter) + close_paren prefix = commonprefix(strings) if prefix: plen = len(prefix) diff --git a/pygments/scanner.py b/pygments/scanner.py index 35dbbadd..3350ac8e 100644 --- a/pygments/scanner.py +++ b/pygments/scanner.py @@ -12,7 +12,7 @@ Have a look at the `DelphiLexer` to get an idea of how to use this scanner. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re @@ -66,7 +66,8 @@ class Scanner(object): def test(self, pattern): """Apply a pattern on the current position and check - if it patches. Doesn't touch pos.""" + if it patches. Doesn't touch pos. + """ return self.check(pattern) is not None def scan(self, pattern): diff --git a/pygments/sphinxext.py b/pygments/sphinxext.py index 2dc9810f..f962f8c6 100644 --- a/pygments/sphinxext.py +++ b/pygments/sphinxext.py @@ -6,7 +6,7 @@ Sphinx extension to generate automatic documentation of lexers, formatters and filters. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -57,6 +57,7 @@ FILTERDOC = ''' ''' + class PygmentsDoc(Directive): """ A directive to collect all lexers/formatters/filters and generate diff --git a/pygments/style.py b/pygments/style.py index b2b990ea..879c4e05 100644 --- a/pygments/style.py +++ b/pygments/style.py @@ -5,13 +5,36 @@ Basic style object. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ from pygments.token import Token, STANDARD_TYPES from pygments.util import add_metaclass +# Default mapping of #ansixxx to RGB colors. +_ansimap = { + # dark + '#ansiblack': '000000', + '#ansidarkred': '7f0000', + '#ansidarkgreen': '007f00', + '#ansibrown': '7f7fe0', + '#ansidarkblue': '00007f', + '#ansipurple': '7f007f', + '#ansiteal': '007f7f', + '#ansilightgray': 'e5e5e5', + # normal + '#ansidarkgray': '555555', + '#ansired': 'ff0000', + '#ansigreen': '00ff00', + '#ansiyellow': 'ffff00', + '#ansiblue': '0000ff', + '#ansifuchsia': 'ff00ff', + '#ansiturquoise': '00ffff', + '#ansiwhite': 'ffffff', +} +ansicolors = set(_ansimap) + class StyleMeta(type): @@ -22,6 +45,8 @@ class StyleMeta(type): obj.styles[token] = '' def colorformat(text): + if text in ansicolors: + return text if text[0:1] == '#': col = text[1:] if len(col) == 6: @@ -79,16 +104,28 @@ class StyleMeta(type): def style_for_token(cls, token): t = cls._styles[token] + ansicolor = bgansicolor = None + color = t[0] + if color.startswith('#ansi'): + ansicolor = color + color = _ansimap[color] + bgcolor = t[4] + if bgcolor.startswith('#ansi'): + bgansicolor = bgcolor + bgcolor = _ansimap[bgcolor] + return { - 'color': t[0] or None, + 'color': color or None, 'bold': bool(t[1]), 'italic': bool(t[2]), 'underline': bool(t[3]), - 'bgcolor': t[4] or None, + 'bgcolor': bgcolor or None, 'border': t[5] or None, 'roman': bool(t[6]) or None, 'sans': bool(t[7]) or None, 'mono': bool(t[8]) or None, + 'ansicolor': ansicolor, + 'bgansicolor': bgansicolor, } def list_styles(cls): diff --git a/pygments/styles/__init__.py b/pygments/styles/__init__.py index d7a0564a..839a9b78 100644 --- a/pygments/styles/__init__.py +++ b/pygments/styles/__init__.py @@ -5,7 +5,7 @@ Contains built-in styles. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -41,6 +41,9 @@ STYLE_MAP = { 'lovelace': 'lovelace::LovelaceStyle', 'algol': 'algol::AlgolStyle', 'algol_nu': 'algol_nu::Algol_NuStyle', + 'arduino': 'arduino::ArduinoStyle', + 'rainbow_dash': 'rainbow_dash::RainbowDashStyle', + 'abap': 'abap::AbapStyle', } diff --git a/pygments/styles/abap.py b/pygments/styles/abap.py new file mode 100644 index 00000000..91286a3a --- /dev/null +++ b/pygments/styles/abap.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +""" + pygments.styles.abap + ~~~~~~~~~~~~~~~~~~~~ + + ABAP workbench like style. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import Keyword, Name, Comment, String, Error, \ + Number, Operator + + +class AbapStyle(Style): + default_style = "" + styles = { + Comment: 'italic #888', + Comment.Special: '#888', + Keyword: '#00f', + Operator.Word: '#00f', + Name: '#000', + Number: '#3af', + String: '#5a2', + + Error: '#F00', + } diff --git a/pygments/styles/algol.py b/pygments/styles/algol.py index a8726009..16461e0b 100644 --- a/pygments/styles/algol.py +++ b/pygments/styles/algol.py @@ -26,7 +26,7 @@ [1] `Revised Report on the Algorithmic Language Algol-60 <http://www.masswerk.at/algol60/report.htm>` - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/algol_nu.py b/pygments/styles/algol_nu.py index 392838f2..366ae215 100644 --- a/pygments/styles/algol_nu.py +++ b/pygments/styles/algol_nu.py @@ -26,7 +26,7 @@ [1] `Revised Report on the Algorithmic Language Algol-60 <http://www.masswerk.at/algol60/report.htm>` - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/arduino.py b/pygments/styles/arduino.py index cb4d17b0..57e3809e 100644 --- a/pygments/styles/arduino.py +++ b/pygments/styles/arduino.py @@ -5,7 +5,7 @@ Arduino® Syntax highlighting style. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -29,7 +29,7 @@ class ArduinoStyle(Style): Comment: "#95a5a6", # class: 'c' Comment.Multiline: "", # class: 'cm' - Comment.Preproc: "#434f54", # class: 'cp' + Comment.Preproc: "#728E00", # class: 'cp' Comment.Single: "", # class: 'c1' Comment.Special: "", # class: 'cs' @@ -38,15 +38,15 @@ class ArduinoStyle(Style): Keyword.Declaration: "", # class: 'kd' Keyword.Namespace: "", # class: 'kn' Keyword.Pseudo: "#00979D", # class: 'kp' - Keyword.Reserved: "", # class: 'kr' + Keyword.Reserved: "#00979D", # class: 'kr' Keyword.Type: "#00979D", # class: 'kt' - Operator: "#434f54", # class: 'o' + Operator: "#728E00", # class: 'o' Operator.Word: "", # class: 'ow' Name: "#434f54", # class: 'n' Name.Attribute: "", # class: 'na' - Name.Builtin: "", # class: 'nb' + Name.Builtin: "#728E00", # class: 'nb' Name.Builtin.Pseudo: "", # class: 'bp' Name.Class: "", # class: 'nc' Name.Constant: "", # class: 'no' @@ -64,7 +64,7 @@ class ArduinoStyle(Style): Name.Variable.Global: "", # class: 'vg' Name.Variable.Instance: "", # class: 'vi' - Number: "#434f54", # class: 'm' + Number: "#8A7B52", # class: 'm' Number.Float: "", # class: 'mf' Number.Hex: "", # class: 'mh' Number.Integer: "", # class: 'mi' diff --git a/pygments/styles/autumn.py b/pygments/styles/autumn.py index 2040659e..71b93b1e 100644 --- a/pygments/styles/autumn.py +++ b/pygments/styles/autumn.py @@ -5,7 +5,7 @@ A colorful style, inspired by the terminal highlighting style. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/borland.py b/pygments/styles/borland.py index 2b1f4ca9..0d13d1aa 100644 --- a/pygments/styles/borland.py +++ b/pygments/styles/borland.py @@ -5,7 +5,7 @@ Style similar to the style used in the Borland IDEs. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/bw.py b/pygments/styles/bw.py index 56d78bd6..f0a6b148 100644 --- a/pygments/styles/bw.py +++ b/pygments/styles/bw.py @@ -5,7 +5,7 @@ Simple black/white only style. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/colorful.py b/pygments/styles/colorful.py index ebedc02f..bfc0b502 100644 --- a/pygments/styles/colorful.py +++ b/pygments/styles/colorful.py @@ -5,7 +5,7 @@ A colorful style, inspired by CodeRay. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/default.py b/pygments/styles/default.py index df99768c..6b9bd446 100644 --- a/pygments/styles/default.py +++ b/pygments/styles/default.py @@ -5,7 +5,7 @@ The default highlighting style. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/emacs.py b/pygments/styles/emacs.py index 27ae19ad..af15f30d 100644 --- a/pygments/styles/emacs.py +++ b/pygments/styles/emacs.py @@ -5,7 +5,7 @@ A highlighting style for Pygments, inspired by Emacs. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/friendly.py b/pygments/styles/friendly.py index d5256a4b..b2d1c0ce 100644 --- a/pygments/styles/friendly.py +++ b/pygments/styles/friendly.py @@ -5,7 +5,7 @@ A modern style based on the VIM pyte theme. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/fruity.py b/pygments/styles/fruity.py index 99bbae6f..1bbe0316 100644 --- a/pygments/styles/fruity.py +++ b/pygments/styles/fruity.py @@ -5,7 +5,7 @@ pygments version of my "fruity" vim theme. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/igor.py b/pygments/styles/igor.py index 8f552709..d4620a42 100644 --- a/pygments/styles/igor.py +++ b/pygments/styles/igor.py @@ -5,7 +5,7 @@ Igor Pro default style. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/lovelace.py b/pygments/styles/lovelace.py index 4009274c..861f778d 100644 --- a/pygments/styles/lovelace.py +++ b/pygments/styles/lovelace.py @@ -9,7 +9,7 @@ A desaturated, somewhat subdued style created for the Lovelace interactive learning environment. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -62,14 +62,18 @@ class LovelaceStyle(Style): Name.Entity: _ESCAPE_LIME, Name.Exception: _EXCEPT_YELLOW, Name.Function: _FUN_BROWN, + Name.Function.Magic: _DOC_ORANGE, Name.Label: _LABEL_CYAN, Name.Namespace: _LABEL_CYAN, Name.Tag: _KW_BLUE, Name.Variable: '#b04040', Name.Variable.Global:_EXCEPT_YELLOW, + Name.Variable.Magic: _DOC_ORANGE, String: _STR_RED, + String.Affix: '#444444', String.Char: _OW_PURPLE, + String.Delimiter: _DOC_ORANGE, String.Doc: 'italic '+_DOC_ORANGE, String.Escape: _ESCAPE_LIME, String.Interpol: 'underline', diff --git a/pygments/styles/manni.py b/pygments/styles/manni.py index dd09f263..f0a325af 100644 --- a/pygments/styles/manni.py +++ b/pygments/styles/manni.py @@ -8,7 +8,7 @@ This is a port of the style used in the `php port`_ of pygments by Manni. The style is called 'default' there. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/monokai.py b/pygments/styles/monokai.py index 9c2a0a87..337e2f89 100644 --- a/pygments/styles/monokai.py +++ b/pygments/styles/monokai.py @@ -7,7 +7,7 @@ http://www.monokai.nl/blog/2006/07/15/textmate-color-theme/ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/murphy.py b/pygments/styles/murphy.py index 1f83cb26..c8270065 100644 --- a/pygments/styles/murphy.py +++ b/pygments/styles/murphy.py @@ -5,7 +5,7 @@ Murphy's style from CodeRay. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/native.py b/pygments/styles/native.py index 33ea3c17..921a58d9 100644 --- a/pygments/styles/native.py +++ b/pygments/styles/native.py @@ -5,7 +5,7 @@ pygments version of my "native" vim theme. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/paraiso_dark.py b/pygments/styles/paraiso_dark.py index f906f87d..5f334bb9 100644 --- a/pygments/styles/paraiso_dark.py +++ b/pygments/styles/paraiso_dark.py @@ -9,7 +9,7 @@ Created with Base16 Builder by Chris Kempson (https://github.com/chriskempson/base16-builder). - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/paraiso_light.py b/pygments/styles/paraiso_light.py index 5424d122..a8112819 100644 --- a/pygments/styles/paraiso_light.py +++ b/pygments/styles/paraiso_light.py @@ -9,7 +9,7 @@ Created with Base16 Builder by Chris Kempson (https://github.com/chriskempson/base16-builder). - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/pastie.py b/pygments/styles/pastie.py index f65940be..d6142908 100644 --- a/pygments/styles/pastie.py +++ b/pygments/styles/pastie.py @@ -7,7 +7,7 @@ .. _pastie: http://pastie.caboo.se/ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/perldoc.py b/pygments/styles/perldoc.py index 47a097ca..24af2df6 100644 --- a/pygments/styles/perldoc.py +++ b/pygments/styles/perldoc.py @@ -7,7 +7,7 @@ .. _perldoc: http://perldoc.perl.org/ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -41,7 +41,7 @@ class PerldocStyle(Style): Operator.Word: '#8B008B', Keyword: '#8B008B bold', - Keyword.Type: '#a7a7a7', + Keyword.Type: '#00688B', Name.Class: '#008b45 bold', Name.Exception: '#008b45 bold', diff --git a/pygments/styles/rainbow_dash.py b/pygments/styles/rainbow_dash.py new file mode 100644 index 00000000..7cf5c9d7 --- /dev/null +++ b/pygments/styles/rainbow_dash.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +""" + pygments.styles.rainbow_dash + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + A bright and colorful syntax highlighting `theme`. + + .. _theme: http://sanssecours.github.io/Rainbow-Dash.tmbundle + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import (Comment, Error, Generic, Name, Number, Operator, + String, Text, Whitespace, Keyword) + +BLUE_LIGHT = '#0080ff' +BLUE = '#2c5dcd' +GREEN = '#00cc66' +GREEN_LIGHT = '#ccffcc' +GREEN_NEON = '#00cc00' +GREY = '#aaaaaa' +GREY_LIGHT = '#cbcbcb' +GREY_DARK = '#4d4d4d' +PURPLE = '#5918bb' +RED = '#cc0000' +RED_DARK = '#c5060b' +RED_LIGHT = '#ffcccc' +RED_BRIGHT = '#ff0000' +WHITE = '#ffffff' +TURQUOISE = '#318495' +ORANGE = '#ff8000' + + +class RainbowDashStyle(Style): + """ + A bright and colorful syntax highlighting theme. + """ + + background_color = WHITE + + styles = { + Comment: 'italic {}'.format(BLUE_LIGHT), + Comment.Preproc: 'noitalic', + Comment.Special: 'bold', + + Error: 'bg:{} {}'.format(RED, WHITE), + + Generic.Deleted: 'border:{} bg:{}'.format(RED_DARK, RED_LIGHT), + Generic.Emph: 'italic', + Generic.Error: RED_BRIGHT, + Generic.Heading: 'bold {}'.format(BLUE), + Generic.Inserted: 'border:{} bg:{}'.format(GREEN_NEON, GREEN_LIGHT), + Generic.Output: GREY, + Generic.Prompt: 'bold {}'.format(BLUE), + Generic.Strong: 'bold', + Generic.Subheading: 'bold {}'.format(BLUE), + Generic.Traceback: RED_DARK, + + Keyword: 'bold {}'.format(BLUE), + Keyword.Pseudo: 'nobold', + Keyword.Type: PURPLE, + + Name.Attribute: 'italic {}'.format(BLUE), + Name.Builtin: 'bold {}'.format(PURPLE), + Name.Class: 'underline', + Name.Constant: TURQUOISE, + Name.Decorator: 'bold {}'.format(ORANGE), + Name.Entity: 'bold {}'.format(PURPLE), + Name.Exception: 'bold {}'.format(PURPLE), + Name.Function: 'bold {}'.format(ORANGE), + Name.Tag: 'bold {}'.format(BLUE), + + Number: 'bold {}'.format(PURPLE), + + Operator: BLUE, + Operator.Word: 'bold', + + String: GREEN, + String.Doc: 'italic', + String.Escape: 'bold {}'.format(RED_DARK), + String.Other: TURQUOISE, + String.Symbol: 'bold {}'.format(RED_DARK), + + Text: GREY_DARK, + + Whitespace: GREY_LIGHT + } diff --git a/pygments/styles/rrt.py b/pygments/styles/rrt.py index 342c9fc6..96f9490c 100644 --- a/pygments/styles/rrt.py +++ b/pygments/styles/rrt.py @@ -5,7 +5,7 @@ pygments "rrt" theme, based on Zap and Emacs defaults. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/sas.py b/pygments/styles/sas.py new file mode 100644 index 00000000..78686fc2 --- /dev/null +++ b/pygments/styles/sas.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +""" + pygments.styles.sas + ~~~~~~~~~~~~~~~~~~~ + + Style inspired by SAS' enhanced program editor. Note This is not + meant to be a complete style. It's merely meant to mimic SAS' + program editor syntax highlighting. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import Keyword, Name, Comment, String, Error, \ + Number, Other, Whitespace, Generic + + +class SasStyle(Style): + """ + Style inspired by SAS' enhanced program editor. Note This is not + meant to be a complete style. It's merely meant to mimic SAS' + program editor syntax highlighting. + """ + + default_style = '' + + styles = { + Whitespace: '#bbbbbb', + Comment: 'italic #008800', + String: '#800080', + Number: 'bold #2e8b57', + Other: 'bg:#ffffe0', + Keyword: '#2c2cff', + Keyword.Reserved: 'bold #353580', + Keyword.Constant: 'bold', + Name.Builtin: '#2c2cff', + Name.Function: 'bold italic', + Name.Variable: 'bold #2c2cff', + Generic: '#2c2cff', + Generic.Emph: '#008800', + Generic.Error: '#d30202', + Error: 'bg:#e3d2d2 #a61717' + } diff --git a/pygments/styles/stata.py b/pygments/styles/stata.py new file mode 100644 index 00000000..2b5f5edd --- /dev/null +++ b/pygments/styles/stata.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +""" + pygments.styles.stata + ~~~~~~~~~~~~~~~~~~~~~ + + Style inspired by Stata's do-file editor. Note this is not meant + to be a complete style. It's merely meant to mimic Stata's do file + editor syntax highlighting. + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.style import Style +from pygments.token import Keyword, Name, Comment, String, Error, \ + Number, Operator, Whitespace + + +class StataStyle(Style): + """ + Style inspired by Stata's do-file editor. Note this is not meant + to be a complete style. It's merely meant to mimic Stata's do file + editor syntax highlighting. + """ + + default_style = '' + + styles = { + Whitespace: '#bbbbbb', + Comment: 'italic #008800', + String: '#7a2424', + Number: '#2c2cff', + Operator: '', + Keyword: 'bold #353580', + Keyword.Constant: '', + Name.Function: '#2c2cff', + Name.Variable: 'bold #35baba', + Name.Variable.Global: 'bold #b5565e', + Error: 'bg:#e3d2d2 #a61717' + } diff --git a/pygments/styles/tango.py b/pygments/styles/tango.py index c65850bd..2abc8c61 100644 --- a/pygments/styles/tango.py +++ b/pygments/styles/tango.py @@ -33,7 +33,7 @@ have been chosen to have the same style. Similarly, keywords (Keyword.*), and Operator.Word (and, or, in) have been assigned the same style. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/trac.py b/pygments/styles/trac.py index bf36ce03..aff39fd4 100644 --- a/pygments/styles/trac.py +++ b/pygments/styles/trac.py @@ -5,7 +5,7 @@ Port of the default trac highlighter design. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/vim.py b/pygments/styles/vim.py index 383fd8f0..888088b1 100644 --- a/pygments/styles/vim.py +++ b/pygments/styles/vim.py @@ -5,7 +5,7 @@ A highlighting style for Pygments, inspired by vim. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/vs.py b/pygments/styles/vs.py index 78efc547..bc3ed2b5 100644 --- a/pygments/styles/vs.py +++ b/pygments/styles/vs.py @@ -5,7 +5,7 @@ Simple style with MS Visual Studio colors. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/xcode.py b/pygments/styles/xcode.py index 3dc9240d..64bfcf03 100644 --- a/pygments/styles/xcode.py +++ b/pygments/styles/xcode.py @@ -5,7 +5,7 @@ Style similar to the `Xcode` default theme. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/token.py b/pygments/token.py index fa3b1e11..43f73c85 100644 --- a/pygments/token.py +++ b/pygments/token.py @@ -5,10 +5,11 @@ Basic token types and the standard tokens. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ + class _TokenType(tuple): parent = None @@ -52,30 +53,30 @@ class _TokenType(tuple): return self -Token = _TokenType() +Token = _TokenType() # Special token types -Text = Token.Text -Whitespace = Text.Whitespace -Escape = Token.Escape -Error = Token.Error +Text = Token.Text +Whitespace = Text.Whitespace +Escape = Token.Escape +Error = Token.Error # Text that doesn't belong to this lexer (e.g. HTML in PHP) -Other = Token.Other +Other = Token.Other # Common token types for source code -Keyword = Token.Keyword -Name = Token.Name -Literal = Token.Literal -String = Literal.String -Number = Literal.Number +Keyword = Token.Keyword +Name = Token.Name +Literal = Token.Literal +String = Literal.String +Number = Literal.Number Punctuation = Token.Punctuation -Operator = Token.Operator -Comment = Token.Comment +Operator = Token.Operator +Comment = Token.Comment # Generic types for non-source code -Generic = Token.Generic +Generic = Token.Generic -# String and some others are not direct childs of Token. +# String and some others are not direct children of Token. # alias them: Token.Token = Token Token.String = String @@ -147,6 +148,7 @@ STANDARD_TYPES = { Name.Entity: 'ni', Name.Exception: 'ne', Name.Function: 'nf', + Name.Function.Magic: 'fm', Name.Property: 'py', Name.Label: 'nl', Name.Namespace: 'nn', @@ -156,13 +158,16 @@ STANDARD_TYPES = { Name.Variable.Class: 'vc', Name.Variable.Global: 'vg', Name.Variable.Instance: 'vi', + Name.Variable.Magic: 'vm', Literal: 'l', Literal.Date: 'ld', String: 's', + String.Affix: 'sa', String.Backtick: 'sb', String.Char: 'sc', + String.Delimiter: 'dl', String.Doc: 'sd', String.Double: 's2', String.Escape: 'se', diff --git a/pygments/unistring.py b/pygments/unistring.py index 49a2819a..6096d110 100644 --- a/pygments/unistring.py +++ b/pygments/unistring.py @@ -8,7 +8,7 @@ Inspired by chartypes_create.py from the MoinMoin project. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/util.py b/pygments/util.py index 07b662d0..45070063 100644 --- a/pygments/util.py +++ b/pygments/util.py @@ -5,7 +5,7 @@ Utility functions. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -14,7 +14,7 @@ import sys split_path_re = re.compile(r'[/\\ ]') -doctype_lookup_re = re.compile(r'''(?smx) +doctype_lookup_re = re.compile(r''' (<\?.*?\?>)?\s* <!DOCTYPE\s+( [a-zA-Z_][a-zA-Z0-9]* @@ -23,8 +23,9 @@ doctype_lookup_re = re.compile(r'''(?smx) "[^"]*")? ) [^>]*> -''') -tag_re = re.compile(r'<(.+?)(\s.*?)?>.*?</.+?>(?uism)') +''', re.DOTALL | re.MULTILINE | re.VERBOSE) +tag_re = re.compile(r'<(.+?)(\s.*?)?>.*?</.+?>', + re.UNICODE | re.IGNORECASE | re.DOTALL | re.MULTILINE) xml_decl_re = re.compile(r'\s*<\?xml[^>]*\?>', re.I) diff --git a/scripts/check_sources.py b/scripts/check_sources.py index 4f5926f6..db09de42 100755 --- a/scripts/check_sources.py +++ b/scripts/check_sources.py @@ -7,7 +7,7 @@ Make sure each Python file has a correct file header including copyright and license information. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -36,7 +36,7 @@ def checker(*suffixes, **kwds): name_mail_re = r'[\w ]+(<.*?>)?' -copyright_re = re.compile(r'^ :copyright: Copyright 2006-2015 by ' +copyright_re = re.compile(r'^ :copyright: Copyright 2006-2017 by ' r'the Pygments team, see AUTHORS\.$', re.UNICODE) copyright_2_re = re.compile(r'^ %s(, %s)*[,.]$' % (name_mail_re, name_mail_re), re.UNICODE) diff --git a/scripts/debug_lexer.py b/scripts/debug_lexer.py index cedd0988..02bb9fef 100755 --- a/scripts/debug_lexer.py +++ b/scripts/debug_lexer.py @@ -8,7 +8,7 @@ the text where Error tokens are being generated, along with some context. - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -109,6 +109,8 @@ def main(fn, lexer=None, options={}): lxcls = find_lexer_class(name) if lxcls is None: raise AssertionError('no lexer found for file %r' % fn) + print('Using lexer: %s (%s.%s)' % (lxcls.name, lxcls.__module__, + lxcls.__name__)) debug_lexer = False # if profile: # # does not work for e.g. ExtendedRegexLexers diff --git a/scripts/get_vimkw.py b/scripts/get_vimkw.py index 45652740..688a0c64 100644 --- a/scripts/get_vimkw.py +++ b/scripts/get_vimkw.py @@ -16,7 +16,7 @@ HEADER = '''\ This file is autogenerated by scripts/get_vimkw.py - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -3,8 +3,8 @@ tag_build = dev tag_date = true [aliases] -release = egg_info -RDb '' +release = egg_info -Db '' upload = upload --sign --identity=36580288 [bdist_wheel] -universal = 1
\ No newline at end of file +universal = 1 @@ -16,7 +16,7 @@ formats that PIL supports and ANSI sequences * it is usable as a command-line tool and as a library - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -24,22 +24,16 @@ try: from setuptools import setup, find_packages have_setuptools = True except ImportError: - try: - import ez_setup - ez_setup.use_setuptools() - from setuptools import setup, find_packages - have_setuptools = True - except ImportError: - from distutils.core import setup - def find_packages(*args, **kwargs): - return [ - 'pygments', - 'pygments.lexers', - 'pygments.formatters', - 'pygments.styles', - 'pygments.filters', - ] - have_setuptools = False + from distutils.core import setup + def find_packages(*args, **kwargs): + return [ + 'pygments', + 'pygments.lexers', + 'pygments.formatters', + 'pygments.styles', + 'pygments.filters', + ] + have_setuptools = False if have_setuptools: add_keywords = dict( @@ -54,7 +48,7 @@ else: setup( name = 'Pygments', - version = '2.1.3', + version = '2.2.0', url = 'http://pygments.org/', license = 'BSD License', author = 'Georg Brandl', @@ -62,7 +56,7 @@ setup( description = 'Pygments is a syntax highlighting package written in Python.', long_description = __doc__, keywords = 'syntax highlighting', - packages = find_packages(exclude=['ez_setup']), + packages = find_packages(), platforms = 'any', zip_safe = False, include_package_data = True, diff --git a/tests/examplefiles/StdGeneric.icl b/tests/examplefiles/StdGeneric.icl new file mode 100644 index 00000000..2e6c3931 --- /dev/null +++ b/tests/examplefiles/StdGeneric.icl @@ -0,0 +1,92 @@ +implementation module StdGeneric + +import StdInt, StdMisc, StdClass, StdFunc + +generic bimap a b :: Bimap .a .b + +bimapId :: Bimap .a .a +bimapId = { map_to = id, map_from = id } + +bimap{|c|} = { map_to = id, map_from = id } + +bimap{|PAIR|} bx by = { map_to= map_to, map_from=map_from } +where + map_to (PAIR x y) = PAIR (bx.map_to x) (by.map_to y) + map_from (PAIR x y) = PAIR (bx.map_from x) (by.map_from y) +bimap{|EITHER|} bl br = { map_to= map_to, map_from=map_from } +where + map_to (LEFT x) = LEFT (bl.map_to x) + map_to (RIGHT x) = RIGHT (br.map_to x) + map_from (LEFT x) = LEFT (bl.map_from x) + map_from (RIGHT x) = RIGHT (br.map_from x) + +bimap{|(->)|} barg bres = { map_to = map_to, map_from = map_from } +where + map_to f = comp3 bres.map_to f barg.map_from + map_from f = comp3 bres.map_from f barg.map_to + +bimap{|CONS|} barg = { map_to= map_to, map_from=map_from } +where + map_to (CONS x) = CONS (barg.map_to x) + map_from (CONS x) = CONS (barg.map_from x) + +bimap{|FIELD|} barg = { map_to= map_to, map_from=map_from } +where + map_to (FIELD x) = FIELD (barg.map_to x) + map_from (FIELD x) = FIELD (barg.map_from x) + +bimap{|OBJECT|} barg = { map_to= map_to, map_from=map_from } +where + map_to (OBJECT x) = OBJECT (barg.map_to x) + map_from (OBJECT x) = OBJECT (barg.map_from x) + +bimap{|Bimap|} x y = {map_to = map_to, map_from = map_from} +where + map_to {map_to, map_from} = + { map_to = comp3 y.map_to map_to x.map_from + , map_from = comp3 x.map_to map_from y.map_from + } + map_from {map_to, map_from} = + { map_to = comp3 y.map_from map_to x.map_to + , map_from = comp3 x.map_from map_from y.map_to + } + +comp3 :: !(.a -> .b) u:(.c -> .a) !(.d -> .c) -> u:(.d -> .b) +comp3 f g h + | is_id f + | is_id h + = cast g + = cast (\x -> g (h x)) + | is_id h + = cast (\x -> f (g x)) + = \x -> f (g (h x)) +where + is_id :: !.(.a -> .b) -> Bool + is_id f = code inline + { + eq_desc e_StdFunc_did 0 0 + pop_a 1 + } + + cast :: !u:a -> u:b + cast f = code inline + { + pop_a 0 + } + +getConsPath :: !GenericConsDescriptor -> [ConsPos] +getConsPath {gcd_index, gcd_type_def={gtd_num_conses}} + = doit gcd_index gtd_num_conses +where + doit i n + | n == 0 + = abort "getConsPath: zero conses\n" + | i >= n + = abort "getConsPath: cons index >= number of conses" + | n == 1 + = [] + | i < (n/2) + = [ ConsLeft : doit i (n/2) ] + | otherwise + = [ ConsRight : doit (i - (n/2)) (n - (n/2)) ] +
\ No newline at end of file diff --git a/tests/examplefiles/capdl_example.cdl b/tests/examplefiles/capdl_example.cdl new file mode 100644 index 00000000..050e56a6 --- /dev/null +++ b/tests/examplefiles/capdl_example.cdl @@ -0,0 +1,64 @@ +#ifdef ARCH_ARM +arch arm11 +#else +arch ia32 +#endif + +objects { + my_ep = ep /* A synchronous endpoint */ + + /* Two thread control blocks */ + tcb1 = tcb + tcb2 = tcb + + /* Four frames of physical memory */ + frame1 = frame (4k) + frame2 = frame (4k) + frame3 = frame (4k) + frame4 = frame (4k) + + /* Two page tables */ + pt1 = pt + pt2 = pt + + /* Two page directories */ + pd1 = pd + pd2 = pd + + /* Two capability nodes */ + cnode1 = cnode (2 bits) + cnode2 = cnode (3 bits) +} +caps { + cnode1 { + 0x1: frame1 (RW) /* read/write */ + 0x2: my_ep (R) /* read-only */ + } + cnode2 { + 0x1: my_ep (W) /* write-only */ + } + tcb1 { + vspace: pd1 + ipc_buffer_slot: frame1 + cspace: cnode1 + } + pd1 { + 0x10: pt1 + } + pt1 { + 0x8: frame1 (RW) + 0x9: frame2 (R) + } + tcb2 { + vspace: pd2 + ipc_buffer_slot: frame3 + cspace: cnode2 + } + pd2 { + 0x10: pt2 + } + pt2 { + 0x10: frame3 (RW) + 0x12: frame4 (R) + } +} diff --git a/tests/examplefiles/demo.frt b/tests/examplefiles/demo.frt new file mode 100644 index 00000000..1b09ebb0 --- /dev/null +++ b/tests/examplefiles/demo.frt @@ -0,0 +1,3 @@ +2 3 + CR . +: F ( blah ) DUP DROP 1 + ; +1 F CR . diff --git a/tests/examplefiles/demo.hbs b/tests/examplefiles/demo.hbs index 1b9ed5a7..ae80cc1b 100644 --- a/tests/examplefiles/demo.hbs +++ b/tests/examplefiles/demo.hbs @@ -10,3 +10,25 @@ {{else}} <button {{action expand}}>Show More...</button> {{/if}} + +{{> myPartial}} +{{> myPartial var="value" }} +{{> myPartial var=../value}} +{{> (myPartial)}} +{{> (myPartial) var="value"}} +{{> (lookup . "myPartial")}} +{{> ( lookup . "myPartial" ) var="value" }} +{{> (lookup ../foo "myPartial") var="value" }} +{{> @partial-block}} + +{{#>myPartial}} +... +{{/myPartial}} + +{{#*inline "myPartial"}} +... +{{/inline}} + +{{../name}} +{{./name}} +{{this/name}} diff --git a/tests/examplefiles/durexmania.aheui b/tests/examplefiles/durexmania.aheui new file mode 100644 index 00000000..89654c00 --- /dev/null +++ b/tests/examplefiles/durexmania.aheui @@ -0,0 +1,4 @@ +우주메이저☆듀렉스전도사♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♂♀♡먊 +삶은밥과야근밥샤주세양♡밥사밥사밥사밥사밥사땅땅땅빵☆따밦내발따밦다빵맣밥밥밥내놔밥줘밥밥밥밗땅땅땅박밝땅땅딻타밟타맣밦밣따박타맣밦밣따박타맣밦밣따박타맣박빵빵빵빵따따따따맣삶몲 +Original Source by @harunene // Run it on AheuiChem(http://yoo2001818.github.io/AheuiChem/) +https://gist.github.com/item4/ca870a63b390da6cc6f1 diff --git a/tests/examplefiles/example.bat b/tests/examplefiles/example.bat index 596f65de..2b45d2bc 100644 --- a/tests/examplefiles/example.bat +++ b/tests/examplefiles/example.bat @@ -205,5 +205,7 @@ for /f "tokens=2 delims==" %%G in ( 'assoc %+;/p extension'),%' ) &>nul ver
if errorlevel 0 if not errorlevel 1 set /a _passed+=1
goto :eof
+FOR /F %%a IN ('%%c%%') DO %%a
+rem %x% %x% %x% %x% %x% %x% %x% %x% %x% %x% %x% %x% %x% %x% %x% %x%
:/?
goto :fail
diff --git a/tests/examplefiles/example.hs b/tests/examplefiles/example.hs index f5e2b555..764cab77 100644 --- a/tests/examplefiles/example.hs +++ b/tests/examplefiles/example.hs @@ -29,3 +29,13 @@ data ĈrazyThings = -- some char literals: charl = ['"', 'a', '\ESC', '\'', ' '] + +-- closed type families +type family Fam (a :: Type) = r :: Type where + Fam Int = True + Fam a = False + +-- type literals +type IntChar = '[Int, Char] +type Falsy = 'False +type Falsy = '(10, 20, 30) diff --git a/tests/examplefiles/example.jsgf b/tests/examplefiles/example.jsgf new file mode 100644 index 00000000..cd79dbed --- /dev/null +++ b/tests/examplefiles/example.jsgf @@ -0,0 +1,28 @@ +#JSGF V1.0 UTF-8 en;
+
+grammar org.pygments.example;
+
+// comment /*
+/*
+ multi-line
+ comment
+ /* without nesting
+ @example doesn't mean anything here.
+*/
+/**/
+
+/**
+ * A silly @example grammar.
+ *
+ * @author David Corbett
+ * @version 1
+* @see <org.pygments.example.en>
+ * @example hello world
+ @example hello ","
+ *
+ **@blah
+ **world
+ */
+public <sentence> = (/1/<en> | / 0.8f /<fr> |/0/<VOID>""""{}{})*<NULL>;
+<org.pygments.example.fr> = bonjour {hello} [,] "le monde" {world};
+<en> = (/56/hello|/3.14e3/"\"hello\"") {hello} {{ {\\//\} } world {world} !+ ;
diff --git a/tests/examplefiles/example.juttle b/tests/examplefiles/example.juttle new file mode 100644 index 00000000..ae861996 --- /dev/null +++ b/tests/examplefiles/example.juttle @@ -0,0 +1,110 @@ +/* Block comment */ +/* + Multiline block + comment +*/ + +// inline comment +function juttleFunction(arg) { + if (arg == null) { + return null; + } + else if (arg == 0) { + return 'zero'; + } + else if (arg == 1) { + return "one"; + } + else { + return 1.1; + } +} + +reducer juttleReducer(field) { + var x = 0; + function update() { + x = *field; + } + + function result() { + return x; + } +} + +sub myemit(limit) { + emit -limit limit +} + +input test: text -default 'input'; +const object = { + xyz: 123, + name: 'something' +}; + +const array = [ + :2016-01-01:, + :2016-01-01T01:00:00:, + :2016-01-01T01:00:00.000:, + :2016-01-01T01:00:00.000Z:, + :2016-01-01T01:00:00.000-0800:, + :2016-01-01T01:00:00.000-08:00:, + :00:00:01:, + :00:00:00.001:, + :now:, + :beginning:, + :end:, + :forever:, + :yesterday:, + :today:, + :tomorrow:, + :1:, + :1.1:, + :1s:, + :1 second:, + :1 seconds:, + :100ms:, + :100 millisecond:, + :100 milliseconds:, + :1d:, + :1 day:, + :1 days:, + :.2h:, + :1.2h:, + :.2 hour:, + :1.2 hours:, + :.5d:, + :1.5d:, + :.5 day:, + :1.5 days:, + :5m:, + :5 minutes:, + :10w:, + :10 weeks:, + :10M:, + :10 months:, + :100y:, + :100 years:, + :1 year and 2 months and 2 days: +]; + +emit + | batch :10 minutes: + | filter x=true + | head 1 + | join + | keep x + | pace -every :1 minute: + | pass + | put y=false + | remove z + | sequence + | skip 1 + | sort field -desc + | split field + | tail 10 + | unbatch + | uniq field +; + +read adapter -last :day: 'search' AND field~/pattern/ OR field == 'string' + | write adapter diff --git a/tests/examplefiles/example.lua b/tests/examplefiles/example.lua index 0289e58c..8ecd6a13 100644 --- a/tests/examplefiles/example.lua +++ b/tests/examplefiles/example.lua @@ -247,4 +247,28 @@ function AucAdvanced.Debug.Assert(test, message) return DebugLib.Assert(addonName, test, message) end +--[==[ +Here follow further tests of Lua syntax. +]]==] +---[[ +local t = { + [ [[ +x +]==] \]]]=1|2; a={b={c={}}}, + 1, 1., 1.2, .2, 1e3, 1.e3, 1.2e3, .2e3, 1.2e+3, 1.2E-3; + 0xA, 0Xa, 0xA., 0x.F, 0xA.F, 0xA.Fp1, 0xA.FP+1, 0Xa.fp-1; +} +function t.f() + goto eof + os.exit() + :: eof :: +end + +function t . a --[==[x]==] .b --[==[y]==] -- +-- () end + . c : d (file) + return '.\a.\b.\f.\n.\r.\t.\v.\\.\".\'.\ +.\z + .\0.\00.\000.\0000.\xFa.\u{1}.\u{1234}' +end diff --git a/tests/examplefiles/example.md b/tests/examplefiles/example.md new file mode 100644 index 00000000..2befb107 --- /dev/null +++ b/tests/examplefiles/example.md @@ -0,0 +1,61 @@ +# this is a header + +## this is a 2nd level header + +* list item 1 + * list item 1.1 +* list item 2 +- list item 3 + +1. numbered list item 1 +1. numbered list item 2 + +- [ ] todo +- [x] done +- [X] done + +The following is italic: *italic* +The following is italic: _italic_ + +The following is not italic: \*italic\* +The following is not italic: \_italic\_ + +The following is not italic: snake*case*word +The following is not italic: snake_case_word + +The following is bold: **bold** **two or more words** +The following is bold: __bold__ __two or more words__ + +The following is not bold: snake**case**word +The following is not bold: snake__case__word + +The following is strikethrough: ~~bold~~ +The following is not strikethrough: snake~~case~~word + +The following is bold with italics inside: **the next _word_ should have been italics** + +> this is a quote + +> this is a multiline +> quote string thing + +this sentence `has monospace` in it + +this sentence @tweets a person about a #topic. + +[google](https://google.com/some/path.html) + + +``` + * this is just unformated + __text__ +``` + +some other text + +```python +from pygments import token +# comment +``` + +some more text diff --git a/tests/examplefiles/example.ng2 b/tests/examplefiles/example.ng2 new file mode 100644 index 00000000..0f424aca --- /dev/null +++ b/tests/examplefiles/example.ng2 @@ -0,0 +1,11 @@ +<div>
+ <p>{{order.DueTime | date:'d. MMMM yyyy HH:mm'}}</p>
+ <p>Status: {{order.OrderState}}</p>
+ <button (click)="deleteOrder()" *ngIf="cancelable" [value]="test" [(twoWayTest)]="foo.bar">Remove</button>
+ <ul>
+ <li *ngFor="#meal of order.Positions">
+ {{meal.Name}}
+ </li>
+ </ul>
+ <p>Preis: <b>{{order.TotalPrice | currency:'EUR':true:'1.2-2'}}</b></p>
+</div>
\ No newline at end of file diff --git a/tests/examplefiles/example.praat b/tests/examplefiles/example.praat index bf2d005f..85573919 100644 --- a/tests/examplefiles/example.praat +++ b/tests/examplefiles/example.praat @@ -2,17 +2,27 @@ form Highlighter test sentence Blank sentence My_sentence This should all be a string text My_text This should also all be a string - word My_word Only the first word is a string, the rest is invalid + word My_word Only the first word is a string, the rest is discarded boolean Binary 1 boolean Text no boolean Quoted "yes" comment This should be a string + optionmenu Choice: 1 + option Foo + option Bar + option 100 real left_Range -123.6 positive right_Range_max 3.3 integer Int 4 natural Nat 4 endform +# Periods do not establish boundaries for keywords +form.var = 10 +# Or operators +not.an.operator$ = "Bad variable name" +bad.or.not = 1 + # External scripts include /path/to/file runScript: "/path/to/file" @@ -51,12 +61,16 @@ endif string$ = "Strings can be 'interpolated'" string$ = "But don't interpolate everything!" +string$(10) + +repeat + string$ = string$ - right$(string$) +until !length(string$) Text... 1 Right 0.2 Half many----hyphens Text... 1 Right -0.4 Bottom aحبيبa Text... 1 Right -0.6 Bottom 日本 Draw circle (mm)... 0.5 0.5 i -x=1 rows = Object_'table'.nrow value$ = Table_'table'$[25, "f0"] @@ -83,17 +97,19 @@ var = if macintosh = 1 then 0 else 1 fi ; This is an inline comment n = numberOfSelected("Sound") for i from newStyle.local to n name = selected$(extractWord$(selected$(), " ")) - sound'i' = selected("Sound", i) + sound'i' = selected("Sound", i+(a*b)) sound[i] = sound'i' endfor -for i from 1 to n +i = 1 +while i < n + i++ # Different styles of object selection select sound'i' sound = selected() sound$ = selected$("Sound") select Sound 'sound$' - selectObject(sound[i]) + selectObject( sound[i]) selectObject: sound # Pause commands @@ -124,14 +140,16 @@ for i from 1 to n # Multi-line command with modifier pitch = noprogress To Pitch (ac): 0, 75, 15, "no", ...0.03, 0.45, 0.01, 0.35, 0.14, 600 + # Formulas are strings + Formula: "if col = 1 then row * Object_'pitch'.dx + 'first' else self fi" # do-style command with assignment minimum = do("Get minimum...", 0, 0, "Hertz", "Parabolic") # New-style multi-line command call with broken strings table = Create Table with column names: "table", 0, - ..."file subject speaker - ...f0 f1 f2 f3 " + + ..."file subject speaker + ... f0 f1 f2 f" + string$(3) + " " + ..."duration response" # Function call with trailing space @@ -156,7 +174,7 @@ for i from 1 to n demoWaitForInput ( ) demo Erase all demo Text: 50, "centre", 50, "half", "Finished" -endfor +endwhile switch$ = if switch == 1 then "a" else ... if switch == 2 then "b" else @@ -207,6 +225,11 @@ assert a != b && c assert a <> b || c assert a < b | c assert a > b + +assert (a)or (b) +assert (a) or(b) +assert (a)and(b) + assert "hello" = "he" + "llo" assert "hello" == "hello world" - " world" @@ -243,3 +266,15 @@ endproc asserterror Unknown symbol:'newline$'« _ assert '_new_style.local' +@proc: a, selected("string"), b +# Comment + +for i to saveSelection.n + selectObject: saveSelection.id[i] + appendInfoLine: selected$() +endfor + +@ok(if selected$("Sound") = "tone" then 1 else 0 fi, + ... "selected sound is tone") + +@ok_formula("selected$(""Sound"") = ""tone""", "selected sound is tone") diff --git a/tests/examplefiles/example.sbl b/tests/examplefiles/example.sbl new file mode 100644 index 00000000..94efada5 --- /dev/null +++ b/tests/examplefiles/example.sbl @@ -0,0 +1,109 @@ +/* Stemmer for Esperanto in UTF-8 */
+
+strings ()
+
+integers ()
+
+booleans ( foreign )
+
+routines (
+ apostrophe
+ canonical_form
+ correlative
+ interjection
+ short_word
+ standard_suffix
+ unuj
+)
+
+externals ( stem )
+
+groupings ( vowel aiou ao ou )
+
+stringdef a' decimal '225'
+stringdef e' hex 'E9'
+stringdef i' hex 'ED'
+stringdef o' hex ' f3'
+stringdef u' hex 'fa '
+
+stringdef cx hex '0109'
+stringdef gx hex '011D'
+stringdef hx hex '0125'
+stringdef jx hex '0135'
+stringdef sx hex '015D'
+stringdef ux hex '016D'
+
+define canonical_form as repeat (
+ [substring]
+ among (
+stringescapes //
+ '/a'/' (<- 'a' set foreign)
+ '/e'/' (<- 'e' set foreign)
+ '/i'/' (<- 'i' set foreign)
+ '/o'/' (<- 'o' set foreign)
+ '/u'/' (<- 'u' set foreign)
+stringescapes `'
+ 'cx' (<- '`cx'')
+ 'gx' (<- '`gx'')
+ 'hx' (<- '`hx'')
+ 'jx' (<- '`jx'')
+ 'sx' (<- '`sx'')
+ 'ux' (<- '`ux'')
+ '' (next)
+ )
+)
+
+backwardmode (
+ stringescapes { }
+
+ define apostrophe as (
+ (['un{'}'] atlimit <- 'unu') or
+ (['l{'}'] atlimit <- 'la') or
+ (['{'}'] <- 'o')
+ )
+
+ define vowel 'aeiou'
+ define aiou vowel - 'e'
+ define ao 'ao'
+ define ou 'ou'
+
+ define short_word as not (loop (maxint * 0 + 4 / 2) gopast vowel)
+
+ define interjection as (
+ among ('adia{ux}' 'aha' 'amen' 'hola' 'hura' 'mia{ux}' 'muu' 'oho')
+ atlimit
+ )
+
+ define correlative as (
+ []
+ // Ignore -al, -am, etc. since they can't be confused with suffixes.
+ test (
+ ('a' or (try 'n'] 'e') or (try 'n' try 'j'] ou))
+ 'i'
+ try ('k' or 't' or '{cx}' or 'nen')
+ atlimit
+ )
+ delete
+ )
+
+ define unuj as (
+ [try 'n' 'j'] 'unu' atlimit delete
+ )
+
+ define standard_suffix as (
+ [
+ try ((try 'n' try 'j' ao) or (try 's' aiou) or (try 'n' 'e'))
+ try '-' try 'a{ux}'
+ ] delete
+ )
+)
+
+define stem as (
+ do canonical_form
+ not foreign
+ backwards (
+ do apostrophe
+ short_word or interjection or
+ correlative or unuj or do standard_suffix
+ )
+)
diff --git a/tests/examplefiles/example.tasm b/tests/examplefiles/example.tasm new file mode 100644 index 00000000..d7202ffb --- /dev/null +++ b/tests/examplefiles/example.tasm @@ -0,0 +1,527 @@ +;----------------------------------------------------------------------------; +; Does A* pathfinding for rockraiders and vehicles +; +; Copyright 2015 Ruben De Smet +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are +; met: +; +; (1) Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; +; (2) Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; +; (3) The name of the author may not be used to +; endorse or promote products derived from this software without +; specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +; IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, +; INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +; HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +; STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +; IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +; POSSIBILITY OF SUCH DAMAGE. +; +;----------------------------------------------------------------------------; + +IDEAL +P386 +MODEL FLAT, C +ASSUME cs:_TEXT,ds:FLAT,es:FLAT,fs:FLAT,gs:FLAT + +INCLUDE "ASTAR.INC" +INCLUDE "READLVL.INC" +INCLUDE "DEBUG.INC" + +STRUC TPriorityField + heuristic dd ? + distance dd ? + x db ? + y db ? + fromx db ? + fromy db ? +ENDS + +STRUC TField + distance dd ? + x db ? + y db ? +ENDS + +CODESEG + +PROC getPath + USES ecx + ARG @@tgtx:dword, \ + @@tgty:dword \ + RETURNS eax, ebx ; eax contains x, ebx contains y + + call getLevelWidth + imul eax, [@@tgty] + add eax, [@@tgtx] + imul eax, SIZE TField + add eax, offset backtraceGraph + mov ecx, eax + + xor eax, eax + xor ebx, ebx + + mov al, [(TField ptr ecx).x] + mov bl, [(TField ptr ecx).y] + + ret +ENDP getPath + +PROC findPath + ; eax will contain a 1 when a path has been found + ; 0 otherwise. + ARG @@srcx:dword, \ + @@srcy:dword, \ + @@tgtx:dword, \ + @@tgty:dword, \ + @@type:dword \ + RETURNS eax + + ; Check whether the target field is "allowed" for + ; the selected vehicle or rock raider + call getField, [@@tgtx], [@@tgty] + mov al, [byte ptr eax] + and eax, 0FFh + + add eax, offset actionTable + mov eax, [eax] + and eax, [@@type] ; TODO: for now, rock raider is hard coded + jnz @canGoToTarget + + mov eax, 0 + ret +@canGoToTarget: + + call cleanData + mov eax, [@@type] + mov [currentType], eax + + mov eax, [@@srcx] + mov [currentOpen.x], al + mov eax, [@@srcy] + mov [currentOpen.y], al + + call distance, [@@srcx], [@@srcy], [@@tgtx], [@@tgty] + ; eax <- distance + call addOpen, [@@srcx], [@@srcy], eax, 0 + +@openListNotEmpty: + call popOpen + cmp eax, 0 + je @openListEmpty + + call addToMap + + call addClosed + + mov eax, [@@tgtx] + cmp [currentOpen.x], al + jne @nextOpen + mov eax, [@@tgty] + cmp [currentOpen.y], al + jne @nextOpen + + jmp @routeFound + + @nextOpen: + call addNeighbours, [@@tgtx], [@@tgty] + + jmp @openListNotEmpty + +@openListEmpty: + mov eax, 0 + ret + +@routeFound: + mov eax, 1 + ret +ENDP findPath + +PROC addToMap + USES eax, ecx + + call getLevelWidth + xor ecx, ecx + mov cl, [currentOpen.y] + imul eax, ecx + mov cl, [currentOpen.x] + add eax, ecx + imul eax, SIZE TField + add eax, offset backtraceGraph + + mov ecx, [currentOpen.distance] + cmp [(TField ptr eax).distance], ecx + jbe @dontAdd + + mov [(TField ptr eax).distance], ecx + mov cl, [currentOpen.fromx] + mov [(TField ptr eax).x], cl + mov cl, [currentOpen.fromy] + mov [(TField ptr eax).y], cl + +@dontAdd: + ret +ENDP addToMap + +; Is closed checks whether the field considered is "closed" for being added to the open list. +; So, it also checks whether we can go on the selected field. +PROC isClosed + USES ebx, ecx, edx + ARG @@x:dword, \ + @@y:dword RETURNS eax + + ; Check bounds first: + + call getLevelWidth + cmp [@@x], eax + ja notWithinBounds ; ja considers -1 > 10 + + call getLevelHeight + cmp [@@y], eax + ja notWithinBounds + + ; Check whether this field is "allowed" for + ; the selected vehicle or rock raider + call getField, [@@x], [@@y] + mov al, [byte ptr eax] + and eax, 0FFh + + add eax, offset actionTable + mov eax, [eax] + and eax, [currentType] ; TODO: for now, rock raider is hard coded + jnz @canGoHere + + + inc eax ; mov eax, 1 + ret + +@canGoHere: + + ; Getting here means the field is okay to walk/fly/whatever on + + xor ecx, ecx + mov cx, [closedlistSize] + cmp cx, 0 ; If empty, return 0 + jne @closedNotEmpty + + mov eax, 0 + ret + +@closedNotEmpty: + mov ebx, offset closedlist + +@loopClosed: + mov edx, [@@x] + cmp [(TField ptr ebx).x], dl + jne @nextClosed + mov edx, [@@y] + cmp [(TField ptr ebx).y], dl + jne @nextClosed + + ; If reached here, yep, contained in closed list + mov eax, 1 + ret + + @nextClosed: + add ebx, SIZE TField + dec ecx + jnz @loopClosed + + mov eax, 0 + ret + +notWithinBounds: + mov eax, 1 + ret +ENDP isClosed + +PROC addNeighbours + USES eax, ebx, ecx, edx + ARG @@tgtx:dword, \ + @@tgty:dword + ; Push all neighbours of currentOpen on openList + + xor ebx, ebx + xor ecx, ecx + + mov bl, [currentOpen.x] + mov cl, [currentOpen.y] + mov edx, [currentOpen.distance] + inc edx ; Next distance is one more. + + ; Up + dec ecx + call isClosed, ebx, ecx + cmp eax, 0 + jne @noUp + call distance, ebx, ecx, [@@tgtx], [@@tgty] + add eax, edx + call addOpen, ebx, ecx, eax, edx + @noUp: + inc ecx + + ; Right + inc ebx + call isClosed, ebx, ecx + cmp eax, 0 + jne @noRight + call distance, ebx, ecx, [@@tgtx], [@@tgty] + add eax, edx + call addOpen, ebx, ecx, eax, edx + @noRight: + dec ebx + + ; Left + dec ebx + call isClosed, ebx, ecx + cmp eax, 0 + jne @noLeft + call distance, ebx, ecx, [@@tgtx], [@@tgty] + add eax, edx + call addOpen, ebx, ecx, eax, edx + @noLeft: + inc ebx + + ; Down + inc ecx + call isClosed, ebx, ecx + cmp eax, 0 + jne @noDown + call distance, ebx, ecx, [@@tgtx], [@@tgty] + add eax, edx + call addOpen, ebx, ecx, eax, edx + @noDown: + dec ecx + + ret +ENDP addNeighbours + +PROC popOpen + ARG RETURNS eax + USES ebx, ecx, edx, esi, edi + ; eax contains the smallest current heuristic + ; ebx contains the index of that field + + cmp [openlistSize], 0 ; If empty, return 0 + jne @goForth + + mov eax, 0 + ret + +@goForth: + + mov eax, 0FFFFFFFFh ; Longest distance possible in 32 bits. + xor ebx, ebx + xor ecx, ecx ; ecx contains the current index + +@searchFurther: + mov edx, ecx + imul edx, SIZE TPriorityField + cmp [(TPriorityField ptr (openlist + edx)).heuristic], eax + ja @notBetter + ; Better guess found, put right values in eax and ebx + mov eax, [(TPriorityField ptr (openlist + edx)).heuristic] + mov ebx, ecx + +@notBetter: + + inc ecx + cmp cx, [openlistSize] + jne @searchFurther + + ; By now, we have found the right item to pop from the priorityqueue. + + ; Move the correct item in currentOpen + mov ecx, SIZE TPriorityField + mov esi, ebx + imul esi, ecx + add esi, offset openlist + + mov edi, offset currentOpen + rep movsb + + ; Now make the remove the thing from the vector + + xor ecx, ecx + mov cx, [openlistSize] + sub ecx, ebx + dec ecx + imul ecx, SIZE TPriorityField + mov edi, esi + sub edi, SIZE TPriorityField + rep movsb + + dec [openlistSize] + mov eax, 1 + ret +ENDP popOpen + +PROC addClosed + USES eax, ebx + + xor ebx, ebx + xor eax, eax + + mov bx, [closedlistSize] + imul ebx, SIZE TField + add ebx, offset closedlist ; ebx contains the target TField + + mov al, [currentOpen.x] + mov [(TField ptr ebx).x], al + mov al, [currentOpen.y] + mov [(TField ptr ebx).y], al + mov eax, [currentOpen.distance] + mov [(TField ptr ebx).distance], eax + + inc [closedlistSize] + cmp [closedlistSize], CLOSED_LIST_SIZE_MAX + jne @noProblemWithClosedVector + + xor eax, eax + mov ax, [closedlistSize] + call crash, offset closedOutOfMemory, eax + +@noProblemWithClosedVector: + ret +ENDP addClosed + +PROC addOpen + USES eax, ebx + ARG @@x:dword, \ + @@y:dword, \ + @@priority:dword, \ + @@distance:dword + + xor eax, eax + mov ax, [openlistSize] + imul eax, SIZE TPriorityField + add eax, offset openlist + + mov ebx, [@@x] + mov [(TPriorityField ptr eax).x], bl + mov ebx, [@@y] + mov [(TPriorityField ptr eax).y], bl + + mov bl, [currentOpen.x] + mov [(TPriorityField ptr eax).fromx], bl + mov bl, [currentOpen.y] + mov [(TPriorityField ptr eax).fromy], bl + + mov ebx, [@@priority] + mov [(TPriorityField ptr eax).heuristic], ebx + mov ebx, [@@distance] + mov [(TPriorityField ptr eax).distance], ebx + + inc [openlistSize] + cmp [openlistSize], OPEN_LIST_SIZE_MAX + jne @noProblem + + xor eax, eax + mov ax, [openlistSize] + call crash, offset openOutOfMemory, eax + +@noProblem: + ret +ENDP + +PROC distance + USES ebx + ARG @@srcx:dword, \ + @@srcy:dword, \ + @@tgtx:dword, \ + @@tgty:dword \ + RETURNS eax + + mov eax, [@@srcx] + sub eax, [@@tgtx] + + jns @noSignChangex + neg eax + + @noSignChangex: + + mov ebx, [@@srcy] + sub ebx, [@@tgty] + + jns @noSignChangey + neg ebx + + @noSignChangey: + add eax, ebx + ret +ENDP distance + +PROC cleanData + USES eax, ecx + mov [openlistSize], 0 + mov [closedlistSize], 0 + + mov [currentOpen.x], -1 + mov [currentOpen.y], -1 + mov [currentOpen.distance], 0 + + call getLevelWidth + mov ecx, eax + call getLevelHeight + imul ecx, eax + + mov eax, offset backtraceGraph +@fieldIter: + mov [(TField ptr eax).distance], 0ffffffffh ; Set to approximately +inf + mov [(TField ptr eax).x], 0 + mov [(TField ptr eax).y], 0 + add eax, SIZE TField + dec ecx + jnz @fieldIter + + ret +ENDP cleanData + +DATASEG + +openOutOfMemory db "Out of openlistSize memory. Hi dev: Please increase$" +closedOutOfMemory db "Out of closedlistSize memory. Hi dev: Please increase$" + +; power | discover | walking | sailing | flying +actionTable db 00001101b, \ ;EMPTY + 00001101b, \ ;RUBBLE + 00000000b, \ ;GRAVEL + 00000000b, \ ;LOOSE ROCK + 00000000b, \ ;HARD ROCK + 00000000b, \ ;MASSIVE ROCK + 00000000b, \ ;KRISTAL SOURCE + 00000000b, \ ;OREROCK + 00001011b, \ ;WATER + 00001001b, \ ;LAVA + 00001101b, \ ;SNAIL HOLE + 00001101b, \ ;EROSION + 00011101b, \ ;POWER PATH + 00011101b, \ ;BUILDING POWER PATH + 00011000b \ ;BUILDING + +UDATASEG + +currentType dd ? +currentOpen TPriorityField ? + +openlist TPriorityField OPEN_LIST_SIZE_MAX dup(?) +openlistSize dw ? +closedlist TField CLOSED_LIST_SIZE_MAX dup(?) +closedlistSize dw ? +backtraceGraph TField MAX_LEVEL_SIZE dup(?) + +END diff --git a/tests/examplefiles/example.whiley b/tests/examplefiles/example.whiley new file mode 100644 index 00000000..74b39370 --- /dev/null +++ b/tests/examplefiles/example.whiley @@ -0,0 +1,296 @@ +/** + * Example Whiley program, taken from the Whiley benchmark suite. + * https://github.com/Whiley/WyBench/blob/master/src/101_interpreter/Main.whiley + */ + +import whiley.lang.System +import whiley.lang.Int +import whiley.io.File +import string from whiley.lang.ASCII +import char from whiley.lang.ASCII + +// ==================================================== +// A simple calculator for expressions +// ==================================================== + +constant ADD is 0 +constant SUB is 1 +constant MUL is 2 +constant DIV is 3 + +// binary operation +type BOp is (int x) where ADD <= x && x <= DIV +type BinOp is { BOp op, Expr lhs, Expr rhs } + +// variables +type Var is { string id } + +// list access +type ListAccess is { + Expr src, + Expr index +} + +// expression tree +type Expr is int | // constant + Var | // variable + BinOp | // binary operator + Expr[] | // array constructor + ListAccess // list access + +// values +type Value is int | Value[] + +// stmts +type Print is { Expr rhs } +type Set is { string lhs, Expr rhs } +type Stmt is Print | Set + +// ==================================================== +// Expression Evaluator +// ==================================================== + +type RuntimeError is { string msg } +type Environment is [{string k, Value v}] + +// Evaluate an expression in a given environment reducing either to a +// value, or a runtime error. The latter occurs if evaluation gets +// "stuck" (e.g. expression is // not well-formed) +function evaluate(Expr e, Environment env) -> Value | RuntimeError: + // + if e is int: + return e + else if e is Var: + return env[e.id] + else if e is BinOp: + Value|RuntimeError lhs = evaluate(e.lhs, env) + Value|RuntimeError rhs = evaluate(e.rhs, env) + // check if stuck + if !(lhs is int && rhs is int): + return {msg: "arithmetic attempted on non-numeric value"} + // switch statement would be good + if e.op == ADD: + return lhs + rhs + else if e.op == SUB: + return lhs - rhs + else if e.op == MUL: + return lhs * rhs + else if rhs != 0: + return lhs / rhs + return {msg: "divide-by-zero"} + else if e is Expr[]: + [Value] r = [] + for i in e: + Value|RuntimeError v = evaluate(i, env) + if v is RuntimeError: + return v + else: + r = r ++ [v] + return r + else if e is ListAccess: + Value|RuntimeError src = evaluate(e.src, env) + Value|RuntimeError index = evaluate(e.index, env) + // santity checks + if src is [Value] && index is int && index >= 0 && index < |src|: + return src[index] + else: + return {msg: "invalid list access"} + else: + return 0 // dead-code + +// ==================================================== +// Expression Parser +// ==================================================== + +type State is { string input, int pos } +type SyntaxError is { string msg, int start, int end } + +function SyntaxError(string msg, int start, int end) -> SyntaxError: + return { msg: msg, start: start, end: end } + +// Top-level parse method +function parse(State st) -> (Stmt,State)|SyntaxError: + // + Var keyword, Var v + Expr e + int start = st.pos + // + keyword,st = parseIdentifier(st) + switch keyword.id: + case "print": + any r = parseAddSubExpr(st) + if !(r is SyntaxError): + e,st = r + return {rhs: e},st + else: + return r // error case + case "set": + st = parseWhiteSpace(st) + v,st = parseIdentifier(st) + any r = parseAddSubExpr(st) + if !(r is SyntaxError): + e,st = r + return {lhs: v.id, rhs: e},st + else: + return r // error case + default: + return SyntaxError("unknown statement",start,st.pos-1) + +function parseAddSubExpr(State st) -> (Expr, State)|SyntaxError: + // + Expr lhs, Expr rhs + // First, pass left-hand side + any r = parseMulDivExpr(st) + // + if r is SyntaxError: + return r + // + lhs,st = r + st = parseWhiteSpace(st) + // Second, see if there is a right-hand side + if st.pos < |st.input| && st.input[st.pos] == '+': + // add expression + st.pos = st.pos + 1 + r = parseAddSubExpr(st) + if !(r is SyntaxError): + rhs,st = r + return {op: ADD, lhs: lhs, rhs: rhs},st + else: + return r + else if st.pos < |st.input| && st.input[st.pos] == '-': + // subtract expression + st.pos = st.pos + 1 + r = parseAddSubExpr(st) + if !(r is SyntaxError): + rhs,st = r + return {op: SUB, lhs: lhs, rhs: rhs},st + else: + return r + // No right-hand side + return (lhs,st) + +function parseMulDivExpr(State st) -> (Expr, State)|SyntaxError: + // First, parse left-hand side + Expr lhs, Expr rhs + any r = parseTerm(st) + if r is SyntaxError: + return r + // + lhs,st = r + st = parseWhiteSpace(st) + // Second, see if there is a right-hand side + if st.pos < |st.input| && st.input[st.pos] == '*': + // add expression + st.pos = st.pos + 1 + r = parseMulDivExpr(st) + if !(r is SyntaxError): + rhs,st = r + return {op: MUL, lhs: lhs, rhs: rhs}, st + else: + return r + else if st.pos < |st.input| && st.input[st.pos] == '/': + // subtract expression + st.pos = st.pos + 1 + r = parseMulDivExpr(st) + if !(r is SyntaxError): + rhs,st = r + return {op: DIV, lhs: lhs, rhs: rhs}, st + else: + return r + // No right-hand side + return (lhs,st) + +function parseTerm(State st) -> (Expr, State)|SyntaxError: + // + st = parseWhiteSpace(st) + if st.pos < |st.input|: + if ASCII.isLetter(st.input[st.pos]): + return parseIdentifier(st) + else if ASCII.isDigit(st.input[st.pos]): + return parseNumber(st) + else if st.input[st.pos] == '[': + return parseList(st) + // + return SyntaxError("expecting number or variable",st.pos,st.pos) + +function parseIdentifier(State st) -> (Var, State): + // + string txt = "" + // inch forward until end of identifier reached + while st.pos < |st.input| && ASCII.isLetter(st.input[st.pos]): + txt = txt ++ [st.input[st.pos]] + st.pos = st.pos + 1 + return ({id:txt}, st) + +function parseNumber(State st) -> (Expr, State)|SyntaxError: + // inch forward until end of identifier reached + int start = st.pos + while st.pos < |st.input| && ASCII.isDigit(st.input[st.pos]): + st.pos = st.pos + 1 + // + int|null iv = Int.parse(st.input[start..st.pos]) + if iv == null: + return SyntaxError("Error parsing number",start,st.pos) + else: + return iv, st + +function parseList(State st) -> (Expr, State)|SyntaxError: + // + st.pos = st.pos + 1 // skip '[' + st = parseWhiteSpace(st) + [Expr] l = [] // initial list + bool firstTime = true + while st.pos < |st.input| && st.input[st.pos] != ']': + if !firstTime && st.input[st.pos] != ',': + return SyntaxError("expecting comma",st.pos,st.pos) + else if !firstTime: + st.pos = st.pos + 1 // skip ',' + firstTime = false + any r = parseAddSubExpr(st) + if r is SyntaxError: + return r + else: + Expr e + e,st = r + // perform annoying error check + l = l ++ [e] + st = parseWhiteSpace(st) + st.pos = st.pos + 1 + return l,st + +// Parse all whitespace upto end-of-file +function parseWhiteSpace(State st) -> State: + while st.pos < |st.input| && ASCII.isWhiteSpace(st.input[st.pos]): + st.pos = st.pos + 1 + return st + +// ==================================================== +// Main Method +// ==================================================== + +public method main(System.Console sys): + if(|sys.args| == 0): + sys.out.println("no parameter provided!") + else: + File.Reader file = File.Reader(sys.args[0]) + string input = ASCII.fromBytes(file.readAll()) + + Environment env = Environment() + State st = {pos: 0, input: input} + while st.pos < |st.input|: + Stmt s + any r = parse(st) + if r is SyntaxError: + sys.out.println("syntax error: " ++ r.msg) + return + s,st = r + Value|RuntimeError v = evaluate(s.rhs,env) + if v is RuntimeError: + sys.out.println("runtime error: " ++ v.msg) + return + if s is Set: + env[s.lhs] = v + else: + sys.out.println(r) + st = parseWhiteSpace(st) + diff --git a/tests/examplefiles/example.xtm b/tests/examplefiles/example.xtm new file mode 100644 index 00000000..927117da --- /dev/null +++ b/tests/examplefiles/example.xtm @@ -0,0 +1,1101 @@ +;;; example.xtm -- Extempore code examples + +;; Author: Ben Swift, Andrew Sorensen +;; Keywords: extempore + +;;; Commentary: + + + +;;; Code: + +;; bit twiddling + +(xtmtest '(bind-func test_bit_twiddle_1 + (lambda () + (bitwise-and 65535 255 15 1))) + + (test_bit_twiddle_1) 1) + +(xtmtest '(bind-func test_bit_twiddle_2 + (lambda () + (bitwise-not -1))) + + (test_bit_twiddle_2) 0) + +(xtmtest '(bind-func test_bit_twiddle_3 + (lambda () + (bitwise-not 0))) + + (test_bit_twiddle_3) -1) + +(xtmtest '(bind-func test_bit_twiddle_4 + (lambda () + (bitwise-shift-right 65535 8) + (bitwise-shift-right 65535 4 4))) + + (test_bit_twiddle_4) 255) + +(xtmtest '(bind-func test_bit_twiddle_5 + (lambda () + (bitwise-shift-left (bitwise-shift-right 65535 8) 4 4))) + + (test_bit_twiddle_5) 65280) + +(xtmtest '(bind-func test_bit_twiddle_6 + (lambda () + (bitwise-and (bitwise-or (bitwise-eor 21844 65534) (bitwise-eor 43690 65534)) 1))) + + (test_bit_twiddle_6) 0) + +;; integer literals default to 64 bit integers +(xtmtest '(bind-func int-literal-test + (lambda (a) + (* a 5))) + + (int-literal-test 6) 30) + +;; float literals default to doubles +(xtmtest '(bind-func float-literal-test + (lambda (a) + (* a 5.0))) + + (float-literal-test 6.0) 30.0) + +;; you are free to recompile an existing closure +(xtmtest '(bind-func int-literal-test + (lambda (a) + (/ a 5))) + + (int-literal-test 30)) + +(xtmtest '(bind-func closure-test1 + (let ((power 0)) + (lambda (x) + (set! power (+ power 1)) ;; set! for closure mutation as per scheme + (* x power)))) + + (closure-test1 2)) + +(xtmtest '(bind-func closure-returns-closure-test + (lambda () + (lambda (x) + (* x 3)))) + + (closure-returns-closure-test)) + +(xtmtest '(bind-func incrementer-test1 + (lambda (i:i64) + (lambda (incr) + (set! i (+ i incr)) + i))) + + (incrementer-test1 0)) + +(define myf (incrementer-test1 0)) + +;; so we need to type f properly +(xtmtest '(bind-func incrementer-test2 + (lambda (f:[i64,i64]* x) + (f x))) + (incrementer-test2 myf 1) 1) + +;; and we can call my-in-maker-wrapper +;; to appy myf +(xtmtest-result (incrementer-test2 myf 1) 2) +(xtmtest-result (incrementer-test2 myf 1) 3) +(xtmtest-result (incrementer-test2 myf 1) 4) + +;; of course the wrapper is only required if you +;; need interaction with the scheme world. +;; otherwise you just call my-inc-maker directly + +;; this avoids the wrapper completely +(xtmtest '(bind-func incrementer-test3 + (let ((f (incrementer-test1 0))) + (lambda () + (f 1)))) + + (incrementer-test3) 1) + +(xtmtest-result (incrementer-test3) 2) +(xtmtest-result (incrementer-test3) 3) + +;; hopefully you're getting the idea. +;; note that once we've compiled something +;; we can then use it any of our new +;; function definitions. + +;; do a little 16bit test +(xtmtest '(bind-func bitsize-sixteen + (lambda (a:i16) + (dtoi16 (* (i16tod a) 5.0)))) + + (bitsize-sixteen 5) 25) + +;; while loop test + +(xtmtest '(bind-func test_while_loop_1 + (lambda () + (let ((count 0)) + (while (< count 5) + (printf "count = %lld\n" count) + (set! count (+ count 1))) + count))) + + (test_while_loop_1) 5) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Closures can be recursive +;; + +(xtmtest '(bind-func recursive-closure-test + (lambda (a) + (if (< a 1) + (printf "done\n") + (begin (printf "a: %lld\n" a) + (recursive-closure-test (- a 1)))))) + + (recursive-closure-test 3)) + +;; check TAIL OPTIMIZATION +;; if there is no tail call optimiation +;; in place then this should blow the +;; stack and crash the test + +;; CANNOT RUN THIS TEST ON WINDOWS (i.e. no salloc)! +(if (not (equal? (sys:platform) "Windows")) + (xtmtest '(bind-func tail_opt_test + (lambda (n:i64) + (let ((a:float* (salloc 8000))) + (if (= n 0) + (printf "tail opt test passed!\n") + (tail_opt_test (- n 1)))))) + + (tail_opt_test 200))) + +(println 'A 'segfault 'here 'incidates 'that 'tail-call-optimizations 'are 'not 'working!) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; some anon lambda tests +;; + +(xtmtest '(bind-func infer_lambdas_test + (lambda () + (let ((a 5) + (b (lambda (x) (* x x))) + (c (lambda (y) (* y y)))) + (c (b a))))) + + (infer_lambdas_test)) + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; a simple tuple example +;; +;; tuple types are represented as <type,type,type>* +;; + +;; make and return a simple tuple +(xtmtest '(bind-func tuple-test1 + (lambda () + (let ((t:<i64,double,i32>* (alloc))) + t))) + + (tuple-test1)) + +;; logview shows [<i64,double,i32>*]* +;; i.e. a closure that takes no arguments +;; and returns the tuple <i64,double,i32>* + + +;; here's another tuple example +;; note that my-test-7's return type is inferred +;; by the tuple-reference index +;; (i.e. i64 being tuple index 0) +(xtmtest '(bind-func tuple-test2 + (lambda () + (let ((a:<i64,double>* (alloc)) ; returns pointer to type <i64,double> + (b 37) + (c 6.4)) + (tuple-set! a 0 b) ;; set i64 to 64 + (tset! a 1 c) ;; set double to 6.4 - tset! is an alias for tuple-set! + (printf "tuple:1 %lld::%f\n" (tuple-ref a 0) (tref a 1)) + ;; we can fill a tuple in a single call by using tfill! + (tfill! a 77 77.7) + (printf "tuple:2 %lld::%f\n" (tuple-ref a 0) (tuple-ref a 1)) + (tuple-ref a 0)))) + + (tuple-test2) 77) + +;; return first element which is i64 +;; should be 64 as we return the +;; first element of the tuple +;; (println (my-test-7)) ; 77 + + +;; tbind binds variables to values +;; based on tuple structure +;; _ (underscore) means don't attempt +;; to match against this position in +;; the tuple (i.e. skip) +(xtmtest '(bind-func tuple-bind-test + (lambda () + (let ((t1:<i32,float,<i32,float>*,double>* (alloc)) + (t2:<i32,float>* (alloc)) + (a 0) (b:float 0.0) (c 0.0)) + (tfill! t2 3 3.3) + (tfill! t1 1 2.0 t2 4.0) + (tbind t1 a b _ c) + c))) + + (tuple-bind-test) 4.0) + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; some array code with *casting* +;; this function returns void +(xtmtest '(bind-func array-test1 + (lambda () + (let ((v1:|5,float|* (alloc)) + (v2:|5,float|* (alloc)) + (i 0) + (k 0)) + (dotimes (i 5) + ;; random returns double so "truncate" to float + ;; which is what v expects + (array-set! v1 i (dtof (random)))) + ;; we can use the afill! function to fill an array + (afill! v2 1.1 2.2 3.3 4.4 5.5) + (dotimes (k 5) + ;; unfortunately printf doesn't like floats + ;; so back to double for us :( + (printf "val: %lld::%f::%f\n" k + (ftod (array-ref v1 k)) + (ftod (aref v2 k))))))) + + (array-test1)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; some crazy array code with +;; closures and arrays +;; try to figure out what this all does +;; +;; this example uses the array type +;; the pretty print for this type is +;; |num,type| num elements of type +;; |5,i64| is an array of 5 x i64 +;; +;; An array is not a pointer type +;; i.e. |5,i64| cannot be bitcast to i64* +;; +;; However an array can be a pointer +;; i.e. |5,i64|* can be bitcast to i64* +;; i.e. |5,i64|** to i64** etc.. +;; +;; make-array returns a pointer to an array +;; i.e. (make-array 5 i64) returns type |5,i64|* +;; +;; aref (array-ref) and aset! (array-set!) +;; can operate with either pointers to arrays or +;; standard pointers. +;; +;; in other words aref and aset! are happy +;; to work with either i64* or |5,i64|* + +(bind-func array-test2 + (lambda (v:|5,i64|*) + (let ((f (lambda (x) + (* (array-ref v 2) x)))) + f))) + +(bind-func array-test3 + (lambda (v:|5,[i64,i64]*|*) + (let ((ff (aref v 0))) ; aref alias for array-ref + (ff 5)))) + +(xtmtest '(bind-func array-test4 + (lambda () + (let ((v:|5,[i64,i64]*|* (alloc)) ;; make an array of closures! + (vv:|5,i64|* (alloc))) + (array-set! vv 2 3) + (aset! v 0 (array-test2 vv)) ;; aset! alias for array-set! + (array-test3 v)))) + + ;; try to guess the answer before you call this!! + (array-test4)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; some conditionals + +(xtmtest '(bind-func cond-test1 + (lambda (x:i64 y) + (if (> x y) + x + y))) + + (cond-test1 12 13)) + +;; returns boolean true +(xtmtest '(bind-func cond-test2 + (lambda (x:i64) + (cond ((= x 1) (printf "A\n")) + ((= x 2) (printf "B\n")) + ((= x 3) (printf "C\n")) + ((= x 4) (printf "D\n")) + (else (printf "E\n"))) + #t)) + + (cond-test2 1)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; making a linear envelop generator +;; for signal processing and alike + +(bind-func envelope-segments + (lambda (points:double* num-of-points:i64) + (let ((lines:[double,double]** (zone-alloc num-of-points)) + (k 0)) + (dotimes (k num-of-points) + (let* ((idx (* k 2)) + (x1 (pointer-ref points (+ idx 0))) + (y1 (pointer-ref points (+ idx 1))) + (x2 (pointer-ref points (+ idx 2))) + (y2 (pointer-ref points (+ idx 3))) + (m (if (= 0.0 (- x2 x1)) 0.0 (/ (- y2 y1) (- x2 x1)))) + (c (- y2 (* m x2))) + (l (lambda (time) (+ (* m time) c)))) + (pointer-set! lines k l))) + lines))) + +(bind-func make-envelope + (lambda (points:double* num-of-points) + (let ((klines:[double,double]** (envelope-segments points num-of-points)) + (line-length num-of-points)) + (lambda (time) + (let ((res -1.0) + (k:i64 0)) + (dotimes (k num-of-points) + (let ((line (pointer-ref klines k)) + (time-point (pointer-ref points (* k 2)))) + (if (or (= time time-point) + (< time-point time)) + (set! res (line time))))) + res))))) + +;; make a convenience wrapper +(xtmtest '(bind-func env-wrap + (let* ((points 3) + (data:double* (zone-alloc (* points 2)))) + (pointer-set! data 0 0.0) ;; point data + (pset! data 1 0.0) + (pset! data 2 2.0) + (pset! data 3 1.0) + (pset! data 4 4.0) + (pset! data 5 0.0) + (let ((f (make-envelope data points))) + (lambda (time:double) + (f time))))) + (env-wrap 0.0) 0.0) + +(xtmtest-result (env-wrap 1.0) 0.5) +(xtmtest-result (env-wrap 2.0) 1.0) +(xtmtest-result (env-wrap 2.5) 0.75) +(xtmtest-result (env-wrap 4.0) 0.0) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; direct access to a closures environment +;; +;; it is possible to directly access a closures +;; environment in order to read or modify data +;; at runtime. +;; +;; You do this using a dot operator +;; To access an environment slot you use +;; closure.slot:type +;; So for example +;; (f.a:i32) +;; would return the 32bit integer symbol 'a' +;; from the closure 'f' +;; +;; To set an environment slot you just +;; add a value of the correct type +;; for example +;; (f.a:i32 565) +;; would set 'a' in 'f' to 565 +;; +;; let's create a closure that capture's 'a' + + +(xtmtest '(bind-func dot-access-test1 + (let ((a:i32 6)) + (lambda () + (printf "a:%d\n" a) + a))) + (dot-access-test1)) + +;; now let's create a new function +;; that calls my-test14 twice +;; once normally +;; then we directly set the closures 'a' binding +;; then call again +;; +(xtmtest '(bind-func dot-access-test2 + (lambda (x:i32) + (dot-access-test1) + (dot-access-test1.a:i32 x) + (dot-access-test1))) + + (dot-access-test2 9)) + +;; of course this works just as well for +;; non-global closures +(xtmtest '(bind-func dot-access-test3 + (lambda (a:i32) + (let ((f (lambda () + (* 3 a)))) + f))) + (dot-access-test3 1)) + +(xtmtest '(bind-func dot-access-test4 + (lambda () + (let ((f (dot-access-test3 5))) + (f.a:i32 7) + (f)))) + + (dot-access-test4) + 21) + +;; and you can get and set closures also! +(xtmtest '(bind-func dot-access-test5 + (lambda () + (let ((f (lambda (x:i64) x))) + (lambda (z) + (f z))))) + + (dot-access-test5)) + +(xtmtest '(bind-func dot-access-test6 + (lambda () + (let ((t1 (dot-access-test5)) + (t2 (dot-access-test5))) + ;; identity of 5 + (printf "%lld:%lld\n" (t1 5) (t2 5)) + (t1.f:[i64,i64]* (lambda (x:i64) (* x x))) + ;; square of 5 + (printf "%lld:%lld\n" (t1 5) (t2 5)) + ;; cube of 5 + (t2.f:[i64,i64]* (lambda (y:i64) (* y y y))) + (printf "%lld:%lld\n" (t1 5) (t2 5)) + void))) + + (dot-access-test6)) ;; 5:5 > 25:5 > 25:125 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; named types + +;; it can sometimes be helpful to allocate +;; a predefined tuple type on the stack +;; you can do this using allocate +(bind-type vec3 <double,double,double>) + +;; String printing! +(bind-func vec3_print:[void,vec3*]* + (lambda (x) + (printf "<%d,%d,%d>" (tref x 0) (tref x 1) (tref x 2)) + void)) + +(bind-poly print vec3_print) + +;; note that point is deallocated at the +;; end of the function call. You can +;; stack allocate (stack-alloc) +;; any valid type (i64 for example) +(xtmtest '(bind-func salloc-test + (lambda () + (let ((point:vec3* (stack-alloc))) + (tset! point 0 0.0) + (tset! point 1 -1.0) + (tset! point 2 1.0) + 1))) + + (salloc-test)) ;; 1 + +;; all named types have 2 default constructors +;; name (zone alloation) + name_h (heap allocation) +;; and a default print poly +(xtmtest '(bind-func data-constructor-test + (lambda () + (let ((v1 (vec3 1.0 2.0 3.0)) + (v2 (vec3_h 4.0 5.0 6.0))) + (println v1 v2) + ;; halloced vec3 needs freeing + (free v2) + void))) + + (data-constructor-test)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; aref-ptr and tref-ptr +;; + +;; aref-ptr and tref-ptr return a pointer to an element +;; just as aref and tref return elements aref-ptr and +;; tref-ptr return a pointer to those elements. + +;; This allows you to do things like create an array +;; with an offset +(xtmtest '(bind-func aref-ptr-test + (lambda () + (let ((arr:|32,i64|* (alloc)) + (arroff (aref-ptr arr 16)) + (i 0) + (k 0)) + ;; load arr + (dotimes (i 32) (aset! arr i i)) + (dotimes (k 16) + (printf "index: %lld\tarr: %lld\tarroff: %lld\n" + k (aref arr k) (pref arroff k)))))) + + (aref-ptr-test)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; arrays +;; Extempore lang supports arrays as for first class +;; aggregate types (in other words as distinct from +;; a pointer). +;; +;; an array is made up of a size and a type +;; |32,i64| is an array of 32 elements of type i64 +;; + +(bind-type tuple-with-array <double,|32,|4,i32||,float>) + +(xtmtest '(bind-func array-test5 + (lambda () + (let ((tup:tuple-with-array* (stack-alloc)) + (t2:|32,i64|* (stack-alloc))) + (aset! t2 0 9) + (tset! tup 2 5.5) + (aset! (aref-ptr (tref-ptr tup 1) 0) 0 0) + (aset! (aref-ptr (tref-ptr tup 1) 0) 1 1) + (aset! (aref-ptr (tref-ptr tup 1) 0) 2 2) + (printf "val: %lld %lld %f\n" + (aref (aref-ptr (tref-ptr tup 1) 0) 1) + (aref t2 0) (ftod (tref tup 2))) + (aref (aref-ptr (tref-ptr tup 1) 0) 1)))) + + (array-test5) 1) ;; val: 1 9 5.5 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Global Variables +;; +;; You can allocate global variables using bind-val +;; + +(bind-val g_var_a i32 5) + +;; increment g_var_a by inc +;; and return new value of g_var_a +(xtmtest '(bind-func global_var_test1 + (lambda (incr) + (set! g_var_a (+ g_var_a incr)) + g_var_a)) + + (global_var_test1 3) 8) ;; 8 + +;; you can bind any primitive type +(bind-val g_var_b double 5.5) +(bind-val g_var_c i1 0) + +(xtmtest '(bind-func global_var_test1b + (lambda () + (* g_var_b (if g_var_c 1.0 4.0)))) + + (global_var_test1b) 22.0) + +;; global strings + +(bind-val g_cstring i8* "Jiblet.") + +(xtmtest '(bind-func test_g_cstring + (lambda () + (let ((i 0)) + (dotimes (i 7) + (printf "g_cstring[%lld] = %c\n" i (pref g_cstring i))) + (printf "\nSpells... %s\n" g_cstring)))) + + (test_g_cstring)) + +(xtmtest '(bind-func test_g_cstring1 + (lambda () + (let ((test_cstring "Niblot.") + (i 0) + (total 0)) + (dotimes (i 7) + (let ((c1 (pref g_cstring i)) + (c2 (pref test_cstring i))) + (printf "checking %c against %c\n" c1 c2) + (if (= c1 c2) + (set! total (+ total 1))))) + total))) + + (test_g_cstring1) 5) + + + + + +;; for tuples, arrays and vectors, bind-val only takes *two* +;; arguments. The tuple/array/vector will be initialised to zero. + +(bind-val g_tuple1 <i64,i64>) +(bind-val g_tuple2 <double,double>) + +(xtmtest '(bind-func test_g_tuple + (lambda () + (tfill! g_tuple1 1 4) + (tfill! g_tuple2 4.0 1.0) + (and (= (tref g_tuple1 0) (dtoi64 (tref g_tuple2 1))) + (= (dtoi64 (tref g_tuple2 0)) (tref g_tuple1 1))))) + + (test_g_tuple) 1) + +;; same thing with arrays + +(bind-val g_array1 |10,double|) +(bind-val g_array2 |10,i64|) + +;; if we just loop over and print the values in each array + +(xtmtest '(bind-func test_g_array11 + (lambda () + (let ((i 0)) + (dotimes (i 10) + (printf "garray_1[%lld] = %f garray_2[%lld] = %lld\n" + i (aref g_array1 i) i (aref g_array2 i)))))) + + (test_g_array11) 1) + +;; but if we loop over and set some values into the arrays + +(xtmtest '(bind-func test_g_array2 + (lambda () + (let ((i 0)) + (dotimes (i 10) + (aset! g_array1 i (i64tod i)) + (aset! g_array2 i i) + (printf "garray_1[%lld] = %f garray_2[%lld] = %lld\n" + i (aref g_array1 i) i (aref g_array2 i))) + (= (dtoi64 (aref g_array1 5)) + (aref g_array2 5))))) + + (test_g_array2) 1) + +;; just to test, let's try a large array + +(bind-val g_array3 |100000000,i64|) + +(xtmtest '(bind-func test_g_array3 + (lambda () + (let ((i 0)) + (dotimes (i 100000000) + (aset! g_array3 i i)) + (= (pref g_array3 87654321) + 87654321)))) + + (test_g_array3) 1) + +;; if you want to bind a global pointer, then the third 'value' +;; argument is the size of the memory to allocate (in elements, not in bytes) + +(bind-val g_ptr0 double* 10) + +(xtmtest '(bind-func test_g_ptr0 + (lambda () + (let ((total 0.0) + (i 0)) + (dotimes (i 10) + (pset! g_ptr0 i (i64tod i)) + (set! total (+ total (pref g_ptr0 i)))) + total))) + + (test_g_ptr0) 45.0) + +(bind-val g_ptr1 |4,i32|* 2) +(bind-val g_ptr2 <i64,double>* 4) + +(xtmtest '(bind-func test_g_ptr1 + (lambda () + (afill! g_ptr1 11 66 35 81) + (tset! g_ptr2 1 35.0) + (printf "%f :: %d\n" (tref g_ptr2 1) (aref g_ptr1 2)) + (aref g_ptr1 3))) + + (test_g_ptr1) 81) ;; should also print 35.000000 :: 35 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Callbacks + +(xtmtest '(bind-func callback-test + (lambda (time:i64 count:i64) + (printf "time: %lld:%lld\n" time count) + (callback (+ time 1000) callback-test (+ time 22050) (+ count 1)))) + + (callback-test (now) 0)) + +;; compiling this will stop the callbacks +;; +;; of course we need to keep the type +;; signature the same [void,i64,i64]* +;; +(xtmtest '(bind-func callback-test + (lambda (time:i64 count:i64) + #t)) + + (callback-test)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; some memzone tests + +(xtmtest '(bind-func memzone-test1 + (lambda () + (let ((b:|5,double|* (zalloc))) + (aset! b 0 + (memzone 1024 + (let ((a:|10,double|* (zalloc))) + (aset! a 0 3.5) + (aref a 0)))) + (let ((c:|9,i32|* (zalloc))) + (aset! c 0 99) + (aref b 0))))) + + (memzone-test1) 3.5) + +(xtmtest '(bind-func memzone-test2 + (lambda () + (memzone 1024 + (let ((k:|15,double|* (zalloc)) + (f (lambda (fa:|15,double|*) + (memzone 1024 + (let ((a:|10,double|* (zalloc)) + (i 0)) + (dotimes (i 10) + (aset! a i (* (aref fa i) (random)))) + a))))) + (f k))))) + + (memzone-test2)) + +(xtmtest '(bind-func memzone-test3 + (lambda () + (let ((v (memzone-test2)) + (i 0)) + (dotimes (i 10) (printf "%lld:%f\n" i (aref v i)))))) + + (memzone-test3)) ;; should print all 0.0's + +(xtmtest '(bind-func memzone-test4 + (lambda () + (memzone 1024 (* 44100 10) + (let ((a:|5,double|* (alloc))) + (aset! a 0 5.5) + (aref a 0))))) + + (memzone-test4) 5.50000) + +;; +;; Large allocation of memory on BUILD (i.e. when the closure is created) +;; requires an optional argument (i.e. an amount of memory to allocate +;; specifically for closure creation) +;; +;; This memory is automatically free'd whenever you recompile the closure +;; (it will be destroyed and replaced by a new allocation of the +;; same amount or whatever new amount you have allocated for closure +;; compilation) +;; +(xtmtest '(bind-func closure-zalloc-test 1000000 + (let ((k:|100000,double|* (zalloc))) + (lambda () + (aset! k 0 1.0) + (aref k 0)))) + + (closure-zalloc-test 1000000)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Ad-Hoc Polymorphism +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; extempore supports ad-hoc polymorphism +;; at some stage in the future this will +;; be implicit - but for the moment +;; it is explicitly defined using bind-poly + +;; ad-hoc polymorphism allows you to provide +;; different specialisations depending on +;; type. In other words, a single 'name' +;; can be bound to multiple function +;; implementations each with a uniqute +;; type. + + +;; poly variables can be for functions of +;; mixed argument lengths +;; +;; so for example: +(bind-func poly-test4 + (lambda (a:i8*) + (printf "%s\n" a))) + +(bind-func poly-test5 + (lambda (a:i8* b:i8*) + (printf "%s %s\n" a b))) + +(bind-func poly-test6 + (lambda (a:i8* b:i8* c:i8*) + (printf "%s %s %s\n" a b c))) + +;; bind these three functions to poly 'print' +(bind-poly testprint poly-test4) +(bind-poly testprint poly-test5) +(bind-poly testprint poly-test6) + +(xtmtest '(bind-func poly-test7 + (lambda () + (testprint "extempore's") + (testprint "extempore's" "polymorphism") + (testprint "extempore's" "polymorphism" "rocks"))) + + (poly-test7)) + +;; polys can Also specialize +;; on the return type +(bind-func poly-test8 + (lambda (a:double) + (* a a))) + +(bind-func poly-test9 + (lambda (a:double) + (dtoi64 (* a a)))) + +(bind-poly sqrd poly-test8) +(bind-poly sqrd poly-test9) + +;; specialize on [i64,double]* +;; +(xtmtest '(bind-func poly-test10:[i64,double]* + (lambda (a) + (+ 1 (sqrd a)))) + (poly-test10 5.0)) + +;; specialize on [double,doube]* +(xtmtest '(bind-func poly-test11:[double,double]* + (lambda (a) + (+ 1.0 (sqrd a)))) + + (poly-test11 5.0)) + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; a little test for zone cleanup +;; +(bind-func MyLittleCleanupTest + (lambda () + (let ((tmp2:i8* (alloc 8))) + (cleanup (println "Clean up before leaving zone!")) + tmp2))) + +(xtmtest '(bind-func cleanup-test + (lambda () + (letz ((tmp:i8* (alloc 8)) + (t2 (MyLittleCleanupTest))) + (begin + (println "In Zone ...") + 1)) + (println "Out of zone ...") + void)) + + (cleanup-test)) + +;;;;;;;;;;;;;;;;;; +;; vector types + +;; (bind-func vector-test1 +;; (lambda () +;; (let ((v1:/4,float/* (alloc)) +;; (v2:/4,float/* (alloc)) +;; (v3:/4,float/* (alloc))) +;; (vfill! v1 4.0 3.0 2.0 1.0) +;; (vfill! v2 1.0 2.0 3.0 4.0) +;; (vfill! v3 5.0 5.0 5.0 5.0) +;; (let ((v4 (* v1 v2)) +;; (v5 (> v3 v4))) ;; unforunately vector conditionals don't work! +;; (printf "mul:%f:%f:%f:%f\n" (ftod (vref v4 0)) (ftod (vref v4 1)) (ftod (vref v4 2)) (ftod (vref v4 3))) +;; (printf "cmp:%d:%d:%d:%d\n" (i1toi32 (vref v5 0)) (i1toi32 (vref v5 1)) (i1toi32 (vref v5 2)) (i1toi32 (vref v5 3))) +;; void)))) + +;; (test-xtfunc (vector-test1)) + +(bind-func vector-test2 + (lambda () + (let ((v1:/4,float/* (alloc)) + (v2:/4,float/* (alloc))) + (vfill! v1 1.0 2.0 4.0 8.0) + (vfill! v2 2.0 2.5 2.25 2.125) + (* v1 v2)))) + +(xtmtest '(bind-func vector-test3 + (lambda () + (let ((a (vector-test2))) + (printf "%f:%f:%f:%f\n" + (ftod (vref a 0)) + (ftod (vref a 1)) + (ftod (vref a 2)) + (ftod (vref a 3))) + void))) + + (vector-test3)) + +;; vectorised sine func +(bind-func vsinf4 + (let ((p:/4,float/* (alloc)) + (b:/4,float/* (alloc)) + (c:/4,float/* (alloc)) + (f1:/4,float/* (alloc)) + (f2:/4,float/* (alloc)) + (i:i32 0) + (p_ 0.225) + (b_ (dtof (/ 4.0 3.1415))) + (c_ (dtof (/ -4.0 (* 3.1415 3.1415))))) + (dotimes (i 4) (vset! p i p_) (vset! b i b_) (vset! c i c_)) + (lambda (x:/4,float/) + ;; no SIMD for abs yet! + (dotimes (i 4) (vset! f1 i (fabs (vref x i)))) + (let ((y (+ (* b x) (* c x f1)))) + ;; no SIMD for abs yet! + (dotimes (i 4) (vset! f2 i (fabs (vref y i)))) + (+ (* p (- (* y f2) y)) y))))) + +(bind-func vcosf4 + (let ((p:/4,float/* (alloc)) + (b:/4,float/* (alloc)) + (c:/4,float/* (alloc)) + (d:/4,float/* (alloc)) + (f1:/4,float/* (alloc)) + (f2:/4,float/* (alloc)) + (i:i32 0) + (p_ 0.225) + (d_ (dtof (/ 3.1415 2.0))) + (b_ (dtof (/ 4.0 3.1415))) + (c_ (dtof (/ -4.0 (* 3.1415 3.1415))))) + (dotimes (i 4) + (vset! p i p_) (vset! b i b_) (vset! c i c_) (vset! d i d_)) + (lambda (x:/4,float/) + ;; offset x for cos + (set! x (+ x d)) + ;; no SIMD for abs yet! + (dotimes (i 4) (vset! f1 i (fabs (vref x i)))) + (let ((y (+ (* b x) (* c x f1)))) + ;; no SIMD for abs yet! + (dotimes (i 4) (vset! f2 i (fabs (vref y i)))) + (+ (* p (- (* y f2) y)) y))))) + + +(xtmtest '(bind-func vector-test4 + (lambda () + (let ((a:/4,float/* (alloc))) + (vfill! a 0.1 0.2 0.3 0.4) + (let ((b (vsinf4 (pref a 0))) + (c (vcosf4 (pref a 0)))) + (printf "precision inaccuracy is expected:\n") + (printf " sinf:\t%f,%f,%f,%f\n" + (ftod (sin 0.1:f)) + (ftod (sin 0.2:f)) + (ftod (sin 0.3:f)) + (ftod (sin 0.4:f))) + (printf "vsinf:\t%f,%f,%f,%f\n" + (ftod (vref b 0)) + (ftod (vref b 1)) + (ftod (vref b 2)) + (ftod (vref b 3))) + (printf " cosf:\t%f,%f,%f,%f\n" + (ftod (cos 0.1:f)) + (ftod (cos 0.2:f)) + (ftod (cos 0.3:f)) + (ftod (cos 0.4:f))) + (printf "vcosf:\t%f,%f,%f,%f\n" + (ftod (vref c 0)) + (ftod (vref c 1)) + (ftod (vref c 2)) + (ftod (vref c 3))) + void)))) + + (vector-test4)) + +;; test the call-as-xtlang macro + +;; make sure it'll handle multiple body forms +(xtmtest-result (call-as-xtlang (println 1) (println 2) 5) + 5) + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; test globalvar as closure +;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(bind-func testinc + (lambda (incr:i64) + (lambda (x:i64) + (+ x incr)))) + +(bind-val GlobalInc [i64,i64]* (testinc 2)) + +(xtmtest '(bind-func ginc + (lambda () + (GlobalInc 5))) + (ginc) 7) + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; syntax highlighting tests ;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; these don't return any values, they're visual tests---do they look +;; right? + +(bind-func hl_test1a:[i32,double,|4,i32|**]* 4000 + "docstring" + (lambda (a b) + (printf "done\n"))) + +(bind-func hl_test1b:[i32]* + (lambda () + (let ((i:i32 6)) + (printf "done\n")))) + +(bind-val hl_test2 <i32,i32>) +(bind-val hl_test3 |4,i8|) +(bind-val hl_test4 double* 10) +(bind-val hl_test5 i8* "teststr") + +(bind-type hl_test_type <i64>) + +(println '(bind-lib testlib testfn [i32,i32]*)) + +;; (and 4 5) +;; (bind-val hl_test4 double* 10) +;; (bind-type hl_test_type <i64> "docstring") +;; (bind-lib testlib testfn [i32,i32]*) diff --git a/tests/examplefiles/example.yaml b/tests/examplefiles/example.yaml index 9c0ed9d0..17544c02 100644 --- a/tests/examplefiles/example.yaml +++ b/tests/examplefiles/example.yaml @@ -1,3 +1,12 @@ +# +# Regression tests +# + +%TAG ! tag:example.com:foo/ +--- +test: !foo/bar {a: 'asdf'} +test2: fred +... # # Examples from the Preview section of the YAML specification diff --git a/tests/examplefiles/example2.cpp b/tests/examplefiles/example2.cpp new file mode 100644 index 00000000..ccd99383 --- /dev/null +++ b/tests/examplefiles/example2.cpp @@ -0,0 +1,20 @@ +/* + * A Test file for the different string literals. + */ + +#include <iostream> + +int main() { + char *_str = "a normal string"; + wchar_t *L_str = L"a wide string"; + char *u8_str = u8"utf-8 string"; + char16_t *u_str = u"utf-16 string"; + char32_t *U_str = U"utf-32 string"; + char *R_str = R""""(raw string with +""" +as a delimiter)""""; + + std::cout << R_str << std::endl; + + return 0; +} diff --git a/tests/examplefiles/fibonacci.tokigun.aheui b/tests/examplefiles/fibonacci.tokigun.aheui new file mode 100644 index 00000000..afa2ca05 --- /dev/null +++ b/tests/examplefiles/fibonacci.tokigun.aheui @@ -0,0 +1,4 @@ +바싹반박나싼순 +뿌멓떠벌번멍뻐 +쌀삭쌀살다순옭 +어어선썬설썩옭 diff --git a/tests/examplefiles/flatline_example b/tests/examplefiles/flatline_example new file mode 100644 index 00000000..5ea73408 --- /dev/null +++ b/tests/examplefiles/flatline_example @@ -0,0 +1,186 @@ +(field "another field" 2) +(f "000001" -2) + +(missing? "a field" 23) + +(random-value "age") +(weighted-random-value "000001") + +(if (missing? "00000") (random-value "000000") (f "000000")) + +(ensure-value "000000") +(ensure-weighted-value "000000") + +(normalize "000001") +(normalize "length" 8 23) + +(z-score "a numeric field") +(z-score 23) + +(field-prop string "00023" name) +(field-prop numeric "00023" summary missing_count) + +(category-count "species" "Iris-versicolor") +(category-count "species" (f "000004")) +(bin-count "age" (f "bin-selector")) +(bin-center "000003" 3) +(bin-center (field "field-selector") 4) + +(let (v (f "age")) + (cond (< v 2) "baby" + (< v 10) "child" + (< v 20) "teenager" + "adult")) + +(segment-label "000000" "baby" 2 "child" 10 "teenager" 20 "adult") +(segment-label 0 "1st fourth" "2nd fourth" "3rd fourth" "4th fourth") + +(let (max (maximum 0) + min (minimum 0) + step (/ (- max min) 4)) + (segment-label 0 "1st fourth" (+ min step) + "2nd fourth" (+ min step step) + "3rd fourth" (+ min step step step) + "4th fourth")) + +(contains-items? "000000" "blue" "green" "darkblue") + +(<= (percentile "age" 0.5) (f "age") (percentile "age" 0.95)) + +(within-percentiles? "age" 0.5 0.95) + +(percentile-label "000023" "1st" "2nd" "3rd" "4th") + +(cond (within-percentiles? "000023" 0 0.25) "1st" + (within-percentiles? "000023" 0.25 0.5) "2nd" + (within-percentiles? "000023" 0.5 0.75) "3rd" + "4th") + +(str 1 "hello " (field "a")) +(str "value_" (+ 3 4) "/" (name "000001")) + +(length "abc") +(length "") + +(levenshtein (f 0) "a random string") +(if (< (levenshtein (f 0) "bluething") 5) "bluething" (f 0)) + +(occurrences "howdy woman, howdy" "howdy") +(occurrences "howdy woman" "Man" true) +(occurrences "howdy man" "Man" true) +(occurrences "hola, Holas" "hola" true "es") + +(md5 "a text") +(sha1 "a text") +(sha256 "") + +(matches? (field "name") ".*\\sHal\\s.*") +(matches? (field "name") "(?i).*\\shal\\s.*") + +(if (matches? (f "result") (re-quote (f "target"))) "GOOD" "MISS") +(matches? (f "name") (str "^" (re-quote (f "salutation")) "\\s *$")) + +(replace "Almost Pig Latin" "\\b(\\w)(\\w+)\\b" "$2$1ay") +(replace-first "swap first two words" "(\\w+)(\\s+)(\\w+)" "$3$2$1") + +(language "this is an English phrase") + +(< (field 0) (field 1)) +(<= (field 0 -1) (field 0) (field 0 1)) +(> (field "date") "07-14-1969") +(>= 23 (f "000004" -2)) + +(= "Dante" (field "Author")) +(= 1300 (field "Year")) +(= (field "Year" -2) (field "Year" -1) (field "Year")) +(!= (field "00033" -1) (field "00033" 1)) + +(and (= 3 (field 1)) (= "meh" (f "a")) (< (f "pregnancies") 5)) +(not true) + +(linear-regression 1 1 2 2 3 3 4 4) +(linear-regression 2.0 3.1 2.3 3.3 24.3 45.2) + +(epoch-fields (f "milliseconds")) +(epoch-year (* 1000 (f "seconds"))) + +(/ (f "a-datetime-string") 1000) +(/ (epoch (f "a-datetime-string")) 1000) + +(epoch-fields (epoch "1969-14-07T06:00:12")) +(epoch-hour (epoch "11~22~30" "hh~mm~ss")) + +(let (x (+ (window "a" -10 10)) + a (/ (* x 3) 4.34) + y (if (< a 10) "Good" "Bad")) + (list x (str (f 10) "-" y) a y)) + +(list (let (z (f 0)) (* 2 (* z z) (log z))) + (let (pi 3.141592653589793 r (f "radius")) (* 4 pi r r))) + +(if (< (field "age") 18) "non-adult" "adult") + +(if (= "oh" (field "000000")) "OH") + +(if (> (field "000001") (mean "000001")) + "above average" + (if (< (field "000001") (mean "000001")) + "below average" + "mediocre")) + +(cond (> (f "000001") (mean "000001")) "above average" + (= (f "000001") (mean "000001")) "below average" + "mediocre") + +(cond (or (= "a" (f 0)) (= "a+" (f 0))) 1 + (or (= "b" (f 0)) (= "b+" (f 0))) 0 + (or (= "c" (f 0)) (= "c+" (f 0))) -1) + +(cond (< (f "age") 2) "baby" + (and (<= 2 (f "age") 10) (= "F" (f "sex"))) "girl" + (and (<= 2 (f "age") 10) (= "M" (f "sex"))) "boy" + (< 10 (f "age") 20) "teenager" + "adult") + +(list (field "age") + (field "weight" -1) + (population "age")) + +(list 1.23 + (if (< (field "age") 10) "child" "adult") + (field 3)) + +(head (cons x lst)) +(tail (cons x lst)) + +(count (list (f 1) (f 2))) +(mode (list a b b c b a c c c)) +(max (list -1 2 -2 0.38)) +(min (list -1.3 2 1)) +(avg (list -1 -2 1 2 0.8 -0.8)) + +(in 3 (1 2 3 2)) +(in "abc" (1 2 3)) +(in (f "size") ("X" "XXL")) + +(< _ 3) +(+ (f "000001" _) 3) +(< -18 _ (f 3)) + +(map (* 2 _) (list (f 0 -1) (f 0) (f 0 1))) + +(all-but "id" "000023") +(fields "000003" 3 "a field" "another" "0002a3b-3") + +(all-with-defaults "species" "Iris-versicolor" + "petal-width" 2.8 + "000002" 0) + +(all-with-numeric-default "median") +(all-with-numeric-default 0) + +(window "000001" -1 2) +(filter (< _ 99.9) (map (+ 32 (* 1.8 _)) (window "Temp" -2 0))) + +(let (now (f "epoch")) + (avg (cond-window "temperature" (< (- (f "epoch") now) 240)))) diff --git a/tests/examplefiles/guidance.smv b/tests/examplefiles/guidance.smv new file mode 100644 index 00000000..671d1e1c --- /dev/null +++ b/tests/examplefiles/guidance.smv @@ -0,0 +1,1124 @@ +-- +-- Shuttle Digital Autopilot +-- by Sergey Berezin (berez@cs.cmu.edu) +-- +MODULE cont_3eo_mode_select(start,smode5,vel,q_bar,apogee_alt_LT_alt_ref, + h_dot_LT_hdot_reg2,alpha_n_GRT_alpha_reg2, + delta_r_GRT_del_r_usp,v_horiz_dnrng_LT_0, + high_rate_sep,meco_confirmed) + +VAR cont_3EO_start: boolean; + RTLS_abort_declared: boolean; + region_selected : boolean; + m_mode: {mm102, mm103, mm601}; + r: {reg-1, reg0, reg1, reg2, reg3, reg102}; + step : {1,2,3,4,5,6,7,8,9,10, exit, undef}; + +ASSIGN + init(cont_3EO_start) := FALSE; + init(m_mode) := {mm102, mm103}; + init(region_selected) := FALSE; + init(RTLS_abort_declared) := FALSE; + init(r) := reg-1; + init(step) := undef; + + next(step) := + case + step = 1 & m_mode = mm102 : exit; + step = 1 : 2; + step = 2 & smode5 : 5; + step = 2 & vel = GRT_vi_3eo_max: exit; + step = 2 : 3; + step = 3 & vel = LEQ_vi_3eo_min : 6; + step = 3 : 4; + step = 4 & apogee_alt_LT_alt_ref: exit; + step = 4 : 6; + step = 5 : 6; + step = 6 & r = reg0 : exit; + step = 6 : 7; + step = 7 : 8; + step = 8 & q_bar = GRT_qbar_reg3 & !high_rate_sep : 10; + step = 8 : 9; + step = 9 : 10; + step = 10: exit; + next(start): 1; + step = exit : undef; + TRUE: step; + esac; + + next(cont_3EO_start) := + case + step = 1 & m_mode = mm102 : TRUE; + step = 10 & meco_confirmed : TRUE; + TRUE : cont_3EO_start; + esac; + + next(r) := + case + step = 1 & m_mode = mm102 : reg102; + step = 2 & !smode5 & vel = GRT_vi_3eo_max: reg0; + step = 4 & apogee_alt_LT_alt_ref: reg0; + step = 5 & v_horiz_dnrng_LT_0 & delta_r_GRT_del_r_usp : reg0; + step = 8 & q_bar = GRT_qbar_reg3 & !high_rate_sep : reg3; + step = 9: case + (h_dot_LT_hdot_reg2 & alpha_n_GRT_alpha_reg2 & + q_bar = GRT_qbar_reg1) | high_rate_sep : reg2; + TRUE : reg1; + esac; + next(step) = 1 : reg-1; + TRUE: r; + esac; + + next(RTLS_abort_declared) := + case + step = 10 & meco_confirmed & m_mode = mm103 : TRUE; + TRUE: RTLS_abort_declared; + esac; + + next(m_mode) := + case + step = 10 & meco_confirmed & m_mode = mm103 : mm601; + TRUE: m_mode; + esac; + + next(region_selected) := + case + next(step) = 1 : FALSE; + next(step) = exit : TRUE; + TRUE : region_selected; + esac; + +MODULE cont_3eo_guide(start,cont_3EO_start, mode_select_completed, et_sep_cmd, + h_dot_LT_0, q_bar_a_GRT_qbar_max_sep, m_mode, r0, + cont_minus_z_compl, t_nav-t_et_sep_GRT_dt_min_z_102, + ABS_q_orb_GRT_q_minus_z_max, ABS_r_orb_GRT_r_minus_z_max, + excess_OMS_propellant, q_bar_a_LT_qbar_oms_dump, + entry_mnvr_couter_LE_0, rcs_all_jet_inhibit, + alt_GRT_alt_min_102_dump, t_nav-t_gmtlo_LT_t_dmp_last, + pre_sep, cond_18, q_orb_LT_0, ABS_alf_err_LT_alf_sep_err, + cond_20b, cond_21, ABS_beta_n_GRT_beta_max, cond_24, cond_26, + cond_27, cond_29, mm602_OK) +VAR + step: {1,a1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20, + b20, c20, d20, 21,22,23,24,25,26,27,28,29,exit, undef}; + call_RTLS_abort_task : boolean; + first3: boolean; -- indicates if it is the first pass + first8: boolean; + first27: boolean; + s_unconv : boolean; + mode_2_indicator : boolean; + et_sep_man_initiate : boolean; + emerg_sep : boolean; + cont_3eo_pr_delay : {minus_z_reg1, minus_z_reg2, + minus_z_reg3, minus_z_reg4, minus_z_reg102, 0, 5}; + etsep_y_drift : {undef, minus_z_reg1, minus_z_reg2, + minus_z_reg3, minus_z_reg4, minus_z_reg102, 0}; + fwd_rcs_dump_enable : boolean; + fcs_accept_icnct : boolean; + oms_rcs_i_c_inh_ena_cmd : boolean; + orbiter_dump_ena : boolean; + frz_3eo : boolean; + high_rate_sep: boolean; + entry_gains : boolean; + cont_sep_cplt : boolean; + pch_cmd_reg4 : boolean; + alpha_ok : boolean; + r : {reg-1, reg0, reg1, reg2, reg3, reg4, reg102}; + early_sep : boolean; +-------------------------------------------- +----- Additional Variables ----------------- +-------------------------------------------- + rtls_lo_f_d_delay : {undef, 0}; + wcb2 : {undef, reg1_0, reg2_neg4, wcb2_3eo, reg4_0, + reg102_undef, post_sep_0}; + q_gcb_i : {undef, quat_reg1, quat_reg2, quat_reg3, quat_reg4, + quat_reg102_undef, quat_entry_M50_to_cmdbody}; + oms_nz_lim : {undef, oms_nz_lim_3eo, oms_nz_lim_iload, oms_nz_lim_std}; + contingency_nz_lim : {undef, contingency_nz_lim_3eo, + contingency_nz_lim_iload, contingency_nz_lim_std}; + + + +ASSIGN + init(entry_gains) := FALSE; + init(frz_3eo) := FALSE; + init(cont_3eo_pr_delay) := 5; + init(etsep_y_drift) := undef; + init(r) := reg-1; + init(step) := undef; + init(call_RTLS_abort_task) := FALSE; + init(first3) := TRUE; + init(first8) := TRUE; + init(first27) := TRUE; + init(cont_sep_cplt) := FALSE; + init(et_sep_man_initiate) := FALSE; + init(alpha_ok) := FALSE; + init(pch_cmd_reg4) := FALSE; + +-- Assumed initializations: + + init(rtls_lo_f_d_delay) := undef; + init(wcb2) := undef; + init(q_gcb_i) := undef; + init(oms_nz_lim) := undef; + init(contingency_nz_lim) := undef; + init(oms_rcs_i_c_inh_ena_cmd) := FALSE; + init(orbiter_dump_ena) := FALSE; +-- init(early_sep) := FALSE; + +------------- + + next(step) := nextstep; + + next(r) := + case + step = a1 & (cont_3EO_start | mode_select_completed) : r0; + step = 21 & cond_21 : reg4; + step = 23 & ABS_beta_n_GRT_beta_max & !high_rate_sep : reg1; + TRUE : r; + esac; + + next(first3) := + case + step = 3 & cont_3EO_start : FALSE; + TRUE : first3; + esac; + + next(first8) := + case + step = 8 & excess_OMS_propellant & cont_3EO_start : FALSE; + TRUE : first8; + esac; + + next(first27) := + case + step = 27 : FALSE; + TRUE: first27; + esac; + + next(s_unconv) := + case + step = 3 : FALSE; + TRUE : s_unconv; + esac; + + next(call_RTLS_abort_task) := + case + step = 3 : TRUE; + TRUE : call_RTLS_abort_task; + esac; + + next(mode_2_indicator) := + case + step = 4 : TRUE; + TRUE : mode_2_indicator; + esac; + + next(et_sep_man_initiate) := + case + step = 5 & h_dot_LT_0 & q_bar_a_GRT_qbar_max_sep & m_mode != mm102 : TRUE; + step = 14 & pre_sep : TRUE; + step = 19 & q_orb_LT_0 : TRUE; + step = d20 : TRUE; + step = 26 & cond_26 : TRUE; + step = 29 & cond_29 : TRUE; + TRUE : et_sep_man_initiate; + esac; + + next(emerg_sep) := + case + next(step) = 1 : FALSE; + step = 5 & h_dot_LT_0 & q_bar_a_GRT_qbar_max_sep & m_mode != mm102: TRUE; + TRUE : emerg_sep; + esac; + + next(cont_3eo_pr_delay) := + case + next(step) = 1 : 5; + step = 5 & h_dot_LT_0 & q_bar_a_GRT_qbar_max_sep & m_mode != mm102 : + minus_z_reg3; + step = 7 & !cont_minus_z_compl & r = reg102 & + t_nav-t_et_sep_GRT_dt_min_z_102 & + (ABS_q_orb_GRT_q_minus_z_max | ABS_r_orb_GRT_r_minus_z_max) : 0; + step = 14 & pre_sep : minus_z_reg102; + step = 19 & q_orb_LT_0 : minus_z_reg4; + step = d20 : minus_z_reg3; + step = 26 & cond_26 : minus_z_reg2; + step = 27 & first27 : minus_z_reg1; + TRUE : cont_3eo_pr_delay; + esac; + + next(etsep_y_drift) := + case + step = 5 & h_dot_LT_0 & q_bar_a_GRT_qbar_max_sep & m_mode != mm102 : + minus_z_reg3; + step = 7 & !cont_minus_z_compl & r = reg102 & + t_nav-t_et_sep_GRT_dt_min_z_102 & + (ABS_q_orb_GRT_q_minus_z_max | ABS_r_orb_GRT_r_minus_z_max) : 0; + step = 14 & pre_sep : minus_z_reg102; + step = 19 & q_orb_LT_0 : minus_z_reg4; + step = d20 : minus_z_reg3; + step = 26 & cond_26 : minus_z_reg2; + step = 27 & first27 : minus_z_reg1; + TRUE : etsep_y_drift; + esac; + + next(fwd_rcs_dump_enable) := + case + step = 8 & excess_OMS_propellant & first8 : FALSE; + TRUE : fwd_rcs_dump_enable; + esac; + + next(fcs_accept_icnct) := + case + step = 9 & q_bar_a_LT_qbar_oms_dump & r != reg102 : TRUE; + TRUE : fcs_accept_icnct; + esac; + + next(oms_rcs_i_c_inh_ena_cmd) := + case +-- next(step) = 1 & oms_rcs_i_c_inh_ena_cmd : {0,1}; + next(step) = 1 & oms_rcs_i_c_inh_ena_cmd : FALSE; -- Assumed initialization + step = 9 & q_bar_a_LT_qbar_oms_dump & r != reg102 : TRUE; + TRUE : oms_rcs_i_c_inh_ena_cmd; + esac; + + next(orbiter_dump_ena) := + case + next(start) = TRUE : FALSE; -- Assumed initialization + step = 9 & q_bar_a_LT_qbar_oms_dump & r != reg102 : TRUE; + step = 13 & alt_GRT_alt_min_102_dump & t_nav-t_gmtlo_LT_t_dmp_last : TRUE; + TRUE : orbiter_dump_ena; + esac; + + next(frz_3eo) := + case + next(step) = 1 : FALSE; + step = 10 & entry_mnvr_couter_LE_0 & !rcs_all_jet_inhibit : FALSE; + step = 28 & !et_sep_man_initiate : TRUE; + TRUE : frz_3eo; + esac; + + next(high_rate_sep) := + case + step = 10 & entry_mnvr_couter_LE_0 & !rcs_all_jet_inhibit : FALSE; + step = 25 : TRUE; + TRUE : high_rate_sep; + esac; + + next(entry_gains) := + case + next(step) = 1 : FALSE; + step = 10 & entry_mnvr_couter_LE_0 & !rcs_all_jet_inhibit : TRUE; + TRUE : entry_gains; + esac; + + next(cont_sep_cplt) := + case + next(step) = 1 : FALSE; + step = 12 & mm602_OK : TRUE; + TRUE : cont_sep_cplt; + esac; + + next(pch_cmd_reg4) := + case + next(step) = 1 : FALSE; + step = 18 & !pch_cmd_reg4 & cond_18 : TRUE; + TRUE : pch_cmd_reg4; + esac; + + next(alpha_ok) := + case + next(step) = 1 : FALSE; + step = 20 & ABS_alf_err_LT_alf_sep_err : TRUE; + TRUE : alpha_ok; + esac; + + next(early_sep) := + case + step = 27 & first27 : + case + cond_27 : TRUE; + TRUE : FALSE; + esac; + TRUE : early_sep; + esac; + +-------------------------------------------- +----- Additional Variables ----------------- +-------------------------------------------- + + next(rtls_lo_f_d_delay) := + case + next(start) = TRUE : undef; -- Assumed initialization + step = 8 & first8 & excess_OMS_propellant : 0; + TRUE : rtls_lo_f_d_delay; + esac; + + next(wcb2) := + case + next(start) = TRUE : undef; -- Assumed initialization + step = 10 & entry_mnvr_couter_LE_0 : post_sep_0; + step = 12 : case + r = reg4 : reg4_0; + TRUE : wcb2_3eo; + esac; + step = 14 & pre_sep : reg102_undef; + step = 15 : case + r = reg4 : reg4_0; + TRUE : wcb2_3eo; + esac; + step = 25 : reg2_neg4; + TRUE : wcb2; + esac; + + next(q_gcb_i) := + case + next(start) = TRUE : undef; -- Assumed initialization + step = 11 : quat_entry_M50_to_cmdbody; + step = 14 & pre_sep : quat_reg102_undef; + step = 16 : case + r = reg4 : quat_reg4; + TRUE : quat_reg3; + esac; + step = 22 : quat_reg2; + +-- Without this step the value "quat_reg2" would remain in "reg1": +-- step = 23 & ABS_beta_n_GRT_beta_max & !high_rate_sep : undef; + + TRUE : q_gcb_i; + esac; + + next(oms_nz_lim) := + case + next(start) = TRUE : undef; -- Assumed initialization + step = 9 & q_bar_a_LT_qbar_oms_dump & r != reg102 : oms_nz_lim_3eo; + step = 12 & mm602_OK : oms_nz_lim_std; + TRUE : oms_nz_lim; + esac; + + next(contingency_nz_lim) := + case + next(start) = TRUE : undef; -- Assumed initialization + step = 9 & q_bar_a_LT_qbar_oms_dump & r != reg102 : + contingency_nz_lim_3eo; + step = 12 & mm602_OK : contingency_nz_lim_std; + TRUE : contingency_nz_lim; + esac; + +DEFINE + finished := step = exit; + idle := step = undef; + + start_cont_3eo_mode_select := + case + step = 1 & !cont_3EO_start : TRUE; + TRUE : FALSE; + esac; + + nextstep := + case + step = 1 : a1; + step = a1 : case + (cont_3EO_start | mode_select_completed) : 2; + TRUE : step; + esac; + step = 2 : case + !cont_3EO_start : exit; + first3 : 3; + TRUE: 4; + esac; + step = 3 : 4; + step = 4 : case + et_sep_cmd : 7; + TRUE : 5; + esac; + step = 5 : case + h_dot_LT_0 & q_bar_a_GRT_qbar_max_sep & + m_mode != mm102 : exit; + TRUE : 6; + esac; + step = 6 : + case + r = reg102 : 13; + r in {reg3, reg4} : 15; + r = reg2 : 22; + r = reg1 : 27; + TRUE : exit; + esac; + step = 7 : case + cont_minus_z_compl : 8; + TRUE : exit; + esac; + step = 8 : case + excess_OMS_propellant & first8 : 9; + TRUE : 10; + esac; + step = 9 : exit; + step = 10 : case + !entry_mnvr_couter_LE_0 | rcs_all_jet_inhibit : exit; + TRUE : 11; + esac; + step = 11 : 12; + step = 12 : exit; + step = 13 : 14; + step = 14 : exit; + step = 15 : 16; + step = 16 : 17; + step = 17 : case + r = reg4 : 18; + TRUE : 20; + esac; + step = 18 : case + pch_cmd_reg4 | cond_18 : 19; + TRUE : exit; + esac; + step = 19 : exit; + step = 20 : case + ABS_alf_err_LT_alf_sep_err : b20; + TRUE : c20; + esac; + step = b20 : case + cond_20b : d20; + TRUE : exit; + esac; + step = c20 : case + alpha_ok : d20; + TRUE : 21; + esac; + step = d20 : exit; + TRUE : nextstep21; + esac; + + nextstep21 := + case + step = 21 : case + cond_21 : 15; + TRUE : exit; + esac; + step = 22 : 23; + step = 23 : case + ABS_beta_n_GRT_beta_max & !high_rate_sep : 27; + TRUE : 24; + esac; + step = 24 : case + cond_24 | high_rate_sep : 25; + TRUE : exit; + esac; + step = 25 : 26; + step = 26 : exit; + step = 27 : 28; + step = 28 : case + !et_sep_man_initiate : 29; + TRUE : exit; + esac; + step = 29 : exit; + start : 1; + step = exit : undef; + TRUE : step; + esac; + + post_sep_mode := step in {7,8,9,10,11,12}; + +------------------------------------------------------------------ +------------------------------------------------------------------ + +MODULE main +VAR + smode5: boolean; + vel : {GRT_vi_3eo_max, GRT_vi_3eo_min, LEQ_vi_3eo_min}; + q_bar: {GRT_qbar_reg3, GRT_qbar_reg1, LEQ_qbar_reg1}; + q_bar_a_GRT_qbar_max_sep : boolean; + q_bar_a_LT_qbar_oms_dump : boolean; + apogee_alt_LT_alt_ref : boolean; + h_dot_LT_hdot_reg2 : boolean; + h_dot_LT_0 : boolean; + alpha_n_GRT_alpha_reg2 : boolean; + delta_r_GRT_del_r_usp : boolean; + v_horiz_dnrng_LT_0: boolean; + meco_confirmed: boolean; + et_sep_cmd : boolean; + cont_minus_z_compl : boolean; + t_nav-t_et_sep_GRT_dt_min_z_102 : boolean; + ABS_q_orb_GRT_q_minus_z_max : boolean; + ABS_r_orb_GRT_r_minus_z_max : boolean; + excess_OMS_propellant : boolean; + entry_mnvr_couter_LE_0 : boolean; + rcs_all_jet_inhibit : boolean; + alt_GRT_alt_min_102_dump : boolean; + t_nav-t_gmtlo_LT_t_dmp_last : boolean; + pre_sep : boolean; + cond_18 : boolean; + q_orb_LT_0 : boolean; + ABS_alf_err_LT_alf_sep_err : boolean; + cond_20b : boolean; + cond_21 : boolean; + ABS_beta_n_GRT_beta_max : boolean; + cond_24 : boolean; + cond_26 : boolean; + cond_27 : boolean; + cond_29 : boolean; + mm602_OK : boolean; + start_guide : boolean; + mated_coast_mnvr : boolean; + + cs: cont_3eo_mode_select(cg.start_cont_3eo_mode_select, + smode5,vel,q_bar,apogee_alt_LT_alt_ref, + h_dot_LT_hdot_reg2,alpha_n_GRT_alpha_reg2, + delta_r_GRT_del_r_usp,v_horiz_dnrng_LT_0, + cg.high_rate_sep,meco_confirmed); + + cg: cont_3eo_guide(start_guide, + cs.cont_3EO_start, cs.region_selected, et_sep_cmd, + h_dot_LT_0, q_bar_a_GRT_qbar_max_sep, cs.m_mode, cs.r, + cont_minus_z_compl, t_nav-t_et_sep_GRT_dt_min_z_102, + ABS_q_orb_GRT_q_minus_z_max, ABS_r_orb_GRT_r_minus_z_max, + excess_OMS_propellant, q_bar_a_LT_qbar_oms_dump, + entry_mnvr_couter_LE_0, rcs_all_jet_inhibit, + alt_GRT_alt_min_102_dump, t_nav-t_gmtlo_LT_t_dmp_last, + pre_sep, cond_18, q_orb_LT_0, ABS_alf_err_LT_alf_sep_err, + cond_20b, cond_21, ABS_beta_n_GRT_beta_max, cond_24, cond_26, + cond_27, cond_29, mm602_OK); + +ASSIGN + init(start_guide) := FALSE; + init(mated_coast_mnvr) := FALSE; + + next(entry_mnvr_couter_LE_0) := + case + !entry_mnvr_couter_LE_0 : {FALSE, TRUE}; + TRUE : TRUE; + esac; + +--------------------------------------------------------------------- +--------------------------------------------------------------------- + next(start_guide) := + case + start_guide : FALSE; + !cg.idle : FALSE; + TRUE : {FALSE, TRUE}; + esac; + + next(smode5) := + case + fixed_values : smode5; + cg.idle : { FALSE, TRUE }; + TRUE : smode5; + esac; + + next(vel) := + case + fixed_values : vel; + cg.idle : {GRT_vi_3eo_max, GRT_vi_3eo_min, LEQ_vi_3eo_min}; + TRUE : vel; + esac; + + next(q_bar) := + case + fixed_values : q_bar; + cg.idle : {GRT_qbar_reg3, GRT_qbar_reg1, LEQ_qbar_reg1}; + TRUE : q_bar; + esac; + + next(q_bar_a_GRT_qbar_max_sep) := + case + fixed_values : q_bar_a_GRT_qbar_max_sep; + cg.idle : { FALSE, TRUE }; + TRUE : q_bar_a_GRT_qbar_max_sep; + esac; + + next(apogee_alt_LT_alt_ref) := + case + fixed_values : apogee_alt_LT_alt_ref; + cg.idle : { FALSE, TRUE }; + TRUE : apogee_alt_LT_alt_ref; + esac; + + next(h_dot_LT_hdot_reg2) := + case + fixed_values : h_dot_LT_hdot_reg2; + cg.idle : { FALSE, TRUE }; + TRUE : h_dot_LT_hdot_reg2; + esac; + + next(h_dot_LT_0) := + case + fixed_values : h_dot_LT_0; + cg.idle : { FALSE, TRUE }; + TRUE : h_dot_LT_0; + esac; + + next(alpha_n_GRT_alpha_reg2) := + case + fixed_values : alpha_n_GRT_alpha_reg2; + cg.idle : { FALSE, TRUE }; + TRUE : alpha_n_GRT_alpha_reg2; + esac; + + next(delta_r_GRT_del_r_usp) := + case + fixed_values : delta_r_GRT_del_r_usp; + cg.idle : { FALSE, TRUE }; + TRUE : delta_r_GRT_del_r_usp; + esac; + + next(v_horiz_dnrng_LT_0) := + case + fixed_values : v_horiz_dnrng_LT_0; + cg.idle : { FALSE, TRUE }; + TRUE : v_horiz_dnrng_LT_0; + esac; + + next(meco_confirmed) := + case + fixed_values : meco_confirmed; + meco_confirmed : TRUE; + cg.idle : { FALSE, TRUE }; + TRUE : meco_confirmed; + esac; + + next(et_sep_cmd) := + case + fixed_values : et_sep_cmd; + et_sep_cmd : TRUE; + cg.idle : { FALSE, TRUE }; + TRUE : et_sep_cmd; + esac; + + next(cont_minus_z_compl) := + case + fixed_values : cont_minus_z_compl; + cg.idle : { FALSE, TRUE }; + TRUE : cont_minus_z_compl; + esac; + + next(t_nav-t_et_sep_GRT_dt_min_z_102) := + case + fixed_values : t_nav-t_et_sep_GRT_dt_min_z_102; + cg.idle : { FALSE, TRUE }; + TRUE : t_nav-t_et_sep_GRT_dt_min_z_102; + esac; + + next(ABS_q_orb_GRT_q_minus_z_max) := + case + fixed_values : ABS_q_orb_GRT_q_minus_z_max; + cg.idle : { FALSE, TRUE }; + TRUE : ABS_q_orb_GRT_q_minus_z_max; + esac; + + next(ABS_r_orb_GRT_r_minus_z_max) := + case + fixed_values : ABS_r_orb_GRT_r_minus_z_max; + cg.idle : { FALSE, TRUE }; + TRUE : ABS_r_orb_GRT_r_minus_z_max; + esac; + + next(excess_OMS_propellant) := + case + fixed_values : excess_OMS_propellant; + cg.idle & excess_OMS_propellant : { FALSE, TRUE }; + TRUE : excess_OMS_propellant; + esac; + + next(q_bar_a_LT_qbar_oms_dump) := + case + fixed_values : q_bar_a_LT_qbar_oms_dump; + cg.idle : { FALSE, TRUE }; + TRUE : q_bar_a_LT_qbar_oms_dump; + esac; + + next(rcs_all_jet_inhibit) := + case + fixed_values : rcs_all_jet_inhibit; + cg.idle : { FALSE, TRUE }; + TRUE : rcs_all_jet_inhibit; + esac; + + next(alt_GRT_alt_min_102_dump) := + case + fixed_values : alt_GRT_alt_min_102_dump; + cg.idle : { FALSE, TRUE }; + TRUE : alt_GRT_alt_min_102_dump; + esac; + + next(t_nav-t_gmtlo_LT_t_dmp_last) := + case + fixed_values : t_nav-t_gmtlo_LT_t_dmp_last; + cg.idle : { FALSE, TRUE }; + TRUE : t_nav-t_gmtlo_LT_t_dmp_last; + esac; + + next(pre_sep) := + case + fixed_values : pre_sep; + cg.idle : { FALSE, TRUE }; + TRUE : pre_sep; + esac; + + next(cond_18) := + case + fixed_values : cond_18; + cg.idle : { FALSE, TRUE }; + TRUE : cond_18; + esac; + + next(q_orb_LT_0) := + case + fixed_values : q_orb_LT_0; + cg.idle : { FALSE, TRUE }; + TRUE : q_orb_LT_0; + esac; + + next(ABS_alf_err_LT_alf_sep_err) := + case + fixed_values : ABS_alf_err_LT_alf_sep_err; + cg.idle : { FALSE, TRUE }; + TRUE : ABS_alf_err_LT_alf_sep_err; + esac; + + next(cond_20b) := + case + fixed_values : cond_20b; + cg.idle : { FALSE, TRUE }; + TRUE : cond_20b; + esac; + + next(cond_21) := + case + fixed_values : cond_21; + cg.idle : { FALSE, TRUE }; + TRUE : cond_21; + esac; + + next(ABS_beta_n_GRT_beta_max) := + case + fixed_values : ABS_beta_n_GRT_beta_max; + cg.idle : { FALSE, TRUE }; + TRUE : ABS_beta_n_GRT_beta_max; + esac; + + next(cond_24) := + case + fixed_values : cond_24; + cg.idle : { FALSE, TRUE }; + TRUE : cond_24; + esac; + + next(cond_26) := + case + fixed_values : cond_26; + cg.idle : { FALSE, TRUE }; + TRUE : cond_26; + esac; + + next(cond_27) := + case + fixed_values : cond_27; + cg.idle : { FALSE, TRUE }; + TRUE : cond_27; + esac; + + next(cond_29) := + case + fixed_values : cond_29; + cg.idle : { FALSE, TRUE }; + TRUE : cond_29; + esac; + + next(mm602_OK) := + case + fixed_values : mm602_OK; + cg.idle : { FALSE, TRUE }; + TRUE : mm602_OK; + esac; + + next(mated_coast_mnvr) := + case + next(cg.step) = 1 : FALSE; + cg.step = 6 & cg.r in {reg1, reg2, reg3, reg4, reg102} : TRUE; + TRUE : mated_coast_mnvr; + esac; + +--------------------------------------------------------------------- +--------------------------------------------------------------------- +DEFINE + fixed_values := FALSE; + + output_ok := + case + cg.q_gcb_i = undef | cg.wcb2 = undef | + cg.cont_3eo_pr_delay = 5 | + cg.etsep_y_drift = undef : + case + !mated_coast_mnvr: 1; + TRUE : undef; + esac; + !mated_coast_mnvr: toint(cg.q_gcb_i = quat_entry_M50_to_cmdbody & + cg.wcb2 = post_sep_0); +-- reg1 never happens? +-- cg.r = reg1 : (cg.q_gcb_i = quat_reg1 & cg.wcb2 = reg1_0 & +-- cg.cont_3eo_pr_delay = minus_z_reg1 & +-- cg.etsep_y_drift = minus_z_reg1) | cg.emerg_sep; + cg.r = reg2 : toint((cg.q_gcb_i = quat_reg2 & cg.wcb2 = reg2_neg4 & + cg.cont_3eo_pr_delay = minus_z_reg2 & + cg.etsep_y_drift = minus_z_reg2) | cg.emerg_sep); + + cg.r = reg3 : toint((cg.q_gcb_i = quat_reg3 & cg.wcb2 = wcb2_3eo & + cg.cont_3eo_pr_delay = minus_z_reg3 & + cg.etsep_y_drift = minus_z_reg3) | cg.emerg_sep); + cg.r = reg4 : toint((cg.q_gcb_i = quat_reg4 & cg.wcb2 = reg4_0 & + cg.cont_3eo_pr_delay = minus_z_reg4 & + cg.etsep_y_drift = minus_z_reg4) | cg.emerg_sep); + cg.r = reg102 : toint((cg.q_gcb_i = quat_reg102_undef & + cg.wcb2 = reg102_undef & + cg.cont_3eo_pr_delay = minus_z_reg102 & + cg.etsep_y_drift = minus_z_reg102) | cg.emerg_sep); + TRUE : 0; + esac; + +--------------------------------------------------------------------- +-------- Specifications --------------------------------------------- +--------------------------------------------------------------------- + +-- Contingency Guide terminates + +SPEC AG(!cg.idle -> AF(cg.finished)) + +-- Contingency guide can be executed infinitely often + +SPEC AG( (cg.idle | cg.finished) -> + EF(!(cg.idle | cg.finished) & EF(cg.finished))) + +-- Contingency mode select task works fine + +SPEC AG(cs.cont_3EO_start & cs.region_selected -> + ((cs.m_mode = mm102 | meco_confirmed) & + cs.r != reg-1 & cs.r != reg0)) + +-- Bad (initial) value never happens again once region is computed +-- unless we restart the task + +--SPEC AG(cs.r != reg-1 -> !E[!cg.start_cont_3eo_mode_select U +-- cs.r = reg-1 & !cg.start_cont_3eo_mode_select]) + +-- Comment out each of the regions and see if this is still true +-- (Check, if ALL of the regions can happen) + +--SPEC AG(cs.r in {reg-1 +-- ,reg0 +-- ,reg1 +-- ,reg2 +-- ,reg3 +-- ,reg102 +-- }) + +-- Comment out each of the regions and see if this is still true +-- (Check, if ALL of the regions can happen) + +--SPEC AG(cg.r in {reg-1 +-- ,reg0 +-- ,reg1 +-- ,reg2 +-- ,reg3 +-- ,reg4 +-- ,reg102 +-- }) + +-- Mode_select starts at the next step after its "start" bit is set: + +--SPEC AG(!cg.start_cont_3eo_mode_select -> +-- AX(cg.start_cont_3eo_mode_select & cs.step in {exit, undef} -> +-- AX(cs.step = 1 & !cs.region_selected))) + +-- During major mode 103, the inertial velocity is monitored. +-- Below an I-loaded velocity, a MECO would constitute a contingency +-- abort. (Must NOT be in SMODE=5 (??)) + +SPEC AG(cg.start_cont_3eo_mode_select & cs.m_mode = mm103 & + vel = LEQ_vi_3eo_min & meco_confirmed & !smode5 -> + A[!cs.region_selected U cs.region_selected & cs.cont_3EO_start]) + +-- Above a certain inertial velocity (in mode 103), the 3E/O field +-- is blanked, indicating that a MECO at this point would not require +-- an OPS 6 contingency abort. + +SPEC AG(cs.region_selected -> + (cs.m_mode = mm103 & vel = GRT_vi_3eo_max -> !cs.cont_3EO_start)) + +-- Between the two velocities, an apogee altitude - velocity curve is +-- constructed based on the current inertial velocity. If the apogee +-- altitude is above this curve, a contingency abort capability is +-- still required and a 3E/O region index will be calculated. +-- Otherwise, the 3E/O field is blanked out and no further contingency +-- abort calculations will be performed. (Must NOT be in SMODE=5 (??)) + +SPEC AG(cg.start_cont_3eo_mode_select & cs.m_mode = mm103 & + vel = GRT_vi_3eo_min & meco_confirmed & !smode5 -> + A[!cs.region_selected U cs.region_selected & + apogee_alt_LT_alt_ref = !cs.cont_3EO_start]) + +-- For an RTLS trajectory (SMODE=5), a check is made on the downrange +-- velocity to see if the vehicle is heading away from the landing site. +-- If this is the case, a 3E/O region index is calculated. If the vehicle +-- is heading back to the landing site, and the current range to the MECO +-- R-V line is greater than an I-loaded value, a 3E/O region index is +-- calculated. Otherwise, an intact abort is possible and the 3E/O field +-- is blanked. + +SPEC AG(cg.start_cont_3eo_mode_select & smode5 & meco_confirmed & + (!v_horiz_dnrng_LT_0 | !delta_r_GRT_del_r_usp) -> + A[!cs.region_selected U cs.region_selected & cs.cont_3EO_start]) + +-- If this task is called prior to SRB separation [mm102], the 3E/O region +-- index is set to 102 and the 3E/O contingency flag is set. + +SPEC AG(cs.m_mode = mm102 & cg.start_cont_3eo_mode_select -> + AX (A [ !cs.region_selected U cs.region_selected & + cs.r = reg102 & cs.cont_3EO_start])) + +-- After SRB separation, on every pass that the 3E/O region index is +-- calculated, a check is made to see if MECO confirmed has occured. If +-- so, a check is made to see if the major mode is 103. If so, an RTLS is +-- automatically invoked to transition to major mode 601. + +SPEC AG(!cs.region_selected & cs.m_mode = mm103 & meco_confirmed -> + A[!cs.region_selected U cs.region_selected & cs.r != reg0 -> + cs.m_mode = mm601 & cs.RTLS_abort_declared]) + +-- Once the 3E/O contingency flag has been set, this task is no longer +-- executed. + +SPEC AG(cs.cont_3EO_start -> AG(!cg.start_cont_3eo_mode_select)) + +-- If MECO confirmed occurs in MM103 and an OPS 6 contingency abort +-- procedure is still required, contingency 3E/O guidance sets the +-- CONT_3EO_START flag ON. Contingency 3E/O guidance then switches +-- from its display support function into an actual auto guidance +-- steering process. [...] Contingency 3E/O guidance sets the RTLS abort +-- declared flag and the MSC performs the transition from from major mode +-- 103 to 601. + +SPEC AG(!cg.idle & !cg.finished & !cs.region_selected & cs.m_mode = mm103 -> + A[ !cg.finished U cg.finished & cs.region_selected & + (cs.cont_3EO_start -> cs.m_mode = mm601 & cs.RTLS_abort_declared) ]) + +-- If MECO confirmed occurs in a major mode 601 and a contingency abort +-- procedure is still required, contingency 3E/O guidance sets the +-- CONT_3EO_START flag ON. [...] Contingency 3E/O guidance then commands +-- 3E/O auto maneuvers in major mode 601. [What are these maneuvers??] + +SPEC AG(cg.finished & cs.m_mode = mm601 & !et_sep_cmd & + meco_confirmed & cs.cont_3EO_start -> + cg.q_gcb_i in {quat_reg1, quat_reg2, quat_reg3, quat_reg4, undef} + | cg.emerg_sep) + +-- If MECO confirmed occurs in a first stage (MM102) [...], contingency +-- 3E/O guidance will command a fast ET separation during SRB tailoff in +-- major mode 102. CONT 3E/O GUID will then command maneuver post-sep in +-- MM601 (???). [ I'm not sure what indicates fast ET sep.: emerg_sep or +-- early_sep, or what? ] + +SPEC AG(cg.finished & cs.m_mode = mm102 & meco_confirmed & pre_sep -> + cg.emerg_sep | et_sep_cmd + | cg.et_sep_man_initiate + | cg.early_sep + ) + +--------------------------------------------- +-- Invariants from Murphi code -------------- +--------------------------------------------- + +--SPEC AG(cg.finished -> (output_ok != 0 | (output_ok = undef & +-- (cg.emerg_sep | !cg.cont_sep_cplt)))) + +--SPEC AG(!cg.finished & !cg.idle -> !mated_coast_mnvr | !et_sep_cmd) + +-- Stronger version !!! + +SPEC AG(cg.finished -> output_ok != 0) + +-- Contingency Guidance shall command an ET separation +-- [under certain conditions :-]. + +SPEC AG(cs.cont_3EO_start & cg.finished & + (cg.r = reg1 -> cond_29) & + (cg.r = reg2 -> cond_24 & cond_26) & + (cg.r = reg3 -> cg.alpha_ok & + (ABS_alf_err_LT_alf_sep_err -> cond_20b)) & + (cg.r = reg4 -> cond_18 & q_orb_LT_0) & + (cg.r = reg102 -> pre_sep) -> + et_sep_cmd | cg.et_sep_man_initiate + | cg.early_sep + | cg.emerg_sep + ) + +-- Contingency Guidance shall command at most one interconnected OMS dump. + +SPEC AG(cg.finished & cg.oms_rcs_i_c_inh_ena_cmd -> + AG(!cg.oms_rcs_i_c_inh_ena_cmd -> AG(!cg.oms_rcs_i_c_inh_ena_cmd))) + +-- Contingency Guidance shall command a transition to glide RTLS +-- (flight mode 602) + +SPEC AG(cg.finished & cs.m_mode = mm601 -> + --cg.cont_sep_cplt | cg.emerg_sep | + cg.call_RTLS_abort_task) + +-- Paper, p. 28, unstated assumption 2: at step 6 the region is +-- among 102, 1-4. + +SPEC AG(cg.step = 6 -> cg.r in {reg102, reg1, reg2, reg3, reg4}) + +-- The transition to mode 602 shall not occur until the entry maneuver +-- has been calculated + +SPEC !E[cg.q_gcb_i = undef U cg.cont_sep_cplt & cg.q_gcb_i = undef] + +-- The entry maneuver calculations shall not commence until the OMS/RCS +-- interconnect, if any, is complete (??? What does it exactly mean???) +-- !!! +--SPEC AG(cg.oms_rcs_i_c_inh_ena_cmd -> +-- !E[cg.oms_rcs_i_c_inh_ena_cmd U +-- cg.q_gcb_i != undef & cg.oms_rcs_i_c_inh_ena_cmd]) + +SPEC AG(cg.oms_rcs_i_c_inh_ena_cmd -> + !E[rcs_all_jet_inhibit U + cg.q_gcb_i != undef & rcs_all_jet_inhibit]) + +-- The OMS dump shall not be considered until the -Z translation is complete. + +SPEC !E[!cont_minus_z_compl & cg.r != reg102 U cg.orbiter_dump_ena] + +-- Completion of -Z translation shall not be checked until ET separation +-- has been commanded + +SPEC !E[!et_sep_cmd U cg.step = 7] + +-- ET separation shall be commanded if and only if an abort maneuver +-- region is assigned [and again there are *certain conditions*]. + +SPEC AG(cg.finished & cs.cont_3EO_start & + (cg.r = reg1 -> cond_29) & + (cg.r = reg2 -> cond_24 & cond_26) & + (cg.r = reg3 -> cg.alpha_ok & + (ABS_alf_err_LT_alf_sep_err -> cond_20b)) & + (cg.r = reg4 -> cond_18 & q_orb_LT_0) & + (cg.r = reg102 -> pre_sep) -> + (cg.et_sep_man_initiate | et_sep_cmd + <-> cg.r in {reg1, reg2, reg3, reg4, reg102})) + +-- The assigned region can not change arbitrarily. + +-- Regions 1 and 2 may interchange, but will not switch to any other region: + +SPEC AG(cg.finished & cs.cont_3EO_start & cg.r in {reg1,reg2} -> + AG(cg.finished -> cg.r in {reg1,reg2})) + +-- Regions 3 and 4 may interchange, but will not switch to any other region: + +SPEC AG(cg.finished & cs.cont_3EO_start & cg.r in {reg3,reg4} -> + AG(cg.finished -> cg.r in {reg3,reg4})) + +-- Region 102 never changes: + +SPEC AG(cg.finished & cg.r = reg102 -> AG(cg.finished -> cg.r = reg102)) diff --git a/tests/examplefiles/hello-world.puzzlet.aheui b/tests/examplefiles/hello-world.puzzlet.aheui new file mode 100644 index 00000000..e7ef3a62 --- /dev/null +++ b/tests/examplefiles/hello-world.puzzlet.aheui @@ -0,0 +1,10 @@ +밤밣따빠밣밟따뿌 +빠맣파빨받밤뚜뭏 +돋밬탕빠맣붏두붇 +볻뫃박발뚷투뭏붖 +뫃도뫃희멓뭏뭏붘 +뫃봌토범더벌뿌뚜 +뽑뽀멓멓더벓뻐뚠 +뽀덩벐멓뻐덕더벅 + +https://github.com/aheui/snippets/blob/master/hello-world/hello-world.puzzlet.aheui diff --git a/tests/examplefiles/plain.bst b/tests/examplefiles/plain.bst new file mode 100644 index 00000000..7adf4bb0 --- /dev/null +++ b/tests/examplefiles/plain.bst @@ -0,0 +1,1097 @@ +% BibTeX standard bibliography style `plain' + % Version 0.99b (8-Dec-10 release) for BibTeX versions 0.99a or later. + % Copyright (C) 1984, 1985, 1988, 2010 Howard Trickey and Oren Patashnik. + % Unlimited copying and redistribution of this file are permitted as long as + % it is unmodified. Modifications (and redistribution of modified versions) + % are also permitted, but only if the resulting file is renamed to something + % besides btxbst.doc, plain.bst, unsrt.bst, alpha.bst, and abbrv.bst. + % This restriction helps ensure that all standard styles are identical. + % The file btxbst.doc has the documentation for this style. + +ENTRY + { address + author + booktitle + chapter + edition + editor + howpublished + institution + journal + key + month + note + number + organization + pages + publisher + school + series + title + type + volume + year + } + {} + { label } + +INTEGERS { output.state before.all mid.sentence after.sentence after.block } + +FUNCTION {init.state.consts} +{ #0 'before.all := + #1 'mid.sentence := + #2 'after.sentence := + #3 'after.block := +} + +STRINGS { s t } + +FUNCTION {output.nonnull} +{ 's := + output.state mid.sentence = + { ", " * write$ } + { output.state after.block = + { add.period$ write$ + newline$ + "\newblock " write$ + } + { output.state before.all = + 'write$ + { add.period$ " " * write$ } + if$ + } + if$ + mid.sentence 'output.state := + } + if$ + s +} + +FUNCTION {output} +{ duplicate$ empty$ + 'pop$ + 'output.nonnull + if$ +} + +FUNCTION {output.check} +{ 't := + duplicate$ empty$ + { pop$ "empty " t * " in " * cite$ * warning$ } + 'output.nonnull + if$ +} + +FUNCTION {output.bibitem} +{ newline$ + "\bibitem{" write$ + cite$ write$ + "}" write$ + newline$ + "" + before.all 'output.state := +} + +FUNCTION {fin.entry} +{ add.period$ + write$ + newline$ +} + +FUNCTION {new.block} +{ output.state before.all = + 'skip$ + { after.block 'output.state := } + if$ +} + +FUNCTION {new.sentence} +{ output.state after.block = + 'skip$ + { output.state before.all = + 'skip$ + { after.sentence 'output.state := } + if$ + } + if$ +} + +FUNCTION {not} +{ { #0 } + { #1 } + if$ +} + +FUNCTION {and} +{ 'skip$ + { pop$ #0 } + if$ +} + +FUNCTION {or} +{ { pop$ #1 } + 'skip$ + if$ +} + +FUNCTION {new.block.checka} +{ empty$ + 'skip$ + 'new.block + if$ +} + +FUNCTION {new.block.checkb} +{ empty$ + swap$ empty$ + and + 'skip$ + 'new.block + if$ +} + +FUNCTION {new.sentence.checka} +{ empty$ + 'skip$ + 'new.sentence + if$ +} + +FUNCTION {new.sentence.checkb} +{ empty$ + swap$ empty$ + and + 'skip$ + 'new.sentence + if$ +} + +FUNCTION {field.or.null} +{ duplicate$ empty$ + { pop$ "" } + 'skip$ + if$ +} + +FUNCTION {emphasize} +{ duplicate$ empty$ + { pop$ "" } + { "{\em " swap$ * "}" * } + if$ +} + +INTEGERS { nameptr namesleft numnames } + +FUNCTION {format.names} +{ 's := + #1 'nameptr := + s num.names$ 'numnames := + numnames 'namesleft := + { namesleft #0 > } + { s nameptr "{ff~}{vv~}{ll}{, jj}" format.name$ 't := + nameptr #1 > + { namesleft #1 > + { ", " * t * } + { numnames #2 > + { "," * } + 'skip$ + if$ + t "others" = + { " et~al." * } + { " and " * t * } + if$ + } + if$ + } + 't + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ +} + +FUNCTION {format.authors} +{ author empty$ + { "" } + { author format.names } + if$ +} + +FUNCTION {format.editors} +{ editor empty$ + { "" } + { editor format.names + editor num.names$ #1 > + { ", editors" * } + { ", editor" * } + if$ + } + if$ +} + +FUNCTION {format.title} +{ title empty$ + { "" } + { title "t" change.case$ } + if$ +} + +FUNCTION {n.dashify} +{ 't := + "" + { t empty$ not } + { t #1 #1 substring$ "-" = + { t #1 #2 substring$ "--" = not + { "--" * + t #2 global.max$ substring$ 't := + } + { { t #1 #1 substring$ "-" = } + { "-" * + t #2 global.max$ substring$ 't := + } + while$ + } + if$ + } + { t #1 #1 substring$ * + t #2 global.max$ substring$ 't := + } + if$ + } + while$ +} + +FUNCTION {format.date} +{ year empty$ + { month empty$ + { "" } + { "there's a month but no year in " cite$ * warning$ + month + } + if$ + } + { month empty$ + 'year + { month " " * year * } + if$ + } + if$ +} + +FUNCTION {format.btitle} +{ title emphasize +} + +FUNCTION {tie.or.space.connect} +{ duplicate$ text.length$ #3 < + { "~" } + { " " } + if$ + swap$ * * +} + +FUNCTION {either.or.check} +{ empty$ + 'pop$ + { "can't use both " swap$ * " fields in " * cite$ * warning$ } + if$ +} + +FUNCTION {format.bvolume} +{ volume empty$ + { "" } + { "volume" volume tie.or.space.connect + series empty$ + 'skip$ + { " of " * series emphasize * } + if$ + "volume and number" number either.or.check + } + if$ +} + +FUNCTION {format.number.series} +{ volume empty$ + { number empty$ + { series field.or.null } + { output.state mid.sentence = + { "number" } + { "Number" } + if$ + number tie.or.space.connect + series empty$ + { "there's a number but no series in " cite$ * warning$ } + { " in " * series * } + if$ + } + if$ + } + { "" } + if$ +} + +FUNCTION {format.edition} +{ edition empty$ + { "" } + { output.state mid.sentence = + { edition "l" change.case$ " edition" * } + { edition "t" change.case$ " edition" * } + if$ + } + if$ +} + +INTEGERS { multiresult } + +FUNCTION {multi.page.check} +{ 't := + #0 'multiresult := + { multiresult not + t empty$ not + and + } + { t #1 #1 substring$ + duplicate$ "-" = + swap$ duplicate$ "," = + swap$ "+" = + or or + { #1 'multiresult := } + { t #2 global.max$ substring$ 't := } + if$ + } + while$ + multiresult +} + +FUNCTION {format.pages} +{ pages empty$ + { "" } + { pages multi.page.check + { "pages" pages n.dashify tie.or.space.connect } + { "page" pages tie.or.space.connect } + if$ + } + if$ +} + +FUNCTION {format.vol.num.pages} +{ volume field.or.null + number empty$ + 'skip$ + { "(" number * ")" * * + volume empty$ + { "there's a number but no volume in " cite$ * warning$ } + 'skip$ + if$ + } + if$ + pages empty$ + 'skip$ + { duplicate$ empty$ + { pop$ format.pages } + { ":" * pages n.dashify * } + if$ + } + if$ +} + +FUNCTION {format.chapter.pages} +{ chapter empty$ + 'format.pages + { type empty$ + { "chapter" } + { type "l" change.case$ } + if$ + chapter tie.or.space.connect + pages empty$ + 'skip$ + { ", " * format.pages * } + if$ + } + if$ +} + +FUNCTION {format.in.ed.booktitle} +{ booktitle empty$ + { "" } + { editor empty$ + { "In " booktitle emphasize * } + { "In " format.editors * ", " * booktitle emphasize * } + if$ + } + if$ +} + +FUNCTION {empty.misc.check} +{ author empty$ title empty$ howpublished empty$ + month empty$ year empty$ note empty$ + and and and and and + key empty$ not and + { "all relevant fields are empty in " cite$ * warning$ } + 'skip$ + if$ +} + +FUNCTION {format.thesis.type} +{ type empty$ + 'skip$ + { pop$ + type "t" change.case$ + } + if$ +} + +FUNCTION {format.tr.number} +{ type empty$ + { "Technical Report" } + 'type + if$ + number empty$ + { "t" change.case$ } + { number tie.or.space.connect } + if$ +} + +FUNCTION {format.article.crossref} +{ key empty$ + { journal empty$ + { "need key or journal for " cite$ * " to crossref " * crossref * + warning$ + "" + } + { "In {\em " journal * "\/}" * } + if$ + } + { "In " key * } + if$ + " \cite{" * crossref * "}" * +} + +FUNCTION {format.crossref.editor} +{ editor #1 "{vv~}{ll}" format.name$ + editor num.names$ duplicate$ + #2 > + { pop$ " et~al." * } + { #2 < + 'skip$ + { editor #2 "{ff }{vv }{ll}{ jj}" format.name$ "others" = + { " et~al." * } + { " and " * editor #2 "{vv~}{ll}" format.name$ * } + if$ + } + if$ + } + if$ +} + +FUNCTION {format.book.crossref} +{ volume empty$ + { "empty volume in " cite$ * "'s crossref of " * crossref * warning$ + "In " + } + { "Volume" volume tie.or.space.connect + " of " * + } + if$ + editor empty$ + editor field.or.null author field.or.null = + or + { key empty$ + { series empty$ + { "need editor, key, or series for " cite$ * " to crossref " * + crossref * warning$ + "" * + } + { "{\em " * series * "\/}" * } + if$ + } + { key * } + if$ + } + { format.crossref.editor * } + if$ + " \cite{" * crossref * "}" * +} + +FUNCTION {format.incoll.inproc.crossref} +{ editor empty$ + editor field.or.null author field.or.null = + or + { key empty$ + { booktitle empty$ + { "need editor, key, or booktitle for " cite$ * " to crossref " * + crossref * warning$ + "" + } + { "In {\em " booktitle * "\/}" * } + if$ + } + { "In " key * } + if$ + } + { "In " format.crossref.editor * } + if$ + " \cite{" * crossref * "}" * +} + +FUNCTION {article} +{ output.bibitem + format.authors "author" output.check + new.block + format.title "title" output.check + new.block + crossref missing$ + { journal emphasize "journal" output.check + format.vol.num.pages output + format.date "year" output.check + } + { format.article.crossref output.nonnull + format.pages output + } + if$ + new.block + note output + fin.entry +} + +FUNCTION {book} +{ output.bibitem + author empty$ + { format.editors "author and editor" output.check } + { format.authors output.nonnull + crossref missing$ + { "author and editor" editor either.or.check } + 'skip$ + if$ + } + if$ + new.block + format.btitle "title" output.check + crossref missing$ + { format.bvolume output + new.block + format.number.series output + new.sentence + publisher "publisher" output.check + address output + } + { new.block + format.book.crossref output.nonnull + } + if$ + format.edition output + format.date "year" output.check + new.block + note output + fin.entry +} + +FUNCTION {booklet} +{ output.bibitem + format.authors output + new.block + format.title "title" output.check + howpublished address new.block.checkb + howpublished output + address output + format.date output + new.block + note output + fin.entry +} + +FUNCTION {inbook} +{ output.bibitem + author empty$ + { format.editors "author and editor" output.check } + { format.authors output.nonnull + crossref missing$ + { "author and editor" editor either.or.check } + 'skip$ + if$ + } + if$ + new.block + format.btitle "title" output.check + crossref missing$ + { format.bvolume output + format.chapter.pages "chapter and pages" output.check + new.block + format.number.series output + new.sentence + publisher "publisher" output.check + address output + } + { format.chapter.pages "chapter and pages" output.check + new.block + format.book.crossref output.nonnull + } + if$ + format.edition output + format.date "year" output.check + new.block + note output + fin.entry +} + +FUNCTION {incollection} +{ output.bibitem + format.authors "author" output.check + new.block + format.title "title" output.check + new.block + crossref missing$ + { format.in.ed.booktitle "booktitle" output.check + format.bvolume output + format.number.series output + format.chapter.pages output + new.sentence + publisher "publisher" output.check + address output + format.edition output + format.date "year" output.check + } + { format.incoll.inproc.crossref output.nonnull + format.chapter.pages output + } + if$ + new.block + note output + fin.entry +} + +FUNCTION {inproceedings} +{ output.bibitem + format.authors "author" output.check + new.block + format.title "title" output.check + new.block + crossref missing$ + { format.in.ed.booktitle "booktitle" output.check + format.bvolume output + format.number.series output + format.pages output + address empty$ + { organization publisher new.sentence.checkb + organization output + publisher output + format.date "year" output.check + } + { address output.nonnull + format.date "year" output.check + new.sentence + organization output + publisher output + } + if$ + } + { format.incoll.inproc.crossref output.nonnull + format.pages output + } + if$ + new.block + note output + fin.entry +} + +FUNCTION {conference} { inproceedings } + +FUNCTION {manual} +{ output.bibitem + author empty$ + { organization empty$ + 'skip$ + { organization output.nonnull + address output + } + if$ + } + { format.authors output.nonnull } + if$ + new.block + format.btitle "title" output.check + author empty$ + { organization empty$ + { address new.block.checka + address output + } + 'skip$ + if$ + } + { organization address new.block.checkb + organization output + address output + } + if$ + format.edition output + format.date output + new.block + note output + fin.entry +} + +FUNCTION {mastersthesis} +{ output.bibitem + format.authors "author" output.check + new.block + format.title "title" output.check + new.block + "Master's thesis" format.thesis.type output.nonnull + school "school" output.check + address output + format.date "year" output.check + new.block + note output + fin.entry +} + +FUNCTION {misc} +{ output.bibitem + format.authors output + title howpublished new.block.checkb + format.title output + howpublished new.block.checka + howpublished output + format.date output + new.block + note output + fin.entry + empty.misc.check +} + +FUNCTION {phdthesis} +{ output.bibitem + format.authors "author" output.check + new.block + format.btitle "title" output.check + new.block + "PhD thesis" format.thesis.type output.nonnull + school "school" output.check + address output + format.date "year" output.check + new.block + note output + fin.entry +} + +FUNCTION {proceedings} +{ output.bibitem + editor empty$ + { organization output } + { format.editors output.nonnull } + if$ + new.block + format.btitle "title" output.check + format.bvolume output + format.number.series output + address empty$ + { editor empty$ + { publisher new.sentence.checka } + { organization publisher new.sentence.checkb + organization output + } + if$ + publisher output + format.date "year" output.check + } + { address output.nonnull + format.date "year" output.check + new.sentence + editor empty$ + 'skip$ + { organization output } + if$ + publisher output + } + if$ + new.block + note output + fin.entry +} + +FUNCTION {techreport} +{ output.bibitem + format.authors "author" output.check + new.block + format.title "title" output.check + new.block + format.tr.number output.nonnull + institution "institution" output.check + address output + format.date "year" output.check + new.block + note output + fin.entry +} + +FUNCTION {unpublished} +{ output.bibitem + format.authors "author" output.check + new.block + format.title "title" output.check + new.block + note "note" output.check + format.date output + fin.entry +} + +FUNCTION {default.type} { misc } + +MACRO {jan} {"January"} + +MACRO {feb} {"February"} + +MACRO {mar} {"March"} + +MACRO {apr} {"April"} + +MACRO {may} {"May"} + +MACRO {jun} {"June"} + +MACRO {jul} {"July"} + +MACRO {aug} {"August"} + +MACRO {sep} {"September"} + +MACRO {oct} {"October"} + +MACRO {nov} {"November"} + +MACRO {dec} {"December"} + +MACRO {acmcs} {"ACM Computing Surveys"} + +MACRO {acta} {"Acta Informatica"} + +MACRO {cacm} {"Communications of the ACM"} + +MACRO {ibmjrd} {"IBM Journal of Research and Development"} + +MACRO {ibmsj} {"IBM Systems Journal"} + +MACRO {ieeese} {"IEEE Transactions on Software Engineering"} + +MACRO {ieeetc} {"IEEE Transactions on Computers"} + +MACRO {ieeetcad} + {"IEEE Transactions on Computer-Aided Design of Integrated Circuits"} + +MACRO {ipl} {"Information Processing Letters"} + +MACRO {jacm} {"Journal of the ACM"} + +MACRO {jcss} {"Journal of Computer and System Sciences"} + +MACRO {scp} {"Science of Computer Programming"} + +MACRO {sicomp} {"SIAM Journal on Computing"} + +MACRO {tocs} {"ACM Transactions on Computer Systems"} + +MACRO {tods} {"ACM Transactions on Database Systems"} + +MACRO {tog} {"ACM Transactions on Graphics"} + +MACRO {toms} {"ACM Transactions on Mathematical Software"} + +MACRO {toois} {"ACM Transactions on Office Information Systems"} + +MACRO {toplas} {"ACM Transactions on Programming Languages and Systems"} + +MACRO {tcs} {"Theoretical Computer Science"} + +READ + +FUNCTION {sortify} +{ purify$ + "l" change.case$ +} + +INTEGERS { len } + +FUNCTION {chop.word} +{ 's := + 'len := + s #1 len substring$ = + { s len #1 + global.max$ substring$ } + 's + if$ +} + +FUNCTION {sort.format.names} +{ 's := + #1 'nameptr := + "" + s num.names$ 'numnames := + numnames 'namesleft := + { namesleft #0 > } + { nameptr #1 > + { " " * } + 'skip$ + if$ + s nameptr "{vv{ } }{ll{ }}{ ff{ }}{ jj{ }}" format.name$ 't := + nameptr numnames = t "others" = and + { "et al" * } + { t sortify * } + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ +} + +FUNCTION {sort.format.title} +{ 't := + "A " #2 + "An " #3 + "The " #4 t chop.word + chop.word + chop.word + sortify + #1 global.max$ substring$ +} + +FUNCTION {author.sort} +{ author empty$ + { key empty$ + { "to sort, need author or key in " cite$ * warning$ + "" + } + { key sortify } + if$ + } + { author sort.format.names } + if$ +} + +FUNCTION {author.editor.sort} +{ author empty$ + { editor empty$ + { key empty$ + { "to sort, need author, editor, or key in " cite$ * warning$ + "" + } + { key sortify } + if$ + } + { editor sort.format.names } + if$ + } + { author sort.format.names } + if$ +} + +FUNCTION {author.organization.sort} +{ author empty$ + { organization empty$ + { key empty$ + { "to sort, need author, organization, or key in " cite$ * warning$ + "" + } + { key sortify } + if$ + } + { "The " #4 organization chop.word sortify } + if$ + } + { author sort.format.names } + if$ +} + +FUNCTION {editor.organization.sort} +{ editor empty$ + { organization empty$ + { key empty$ + { "to sort, need editor, organization, or key in " cite$ * warning$ + "" + } + { key sortify } + if$ + } + { "The " #4 organization chop.word sortify } + if$ + } + { editor sort.format.names } + if$ +} + +FUNCTION {presort} +{ type$ "book" = + type$ "inbook" = + or + 'author.editor.sort + { type$ "proceedings" = + 'editor.organization.sort + { type$ "manual" = + 'author.organization.sort + 'author.sort + if$ + } + if$ + } + if$ + " " + * + year field.or.null sortify + * + " " + * + title field.or.null + sort.format.title + * + #1 entry.max$ substring$ + 'sort.key$ := +} + +ITERATE {presort} + +SORT + +STRINGS { longest.label } + +INTEGERS { number.label longest.label.width } + +FUNCTION {initialize.longest.label} +{ "" 'longest.label := + #1 'number.label := + #0 'longest.label.width := +} + +FUNCTION {longest.label.pass} +{ number.label int.to.str$ 'label := + number.label #1 + 'number.label := + label width$ longest.label.width > + { label 'longest.label := + label width$ 'longest.label.width := + } + 'skip$ + if$ +} + +EXECUTE {initialize.longest.label} + +ITERATE {longest.label.pass} + +FUNCTION {begin.bib} +{ preamble$ empty$ + 'skip$ + { preamble$ write$ newline$ } + if$ + "\begin{thebibliography}{" longest.label * "}" * write$ newline$ +} + +EXECUTE {begin.bib} + +EXECUTE {init.state.consts} + +ITERATE {call.type$} + +FUNCTION {end.bib} +{ newline$ + "\end{thebibliography}" write$ newline$ +} + +EXECUTE {end.bib} diff --git a/tests/examplefiles/postgresql_test.txt b/tests/examplefiles/postgresql_test.txt index 190d184f..28db5ee3 100644 --- a/tests/examplefiles/postgresql_test.txt +++ b/tests/examplefiles/postgresql_test.txt @@ -45,3 +45,37 @@ $$; SELECT U&'\0441\043B\043E\043D' FROM U&"\0441\043B\043E\043D"; +-- Escapes +SELECT E'1\n2\n3'; + +-- DO example from postgresql documentation +/* + * PostgreSQL is Copyright © 1996-2016 by the PostgreSQL Global Development Group. + * + * Postgres95 is Copyright © 1994-5 by the Regents of the University of California. + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written agreement + * is hereby granted, provided that the above copyright notice and this paragraph + * and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, + * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS-IS" BASIS, + * AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, + * SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + */ +DO $$DECLARE r record; +BEGIN + FOR r IN SELECT table_schema, table_name FROM information_schema.tables + WHERE table_type = 'VIEW' AND table_schema = 'public' + LOOP + EXECUTE 'GRANT ALL ON ' || quote_ident(r.table_schema) || '.' || quote_ident(r.table_name) || ' TO webuser'; + END LOOP; +END$$; diff --git a/tests/examplefiles/rnc_example.rnc b/tests/examplefiles/rnc_example.rnc new file mode 100644 index 00000000..a1440302 --- /dev/null +++ b/tests/examplefiles/rnc_example.rnc @@ -0,0 +1,33 @@ +# This is a sample RNC file from the tutorial for the 2003 Working Draft +# http://relaxng.org/compact-tutorial-20030326.html + +element html { + element head { + element title { text } + }, + element body { + element table { + attribute class { "addressBook" }, + element tr { + attribute class { "card" }, + element td { + attribute class { "name" }, + mixed { + element span { + attribute class { "givenName" }, + text + }?, + element span { + attribute class { "familyName" }, + text + }? + } + }, + element td { + attribute class { "email" }, + text + } + }+ + } + } +} diff --git a/tests/examplefiles/test.bib b/tests/examplefiles/test.bib new file mode 100644 index 00000000..87e558d8 --- /dev/null +++ b/tests/examplefiles/test.bib @@ -0,0 +1,77 @@ +This is an example BibTeX file. +This text is a comment. + +@preamble{"%%% example BibTeX file"} + +@Preamble{"\newcommand{\noopsort}[1]{} " + "\newcommand{\noopsort}[1]{} "} + +@String{SCI = "Science"} + +@STRING{JFernandez = "Fernandez, Julio M."} +@StRiNg{HGaub = "Gaub, Hermann E."} +@string{MGautel = "Gautel, Mathias"} +@String{FOesterhelt = "Oesterhelt, Filipp"} +@String{MRief = "Rief, Matthias"} + +@Article{rief97b, + author = MRief #" and "# MGautel #" and "# FOesterhelt + #" and "# JFernandez #" and "# HGaub, + title = "Reversible Unfolding of Individual Titin + Immunoglobulin Domains by {AFM}", + journal = SCI, + volume = 276, + number = 5315, + pages = "1109--1112", + year = 1997, + doi = "10.1126/science.276.5315.1109", + URL = "http://www.sciencemag.org/cgi/content/abstract/276/5315/1109", + eprint = "http://www.sciencemag.org/cgi/reprint/276/5315/1109.pdf", +} + + +Parens can be used instead of braces: + +@ARTICLE(ruckenstein-diffusion, + author = "Liu, Hongquin and Ruckenstein, Eli", + language = "english", + title = "Predicting the Diffusion Coefficient in Supercritical Fluids", + journal = "Ind. Eng. Chem. Res.", + volume = "36", + year = "1997", + pages = "888-895" +) + +@book{ + viktorov-methods, + author = "Викторов, Михаил Маркович", + publisher = "Л.: <<Химия>>", + title = "Методы вычисления физико-химических величин и прикладные расчёты", + language = "russian", + year = "1977", + isbn = "000-0000000000", +} + +@comment{jackson-commented-out, + author = "Jackson, P\'eter", + publisher = "Some Publisher", + language = "english", + title = "Some Title", + series = "Some series", + booktitle = "Commented Out", + number = "3", + edition = "Second", + year = "1933", + pages = "44--59" +} + +@booklet{test-booklet, + author = "de Last, Jr., First Middle", + language = "english", + title = "Just a booklet", + year = 2006, + month = jan, + address = "Moscow", + howpublished = "Published by Foo" +} + diff --git a/tests/examplefiles/test.cr b/tests/examplefiles/test.cr new file mode 100644 index 00000000..028ff6f3 --- /dev/null +++ b/tests/examplefiles/test.cr @@ -0,0 +1,2871 @@ +# Examples taken from http://crystal-lang.org/docs/ +# Copyright 2012-2016 Manas Technology Solutions. + + +require "http/server" + +server = HTTP::Server.new(8080) do |context| + context.response.content_type = "text/plain" + context.response.print "Hello world! The time is #{Time.now}" +end + +puts "Listening on http://0.0.0.0:8080" +server.listen + + +module HTTP + class RequestHandler + end +end + +alias NumericValue = Float32 | Float64 | Int32 | Int64 + +enum Time::DayOfWeek +end + + +$global_greeting = "Hello world" + +class Greeting + @@default_greeting = "Hello world" + + def initialize(@custom_greeting = nil) + end + + def print_greeting + greeting = @custom_greeting || @@default_greeting + puts greeting + end +end + + +LUCKY_NUMBERS = [3, 7, 11] +DOCUMENTATION_URL = "http://crystal-lang.org/docs" + + +module Scorecard + class Parser + def parse(score_text) + begin + score_text.scan(SCORE_PATTERN) do |match| + handle_match(match) + end + rescue err : ParseError + # handle error ... + end + end + end +end + + +module Money + CURRENCIES = { + "EUR" => 1.0, + "ARS" => 10.55, + "USD" => 1.12, + "JPY" => 134.15, + } + + class Amount + getter :currency, :value + + def initialize(@currency, @value) + end + end + + class CurrencyConversion + def initialize(@amount, @target_currency) + end + + def amount + # implement conversion ... + end + end +end + + +i = 0 +while i < 10 + proc = ->(x : Int32) do + spawn do + puts(x) + end + end + proc.call(i) + i += 1 +end + +Fiber.yield + + +# A buffered channel of capacity 2 +channel = Channel(Int32).new(2) + +spawn do + channel.send(1) + channel.send(2) + channel.send(3) +end + +3.times do |i| + puts channel.receive +end + + +class MyDictionary(K, V) +end + + +MyBox.new(1) #:: MyBox(Int32) +MyBox.new("hello") #:: MyBox(String) + + +module Moo(T) + def t + T + end +end + +class Foo(U) + include Moo(U) + + def initialize(@value : U) + end +end + +foo = Foo.new(1) +foo.t # Int32 + + +class Parent(T) +end + +class Int32Child < Parent(Int32) +end + +class GenericChild(T) < Parent(T) +end + + +class Person +end + + +a = 1 +ptr = pointerof(a) +ptr[100_000] = 2 # undefined behaviour, probably a segmentation fault + + +alias Int32OrString = Int32 | String + + +alias Int32OrNil = Int32? + + +alias Int32OrNil_ = Int32 | ::Nil + + +alias Int32Ptr = Int32* + + +alias Int32Ptr_ = Pointer(Int32) + + +alias Int32_8 = Int32[8] + + +alias Int32_8_ = StaticArray(Int32, 8) + + +alias Int32StringTuple = {Int32, String} + + +alias Int32StringTuple_ = Tuple(Int32, String) + + +alias Int32ToString = Int32 -> String + + +alias Int32ToString_ = Proc(Int32, String) + + +alias ProcThatReturnsInt32 = -> Int32 + + +alias Int32AndCharToString = Int32, Char -> String + + +alias ComplexProc = (Int32 -> Int32) -> String + + +def foo(x : Int32) + "instance" +end + +def foo(x : Int32.class) + "class" +end + +foo 1 # "instance" +foo Int32 # "class" + + +class Parent +end + +class Child1 < Parent +end + +class Child2 < Parent +end + +ary = [] of Parent.class +ary << Child1 +ary << Child2 + + +# Same as not specifying a restriction, not very useful +def foo(x : _) +end + +# A bit more useful: any two arguments Proc that returns an Int32: +def foo(x : _, _ -> Int32) +end + + +#alias SameAsInt32 = typeof(2) +#alias Int32OrString_ = typeof(1, "a") + + +class Person + def initialize(name) + @name = name + @age = 0 + end + + def name + @name + end + + def age + @age + end +end + + +john = Person.new "John" +peter = Person.new "Peter" + +john.name #=> "John" +john.age #=> 0 + +peter.name #=> "Peter" + + +class Person + def self.new(name) + instance = Person.allocate + instance.initialize(name) + instance + end + end + + +if a.is_a?(String) + # here a is a String +end + +if b.is_a?(Number) + # here b is a Number +end + + +a = some_condition ? 1 : "hello" +# a : Int32 | String + +if a.is_a?(Number) + # a : Int32 +else + # a : String +end + + +if a.is_a?(String) && b.is_a?(Number) + # here a is a String and b is a Number +end + + +a.+(b) + + +struct Vector2 + getter x, y + + def initialize(@x, @y) + end + + def +(other) + Vector2.new(x + other.x, y + other.y) + end +end + +v1 = Vector2.new(1, 2) +v2 = Vector2.new(3, 4) +v1 + v2 #=> Vector2(@x=4, @y=6) + + + + +struct Vector2 + def - + Vector2.new(-x, -y) + end +end + +v1 = Vector2.new(1, 2) +-v1 #=> Vector2(@x=-1, @y=-2) + + + + + +class MyArray + def [](index) + # ... + end + + def [](index1, index2, index3) + # ... + end + + def []=(index, value) + # ... + end +end + +array = MyArray.new + +array[1] # invokes the first method +array[1, 2, 3] # invokes the second method +array[1] = 2 # invokes the third method + +array.[](1) # invokes the first method +array.[](1, 2, 3) # invokes the second method +array.[]=(1, 2) # invokes the third method + + +raise "OH NO!" +raise Exception.new("Some error") + + +class MyException < Exception +end + + +begin + raise MyException.new("OH NO!") +rescue ex : MyException + puts "Rescued MyException: #{ex.message}" +end + + +begin + # ... +rescue ex : MyException | MyOtherException + # only MyException or MyOtherException +rescue + # any other kind of exception +ensure + puts "Cleanup..." +end + + +def some_method + something_dangerous +rescue + # execute if an exception is raised +end + + +array = [1, 2, 3] +array[4] # raises because of IndexError +array[4]? # returns nil because of index out of bounds + + +def some_proc(&block : Int32 -> Int32) + block +end + +x = 0 +proc = ->(i : Int32) { x += i } +proc = some_proc(&proc) +proc.call(1) #=> 1 +proc.call(10) #=> 11 +x #=> 11 + + +def add(x, y) + x + y +end + +adder = ->add(Int32, Int32) +adder.call(1, 2) #=> 3 + + +module Curses + class Window + end +end + +Curses::Window.new + + +module ItemsSize + def size + items.size + end +end + +class Items + include ItemsSize + + def items + [1, 2, 3] + end +end + +items = Items.new +items.size #=> 3 + + +module Base64 + extend self + + def encode64(string) + # ... + end + + def decode64(string) + # ... + end +end + +Base64.encode64 "hello" #=> "aGVsbG8=" + + +if some_condition + a = 1 +else + a = "hello" +end + +a_as_int = a as Int32 +a_as_int.abs # works, compiler knows that a_as_int is Int32 + + +ptr = Pointer(Int32).malloc(1) +ptr as Int8* #:: Pointer(Int8) + + +array = [1, 2, 3] + +# object_id returns the address of an object in memory, +# so we create a pointer with that address +ptr = Pointer(Void).new(array.object_id) + +# Now we cast that pointer to the same type, and +# we should get the same value +array2 = ptr as Array(Int32) +array2.same?(array) #=> true + + +a = 1 +b = a as Int32 | Float64 +b #:: Int32 | Float64 + + +ary = [1, 2, 3] + +# We want to create an array 1, 2, 3 of Int32 | Float64 +ary2 = ary.map { |x| x as Int32 | Float64 } + +ary2 #:: Array(Int32 | Float64) +ary2 << 1.5 # OK + + +class Person + def initialize(@name) + end + + def name + @name + end +end + +a = [] of Person +x = a.map { |f| f.name } # Error: can't infer block return type + + +a = [] of Person +x = a.map { |f| f.name as String } # OK + + +Person.new "John" + +a = [] of Person +x = a.map { |f| f.name } # OK + + +loop do + do_something + break if some_condition +end + + +class Point + def initialize(@x, @y) + end +end + +Point.new 1, 2 + +# 2 x Int32 = 2 x 4 = 8 +instance_sizeof(Point) #=> 12 + + +a = 1 +while a < 5 + a += 1 + if a == 3 + next + end + puts a +end +# The above prints the numbers 2, 4 and 5 + + +lib C + # In C: double cos(double x) + fun cos(value : Float64) : Float64 + + fun getch : Int32 + + fun srand(seed : UInt32) + + fun exit(status : Int32) : NoReturn + + fun printf(format : UInt8*, ...) : Int32 +end + +C.cos(1.5) #=> 0.0707372 +C.srand(1_u32) + +a = 1 +b = 2 +C.printf "%d + %d = %d\n", a, b, a + b + + +lib LibSDL + fun init = SDL_Init(flags : UInt32) : Int32 +end + +lib LLVMIntrinsics + fun ceil_f32 = "llvm.ceil.f32"(value : Float32) : Float32 +end + +lib MyLib + fun my_fun(some_size : LibC::SizeT) +end + +@[Link("pcre")] +lib LibPCRE +end + + +lib C + ifdef x86_64 + alias SizeT = UInt64 + else + alias SizeT = UInt32 + end + + fun memcmp(p1 : Void*, p2 : Void*, size : C::SizeT) : Int32 +end + + +lib X + enum SomeEnum + Ten = 10 + Twenty = 10 * 2 + ThirtyTwo = 1 << 5 + end +end + + +lib X + enum SomeEnum + A = 1_u32 + end +end + + +X::SomeEnum::Zero #=> 0_i8 +X::SomeEnum::Two #=> 2_i8 + + +lib X + fun callback(f : Int32 -> Int32) +end + + +f = ->(x : Int32) { x + 1 } +X.callback(f) + + +X.callback ->(x) { x + 1 } + + +X.callback nil + + +lib LibFoo + fun store_callback(callback : ->) + fun execute_callback +end + +LibFoo.store_callback ->{ raise "OH NO!" } +LibFoo.execute_callback + + +lib LibFoo + fun store_callback(callback : ->) + + @[Raises] + fun execute_callback +end + + +@[Link("pcre")] +lib PCRE + INFO_CAPTURECOUNT = 2 +end + +PCRE::INFO_CAPTURECOUNT #=> 2 + + +lib U + # In C: + # + # union IntOrFloat { + # int some_int; + # double some_float; + # }; + union IntOrFloat + some_int : Int32 + some_float : Float64 + end +end + + +value = U::IntOrFloat.new + + +value = uninitialized U::IntOrFlaot +value.some_int #=> some garbage value + + +value = U::IntOrFloat.new +value.some_int = 1 +value.some_int #=> 1 +value.some_float #=> 4.94066e-324 + + +def change_it(value) + value.some_int = 1 +end + +value = U::IntOrFloat.new +change_it value +value.some_int #=> 0 + + +lib C + # In C: + # + # struct TimeZone { + # int minutes_west; + # int dst_time; + # }; + struct TimeZone + minutes_west : Int32 + dst_time : Int32 + end +end + + +lib C + # This is a forward declaration + struct Node + end + + struct Node + node : Node* + end +end + + +tz = C::TimeZone.new + + +tz = uninitialized C::TimeZone +tz.minutes_west #=> some garbage value + + +tz = C::TimeZone.new +tz.minutes_west = 1 +tz.minutes_west #=> 1 + + +tz = C::TimeZone.new minutes_west: 1, dst_time: 2 +tz.minutes_west #=> 1 +tz.dst_time #=> 2 + + +def change_it(tz) + tz.minutes_west = 1 +end + +tz = C::TimeZone.new +change_it tz +tz.minutes_west #=> 0 + + +lib C + $errno : Int32 +end + + +C.errno #=> some value +C.errno = 0 +C.errno #=> 0 + + +lib C + @[ThreadLocal] + $errno : Int32 +end + + +lib C + fun waitpid(pid : Int32, status_ptr : Int32*, options : Int32) : Int32 +end + + +status_ptr = uninitialized Int32 + +C.waitpid(pid, pointerof(status_ptr), options) + + +C.waitpid(pid, out status_ptr, options) + + +lib X + type CInt = Int32 +end + + +ifdef x86_64 + # some specific code for 64 bits platforms +else + # some specific code for non-64 bits platforms +end + + +ifdef linux && x86_64 + # some specific code for linux 64 bits +end + + +lib C + ifdef linux + struct SomeStruct + some_field : Int32 + end + else + struct SomeStruct + some_field : Int64 + end + end +end + + +# Assigns to a local variable +local = 1 + +# Assigns to a global variable +$global = 4 + +class Testing + # Assigns to an instance variable + @instance = 2 + + # Assigns to a class variable + @@class = 3 +end + + +local += 1 # same as: local = local + 1 + +# The above is valid with these operators: +# +, -, *, /, %, |, &, ^, **, <<, >> + +local ||= 1 # same as: local || (local = 1) +local &&= 1 # same as: local && (local = 1) + + +# A setter +person.name=("John") + +# The above can be written as: +person.name = "John" + +# An indexed assignment +objects.[]=(2, 3) + +# The above can be written as: +objects[2] = 3 + +# Not assignment-related, but also syntax sugar: +objects.[](2, 3) + +# The above can be written as: +objects[2, 3] + + +person.age += 1 # same as: person.age = person.age + 1 + +person.name ||= "John" # same as: person.name || (person.name = "John") +person.name &&= "John" # same as: person.name && (person.name = "John") + +objects[1] += 2 # same as: objects[1] = objects[1] + 2 + +objects[1] ||= 2 # same as: objects[1]? || (objects[1] = 2) +objects[1] &&= 2 # same as: objects[1]? && (objects[1] = 2) + + +alias PInt32 = Pointer(Int32) + +ptr = PInt32.malloc(1) # : Pointer(Int32) + + +alias RecArray = Array(Int32) | Array(RecArray) + +ary = [] of RecArray +ary.push [1, 2, 3] +ary.push ary +ary #=> [[1, 2, 3], [...]] + + +module Json + alias Type = Nil | + Bool | + Int64 | + Float64 | + String | + Array(Type) | + Hash(String, Type) +end + + +a = 1 +if a > 0 + a = 10 +end +a #=> 10 + +b = 1 +if b > 2 + b = 10 +else + b = 20 +end +b #=> 20 + + +if some_condition + do_something +elsif some_other_condition + do_something_else +else + do_that +end + + +a = 1 +if some_condition + a = "hello" +else + a = true +end +# a : String | Bool + +b = 1 +if some_condition + b = "hello" +end +# b : Int32 | String + +if some_condition + c = 1 +else + c = "hello" +end +# c : Int32 | String + +if some_condition + d = 1 +end +# d : Int32 | Nil + + +a = 1 +if some_condition + a = "hello" + # a : String + a.size +end +# a : String | Int32 + + +if some_condition + e = 1 +else + e = "hello" + # e : String + return +end +# e : Int32 + + +enum Color : UInt8 + Red # 0 + Green # 1 + Blue = 5 # overwritten to 5 + Yellow # 6 (5 + 1) + + def red? + self == Color::Red + end +end + +Color::Red.value #:: UInt8 + + +@[Flags] +enum IOMode + Read # 1 + Write # 2 + Async # 4 +end + + +IOMode::None.value #=> 0 +IOMode::All.value #=> 7 + + +puts(Color::Red) # prints "Red" +puts(IOMode::Write | IOMode::Async) # prints "Write, Async" + + +puts Color.new(1) #=> prints "Green" + + +puts Color.new(10) #=> prints "10" + + +Color::Red.red? #=> true +Color::Blue.red? #=> false + + +def paint(color : Color) + case color + when Color::Red + # ... + else + # Unusual, but still can happen + raise "unknown color: #{color}" + end +end + +paint Color::Red + + +def paint(color : Symbol) + case color + when :red + # ... + else + raise "unknown color: #{color}" + end +end + +paint :red + + +name = "Crystal" +age = 1 + + +flower = "Tulip" +# At this point 'flower' is a String + +flower = 1 +# At this point 'flower' is an Int32 + + +class Foo + def finalize + # Invoked when Foo is garbage-collected + puts "Bye bye from #{self}!" + end +end + +# Prints "Bye bye ...!" for ever +loop do + Foo.new +end + + +# Defines a method in the program +def add(x, y) + x + y +end + +# Invokes the add method in the program +add(1, 2) #=> 3 + + +def even?(num) + if num % 2 == 0 + return true + end + + return false +end + + +def add(x, y) + x + y +end + +class Foo + def bar + # invokes the program's add method + add(1, 2) + + # invokes Foo's baz method + baz(1, 2) + end + + def baz(x, y) + x * y + end +end + + +def baz(x, y) + x + y +end + +class Foo + def bar + baz(4, 2) #=> 2 + ::baz(4, 2) #=> 6 + end + + def baz(x, y) + x - y + end +end + + +x = 1 + +def add(y) + x + y # error: undefined local variable or method 'x' +end + +add(2) + + +add 1, 2 # same as add(1, 2) + + +class Counter + @@instances = 0 + + def initialize + @@instances += 1 + end + + def self.instances + @@instances + end +end + +Counter.instances #=> 0 +Counter.new +Counter.new +Counter.new +Counter.instances #=> 3 + + +class Counter + def self.increment + @@instances += 1 + end +end + +Counter.increment # Error: undefined method '+' for Nil + + +class Parent + @@counter = 0 +end + +class Child < Parent + def self.counter + @@counter + end +end + +Child.counter #=> nil + + +unless some_condition + then_expression +else + else_expression +end + +# Can also be written as a suffix +close_door unless door_closed? + + +a = 1 +b = typeof(a) #=> Int32 + + +typeof(1, "a", 'a') #=> (Int32 | String | Char) + + +hash = {} of Int32 => String +another_hash = typeof(hash).new #:: Hash(Int32, String) + + +class Array + def self.elem_type(typ) + if typ.is_a?(Array) + elem_type(typ.first) + else + typ + end + end +end + +nest = [1, ["b", [:c, ['d']]]] +flat = Array(typeof(Array.elem_type(nest))).new +typeof(nest) #=> Array(Int32 | Array(String | Array(Symbol | Array(Char)))) +typeof(flat) #=> Array(String | Int32 | Symbol | Char) + + +a = 2 if some_condition + + +x = 0 +proc = ->{ x += 1; x } +proc.call #=> 1 +proc.call #=> 2 +x #=> 2 + + +def counter + x = 0 + ->{ x += 1; x } +end + +proc = counter +proc.call #=> 1 +proc.call #=> 2 + + +def foo + yield +end + +x = 1 +foo do + x = "hello" +end +x # : Int32 | String + + +x = 1 +foo do + x = "hello" +end +x # : Int32 | String + +x = 'a' +x # : Char + + +def capture(&block) + block +end + +x = 1 +capture { x = "hello" } + +x = 'a' +x # : Int32 | String | Char + + +def capture(&block) + block +end + +x = 1 +->{ x = "hello" } + +x = 'a' +x # : Int32 | String | Char + + +abstract class Animal + # Makes this animal talk + abstract def talk +end + +class Dog < Animal + def talk + "Woof!" + end +end + +class Cat < Animal + def talk + "Miau" + end +end + +class Person + getter pet + + def initialize(@name, @pet) + end +end + +john = Person.new "John", Dog.new +peter = Person.new "Peter", Cat.new + + +john.pet.talk #=> "Woof!" + + +a = 1 > 2 ? 3 : 4 + +# The above is the same as: +a = if 1 > 2 + 3 + else + 4 + end + + +def some_method : String + "hello" +end + + +PI = 3.14 + +module Earth + RADIUS = 6_371_000 +end + +PI #=> 3.14 +Earth::RADIUS #=> 6_371_000 + + +TEN = begin + a = 0 + while a < 10 + a += 1 + end + a +end + +TEN #=> 10 + + +class Person + getter name + + def initialize(@name) + @age = 0 + end +end + +john = Person.new "John" +john.name #=> "John" +john.name.size #=> 4 + + +one = Person.new 1 +one.name #=> 1 +one.name + 2 #=> 3 + + +john = Person.new "John" +one = Person.new 1 + + +john = Person.new "John" +one = Person.new 1 + +# Error: undefined method 'size' for Int32 +john.name.size + +# Error: no overload matches 'String#+' with types Int32 +john.name + 3 + + +john = Person.new "John" +john.name.size +one = Person.new 1 + + +class Person + getter name + + def initialize(@name) + @age = 0 + end + + def address + @address + end + + def address=(@address) + end +end + +john = Person.new "John" +john.address = "Argentina" + + +# Error: undefined method 'size' for Nil +john.address.size + + +class Person + @age = 0 + + def initialize(@name) + end +end + + +class Person + @age : Int32 + + def initialize(@name) + @age = 0 + end +end + + +a = if 2 > 1 + 3 + else + 4 + end +a #=> 3 + + +if 1 > 2 +else + 3 +end + + +def twice(&block) + yield + yield +end + + +twice() do + puts "Hello!" +end + +twice do + puts "Hello!" +end + +twice { puts "Hello!" } + + +def twice + yield 1 + yield 2 +end + +twice do |i| + puts "Got #{i}" +end + + +twice { |i| puts "Got #{i}" } + + +def many + yield 1, 2, 3 +end + +many do |x, y, z| + puts x + y + z +end + +# Output: 6 + + +def many + yield 1, 2, 3 +end + +many do |x, y| + puts x + y +end + +# Output: 3 + + +def twice + yield + yield +end + +twice do |i| + puts i.inspect +end + + +def some + yield 1, 'a' + yield true, "hello" + yield 2 +end + +some do |first, second| + # first is Int32 | Bool + # second is Char | String | Nil +end + + +method do |argument| + argument.some_method +end + + +method(&.some_method) + + +method &.some_method(arg1, arg2) + + +method &.+(2) +method &.[index] + + +def twice + v1 = yield 1 + puts v1 + + v2 = yield 2 + puts v2 +end + +twice do |i| + i + 1 +end + + +ary = [1, 2, 3] +ary.map { |x| x + 1 } #=> [2, 3, 4] +ary.select { |x| x % 2 == 1 } #=> [1, 3] + + +def transform(value) + yield value +end + +transform(1) { |x| x + 1 } #=> 2 + + +def thrice + puts "Before 1" + yield 1 + puts "Before 2" + yield 2 + puts "Before 3" + yield 3 + puts "After 3" +end + +thrice do |i| + if i == 2 + break + end +end + + +def twice + yield 1 + yield 2 +end + +twice { |i| i + 1 } #=> 3 +twice { |i| break "hello" } #=> "hello" + + +value = twice do |i| + if i == 1 + break "hello" + end + i + 1 +end +value #:: Int32 | String + + +values = twice { break 1, 2 } +values #=> {1, 2} + + +value = twice { break } +value #=> nil + + +def twice + yield 1 + yield 2 +end + +twice do |i| + if i == 1 + puts "Skipping 1" + next + end + + puts "Got #{i}" +end + + + +def twice + v1 = yield 1 + puts v1 + + v2 = yield 2 + puts v2 +end + +twice do |i| + if i == 1 + next 10 + end + + i + 1 +end + +# Output +# 10 +# 3 + + +class Foo + def one + 1 + end + + def yield_with_self + with self yield + end + + def yield_normally + yield + end +end + +def one + "one" +end + +Foo.new.yield_with_self { one } # => 1 +Foo.new.yield_normally { one } # => "one" + + +def twice + yield 1 + yield 2 +end + +twice do |i| + puts "Got: #{i}" +end + + +i = 1 +puts "Got: #{i}" +i = 2 +puts "Got: #{i}" + + +3.times do |i| + puts i +end + + +struct Int + def times + i = 0 + while i < self + yield i + i += 1 + end + end +end + + +i = 0 +while i < 3 + puts i + i += 1 +end + + +class Person + def initialize(@name) + end + + def greet + puts "Hi, I'm #{@name}" + end +end + +class Employee < Person +end + +employee = Employee.new "John" +employee.greet # "Hi, I'm John" + + +class Person + def initialize(@name) + end +end + +class Employee < Person + def initialize(@name, @company_name) + end +end + +Employee.new "John", "Acme" # OK +Employee.new "Peter" # Error: wrong number of arguments + # for 'Employee:Class#new' (1 for 2) + + +class Person + def greet(msg) + puts "Hi, #{msg}" + end +end + +class Employee < Person + def greet(msg) + puts "Hello, #{msg}" + end +end + +p = Person.new +p.greet "everyone" # "Hi, everyone" + +e = Employee.new +e.greet "everyone" # "Hello, everyone" + + +class Person + def greet(msg) + puts "Hi, #{msg}" + end +end + +class Employee < Person + def greet(msg : Int32) + puts "Hi, this is a number: #{msg}" + end +end + +e = Employee.new +e.greet "everyone" # "Hi, everyone" + +e.greet 1 # "Hi, this is a number: 1" + + +class Person + def greet(msg) + puts "Hello, "#{msg}" + end +end + +class Employee < Person + def greet(msg) + super # Same as: super(msg) + super("another message") + end +end + + +def int_to_int(&block : Int32 -> Int32) + block +end + +proc = int_to_int { |x| x + 1 } +proc.call(1) #=> 2 + + +class Model + def on_save(&block) + @on_save_callback = block + end + + def save + if callback = @on_save_callback + callback.call + end + end +end + +model = Model.new +model.on_save { puts "Saved!" } +model.save # prints "Saved!" + + +def some_proc(&block : Int32 ->) + block +end + +proc = some_proc { |x| x + 1 } +proc.call(1) # void + + +def some_proc(&block : Int32 -> _) + block +end + +proc = some_proc { |x| x + 1 } +proc.call(1) # 2 + +proc = some_proc { |x| x.to_s } +proc.call(1) # "1" + + +macro update_x + x = 1 +end + +x = 0 +update_x +x #=> 1 + + +macro dont_update_x + %x = 1 + puts %x +end + +x = 0 +dont_update_x # outputs 1 +x #=> 0 + + +macro fresh_vars_sample(*names) + # First declare vars + {% for name, index in names %} + print "Declaring: ", "%name{index}", '\n' + %name{index} = {{index}} + {% end %} + + # Then print them + {% for name, index in names %} + print "%name{index}: ", %name{index}, '\n' + {% end %} +end + +fresh_vars_sample a, b, c + +# Sample output: +# Declaring: __temp_255 +# Declaring: __temp_256 +# Declaring: __temp_257 +# __temp_255: 0 +# __temp_256: 1 +# __temp_257: 2 + + +class Object + macro def instance_vars_names : Array(String) + {{ @type.instance_vars.map &.name.stringify }} + end +end + +class Person + def initialize(@name, @age) + end +end + +person = Person.new "John", 30 +person.instance_vars_names #=> ["name", "age"] + + +class Object + macro def has_instance_var?(name) : Bool + # We cannot access name inside the macro expansion here, + # instead we need to use the macro language to construct an array + # and do the inclusion check at runtime. + {{ @type.instance_vars.map &.name.stringify }}.includes? name + end +end + +person = Person.new "John", 30 +person.has_instance_var?("name") #=> true +person.has_instance_var?("birthday") #=> false + + +class Parent + macro inherited + def {{@type.name.downcase.id}} + 1 + end + end +end + +class Child < Parent +end + +Child.new.child #=> 1 + + +macro method_missing(name, args, block) + print "Got ", {{name.id.stringify}}, " with ", {{args.size}}, " arguments", '\n' +end + +foo # Prints: Got foo with 0 arguments +bar 'a', 'b' # Prints: Got bar with 2 arguments + + +sizeof(Int32) #=> 4 +sizeof(Int64) #=> 8 + + +# On a 64 bits machine +sizeof(Pointer(Int32)) #=> 8 +sizeof(String) #=> 8 + + +a = 1 +sizeof(typeof(a)) #=> 4 + + +class Foo + macro emphasize(value) + "***#{ {{value}} }***" + end + + def yield_with_self + with self yield + end +end + +Foo.new.yield_with_self { emphasize(10) } #=> "***10***" + + +# This generates: +# +# def :foo +# 1 +# end +define_method :foo, 1 + + +macro define_method(name, content) + def {{name.id}} + {{content}} + end +end + +# This correctly generates: +# +# def foo +# 1 +# end +define_method :foo, 1 + + +macro define_method(name, content) + def {{name}} + {% if content == 1 %} + "one" + {% else %} + {{content}} + {% end %} + end +end + +define_method foo, 1 +define_method bar, 2 + +foo #=> one +bar #=> 2 + + +{% if env("TEST") %} + puts "We are in test mode" +{% end %} + + +macro define_dummy_methods(names) + {% for name, index in names %} + def {{name.id}} + {{index}} + end + {% end %} +end + +define_dummy_methods [foo, bar, baz] + +foo #=> 0 +bar #=> 1 +baz #=> 2 + + +macro define_dummy_methods(hash) + {% for key, value in hash %} + def {{key.id}} + {{value}} + end + {% end %} +end +define_dummy_methods({foo: 10, bar: 20}) +foo #=> 10 +bar #=> 20 + + +{% for name, index in ["foo", "bar", "baz"] %} + def {{name.id}} + {{index}} + end +{% end %} + +foo #=> 0 +bar #=> 1 +baz #=> 2 + + +macro define_dummy_methods(*names) + {% for name, index in names %} + def {{name.id}} + {{index}} + end + {% end %} +end + +define_dummy_methods foo, bar, baz + +foo #=> 0 +bar #=> 1 +baz #=> 2 + + +macro println(*values) + print {{*values}}, '\n' +end + +println 1, 2, 3 # outputs 123\n + + +VALUES = [1, 2, 3] + +{% for value in VALUES %} + puts {{value}} +{% end %} + + +until some_condition + do_this +end + +# The above is the same as: +while !some_condition + do_this +end + + +a = some_condition ? nil : 3 +# a is Int32 or Nil + +if a + # Since the only way to get here is if a is truthy, + # a can't be nil. So here a is Int32. + a.abs +end + + +if a = some_expression + # here a is not nil +end + + +if a && b + # here both a and b are guaranteed not to be Nil +end + + +if @a + # here @a can be nil +end + + +# First option: assign it to a variable +if a = @a + # here a can't be nil +end + +# Second option: use `Object#try` found in the standard library +@a.try do |a| + # here a can't be nil +end + + +if method # first call to a method that can return Int32 or Nil + # here we know that the first call did not return Nil + method # second call can still return Int32 or Nil +end + + +class Person + def become_older(by = 1) + @age += by + end +end + +john = Person.new "John" +john.age #=> 0 + +john.become_older +john.age #=> 1 + +john.become_older 2 +john.age #=> 3 + + +john.become_older by: 5 + + +def some_method(x, y = 1, z = 2, w = 3) + # do something... +end + +some_method 10 # x = 10, y = 1, z = 2, w = 3 +some_method 10, z: 10 # x = 10, y = 1, z = 10, w = 3 +some_method 10, w: 1, y: 2, z: 3 # x = 10, y = 2, z = 3, w = 1 + + +case exp +when value1, value2 + do_something +when value3 + do_something_else +else + do_another_thing +end + + +case var +when String + # var : String + do_something +when Int32 + # var : Int32 + do_something_else +else + # here var is neither a String nor an Int32 + do_another_thing +end + + +case num +when .even? + do_something +when .odd? + do_something_else +end + + +case +when cond1, cond2 + do_something +when cond3 + do_something_else +end + + +a = 1 +a.responds_to?(:abs) #=> true +a.responds_to?(:size) #=> false + + +foo_or_bar = /foo|bar/ +heeello = /h(e+)llo/ +integer = /\d+/ + + +r = /foo/imx + + +slash = /\// + + +r = %r(regex with slash: /) + + +"hello world" + + +"\"" # double quote +"\\" # backslash +"\e" # escape +"\f" # form feed +"\n" # newline +"\r" # carriage return +"\t" # tab +"\v" # vertical tab + + +"\101" # == "A" +"\123" # == "S" +"\12" # == "\n" +"\1" # string with one character with code point 1 + + +"\u0041" # == "A" + + +"\u{41}" # == "A" +"\u{1F52E}" # == "🔮" + + +"hello + world" # same as "hello\n world" + + +"hello " \ +"world, " \ +"no newlines" # same as "hello world, no newlines" + + +"hello \ + world, \ + no newlines" # same as "hello world, no newlines" + + +# Supports double quotes and nested parenthesis +%(hello ("world")) # same as "hello (\"world\")" + +# Supports double quotes and nested brackets +%[hello ["world"]] # same as "hello [\"world\"]" + +# Supports double quotes and nested curlies +%{hello {"world"}} # same as "hello {\"world\"}" + +# Supports double quotes and nested angles +%<hello <"world">> # same as "hello <\"world\">" + + +<<-XML +<parent> + <child /> +</parent> +XML + + +# Same as "Hello\n world" +<<-STRING + Hello + world + STRING + +# Same as " Hello\n world" +<<-STRING + Hello + world + STRING + + +a = 1 +b = 2 +"sum = #{a + b}" # "sum = 3" + + +1.0 # Float64 +1.0_f32 # Float32 +1_f32 # Float32 + +1e10 # Float64 +1.5e10 # Float64 +1.5e-7 # Float64 + ++1.3 # Float64 +-0.5 # Float64 + + +1_000_000.111_111 # better than 1000000.111111 + + +'a' +'z' +'0' +'_' +'あ' + + +'\'' # single quote +'\\' # backslash +'\e' # escape +'\f' # form feed +'\n' # newline +'\r' # carriage return +'\t' # tab +'\v' # vertical tab + + +'\101' # == 'A' +'\123' # == 'S' +'\12' # == '\n' +'\1' # code point 1 + + +'\u0041' # == 'A' + + +'\u{41}' # == 'A' +'\u{1F52E}' # == '🔮' + + +{1 => 2, 3 => 4} # Hash(Int32, Int32) +{1 => 2, 'a' => 3} # Hash(Int32 | Char, Int32) + + +{} of Int32 => Int32 # same as Hash(Int32, Int32).new + + +{key1: 'a', key2: 'b'} # Hash(Symbol, Char) + + +{"key1": 'a', "key2": 'b'} # Hash(String, Char) + + +MyType{"foo": "bar"} + + +tmp = MyType.new +tmp["foo"] = "bar" +tmp + + +tmp = MyType(typeof("foo"), typeof("bar")).new +tmp["foo"] = "bar" +tmp + + +MyType(String, String) {"foo": "bar"} + + +:hello +:good_bye + +# With spaces and symbols +:"symbol with spaces" + +# Ending with question and exclamation marks +:question? +:exclamation! + +# For the operators +:+ +:- +:* +:/ +:== +:< +:<= +:> +:>= +:! +:!= +:=~ +:!~ +:& +:| +:^ +:~ +:** +:>> +:<< +:% +:[] +:[]? +:[]= +:<=> +:=== + + +x..y # an inclusive range, in mathematics: [x, y] +x...y # an exclusive range, in mathematics: [x, y) + + +# A proc without arguments +->{ 1 } # Proc(Int32) + +# A proc with one argument +->(x : Int32) { x.to_s } # Proc(Int32, String) + +# A proc with two arguments: +->(x : Int32, y : Int32) { x + y } # Proc(Int32, Int32, Int32) + + +Proc(Int32, String).new { |x| x.to_s } # Proc(Int32, String) + + +proc = ->(x : Int32, y : Int32) { x + y } +proc.call(1, 2) #=> 3 + + +def one + 1 +end + +proc = ->one +proc.call #=> 1 + + +def plus_one(x) + x + 1 +end + +proc = ->plus_one(Int32) +proc.call(41) #=> 42 + + +str = "hello" +proc = ->str.count(Char) +proc.call('e') #=> 1 +proc.call('l') #=> 2 + + +tuple = {1, "hello", 'x'} # Tuple(Int32, String, Char) +tuple[0] #=> 1 (Int32) +tuple[1] #=> "hello" (String) +tuple[2] #=> 'x' (Char) + + +[1, 2, 3] # Array(Int32) +[1, "hello", 'x'] # Array(Int32 | String | Char) + + +[] of Int32 # same as Array(Int32).new + + +%w(one two three) # ["one", "two", "three"] + + +%i(one two three) # [:one, :two, :three] + + +MyType{1, 2, 3} + + +tmp = MyType.new +tmp << 1 +tmp << 2 +tmp << 3 +tmp + + +tmp = MyType(typeof(1, 2, 3)).new +tmp << 1 +tmp << 2 +tmp << 3 +tmp + + +MyType(Int32 | String) {1, 2, "foo"} + + +nil + + +1 # Int32 + +1_i8 # Int8 +1_i16 # Int16 +1_i32 # Int32 +1_i64 # Int64 + +1_u8 # UInt8 +1_u16 # UInt16 +1_u32 # UInt32 +1_u64 # UInt64 + ++10 # Int32 +-20 # Int32 + +2147483648 # Int64 +9223372036854775808 # UInt64 + + +1_000_000 # better than 1000000 + + +0b1101 # == 13 + + +0o123 # == 83 + + +0xFE012D # == 16646445 +0xfe012d # == 16646445 + + +true # A Bool that is true +false # A Bool that is false + + +a = 1 + +ptr = pointerof(a) +ptr.value = 2 + +a #=> 2 + + +class Point + def initialize(@x, @y) + end + + def x + @x + end + + def x_ptr + pointerof(@x) + end +end + +point = Point.new 1, 2 + +ptr = point.x_ptr +ptr.value = 10 + +point.x #=> 10 + + +def add(x : Number, y : Number) + x + y +end + +# Ok +add 1, 2 # Ok + +# Error: no overload matches 'add' with types Bool, Bool +add true, false + + +def add(x, y) + x + y +end + +add true, false + + +# A class that has a + method but isn't a Number +class Six + def +(other) + 6 + other + end +end + +# add method without type restrictions +def add(x, y) + x + y +end + +# OK +add Six.new, 10 + +# add method with type restrictions +def restricted_add(x : Number, y : Number) + x + y +end + +# Error: no overload matches 'restricted_add' with types Six, Int32 +restricted_add Six.new, 10 + + +class Person + def ==(other : self) + other.name == name + end + + def ==(other) + false + end +end + +john = Person.new "John" +another_john = Person.new "John" +peter = Person.new "Peter" + +john == another_john #=> true +john == peter #=> false (names differ) +john == 1 #=> false (because 1 is not a Person) + + +class Person + def self.compare(p1 : self, p2 : self) + p1.name == p2.name + end +end + +john = Person.new "John" +peter = Person.new "Peter" + +Person.compare(john, peter) # OK + + +def foo(x : Int32) +end + +foo 1 # OK +foo "hello" # Error + + +def foo(x : Int32.class) +end + +foo Int32 # OK +foo String # Error + + +def foo(x : Int32.class) + puts "Got Int32" +end + +def foo(x : String.class) + puts "Got String" +end + +foo Int32 # prints "Got Int32" +foo String # prints "Got String" + + +def foo(*args : Int32) +end + +def foo(*args : String) +end + +foo 1, 2, 3 # OK, invokes first overload +foo "a", "b", "c" # OK, invokes second overload +foo 1, 2, "hello" # Error +foo() # Error + + +def foo + # This is the empty-tuple case +end + + +def foo(x : T) + T +end + +foo(1) #=> Int32 +foo("hello") #=> String + + +def foo(x : Array(T)) + T +end + +foo([1, 2]) #=> Int32 +foo([1, "a"]) #=> (Int32 | String) + + +def foo(x : T.class) + Array(T) +end + +foo(Int32) #=> Array(Int32) +foo(String) #=> Array(String) + + +class Person + # Increases age by one + def become_older + @age += 1 + end + + # Increases age by the given number of years + def become_older(years : Int32) + @age += years + end + + # Increases age by the given number of years, as a String + def become_older(years : String) + @age += years.to_i + end + + # Yields the current age of this person and increases + # its age by the value returned by the block + def become_older + @age += yield @age + end +end + +person = Person.new "John" + +person.become_older +person.age #=> 1 + +person.become_older 5 +person.age #=> 6 + +person.become_older "12" +person.age #=> 18 + +person.become_older do |current_age| + current_age < 20 ? 10 : 30 +end +person.age #=> 28 + + +a = 1 +a.is_a?(Int32) #=> true +a.is_a?(String) #=> false +a.is_a?(Number) #=> true +a.is_a?(Int32 | String) #=> true + + +# One for each thread +@[ThreadLocal] +$values = [] of Int32 + + +@[AlwaysInline] +def foo + 1 +end + + +@[NoInline] +def foo + 1 +end + + +lib LibFoo + @[CallConvention("X86_StdCall")] + fun foo : Int32 +end + + +def sum(*elements) + total = 0 + elements.each do |value| + total += value + end + total +end + +# elements is Tuple(Int32, Int32, Int32, Float64) +sum 1, 2, 3, 4.5 + + +if a.responds_to?(:abs) + # here a's type will be reduced to those responding to the 'abs' method +end + + +a = some_condition ? 1 : "hello" +# a : Int32 | String + +if a.responds_to?(:abs) + # here a will be Int32, since Int32#abs exists but String#abs doesn't +else + # here a will be String +end + + +if (a = @a).responds_to?(:abs) + # here a is guaranteed to respond to `abs` +end + + +def capture(&block) + block +end + +def invoke(&block) + block.call +end + +proc = capture { puts "Hello" } +invoke(&proc) # prints "Hello" + + + + +def capture(&block) + block +end + +def twice + yield + yield +end + +proc = capture { puts "Hello" } +twice &proc + + +twice &->{ puts "Hello" } + + +def say_hello + puts "Hello" +end + +twice &->say_hello + + +def foo + yield 1 +end + +def wrap_foo + puts "Before foo" + foo do |x| + yield x + end + puts "After foo" +end + +wrap_foo do |i| + puts i +end + + +def foo + yield 1 +end + +def wrap_foo(&block : Int32 -> _) + puts "Before foo" + foo(&block) + puts "After foo" +end + +wrap_foo do |i| + puts i +end + + +foo_forward do |i| + break # error +end + + +a = 2 +while (a += 1) < 20 + if a == 10 + # goes to 'puts a' + break + end +end +puts a #=> 10 + + +class Person + private def say(message) + puts message + end + + def say_hello + say "hello" # OK, no receiver + self.say "hello" # Error, self is a receiver + + other = Person.new "Other" + other.say "hello" # Error, other is a receiver + end +end + + +class Employee < Person + def say_bye + say "bye" # OK + end +end + + +module Namespace + class Foo + protected def foo + puts "Hello" + end + end + + class Bar + def bar + # Works, because Foo and Bar are under Namespace + Foo.new.foo + end + end +end + +Namespace::Bar.new.bar + + +class Person + protected def self.say(message) + puts message + end + + def say_hello + Person.say "hello" + end +end + + +buffer = uninitialized UInt8[256] diff --git a/tests/examplefiles/test.erl b/tests/examplefiles/test.erl index 5b983e75..d4ab4825 100644 --- a/tests/examplefiles/test.erl +++ b/tests/examplefiles/test.erl @@ -152,6 +152,18 @@ a_binary() -> a_list_comprehension() -> [X*2 || X <- [1,2,3]]. +a_map() -> + M0 = #{ a => 1, b => 2 }, + M1 = M0#{ b := 200 }. + +escape_sequences() -> + [ "\b\d\e\f\n\r\s\t\v\'\"\\" + , "\1\12\123" % octal + , "\x01" % short hex + , "\x{fff}" % long hex + , "\^a\^A" % control characters + ]. + map(Fun, [H|T]) -> [Fun(H) | map(Fun, T)]; diff --git a/tests/examplefiles/test.escript b/tests/examplefiles/test.escript new file mode 100644 index 00000000..3fafb803 --- /dev/null +++ b/tests/examplefiles/test.escript @@ -0,0 +1,4 @@ +#!/usr/bin/env escript + +main(_Args) -> + ok. diff --git a/tests/examplefiles/test.hsail b/tests/examplefiles/test.hsail new file mode 100644 index 00000000..f9c25091 --- /dev/null +++ b/tests/examplefiles/test.hsail @@ -0,0 +1,62 @@ +module &__llvm_hsail_module:1:0:$full:$large:$near; + +prog kernel &mmul2d( + kernarg_u64 %__arg_p0, + kernarg_u64 %__arg_p1, + kernarg_u64 %__arg_p2, + kernarg_u64 %__arg_p3) +{ + pragma "AMD RTI", "ARGSTART:mmul2d"; + pragma "AMD RTI", "version:3:1:104"; + pragma "AMD RTI", "device:generic"; + pragma "AMD RTI", "uniqueid:1025"; + pragma "AMD RTI", "function:1:0"; + pragma "AMD RTI", "memory:64bitABI"; + pragma "AMD RTI", "privateid:1"; + pragma "AMD RTI", "ARGEND:mmul2d"; + // BB#0: // %top + mov_f64 $d1, 0.0E+0; + gridsize_u32 $s0, 0; + workitemabsid_u32 $s1, 1; + workitemabsid_u32 $s2, 0; + cvt_u64_u32 $d0, $s2; + cvt_u64_u32 $d3, $s1; + cvt_u64_u32 $d4, $s0; + ld_kernarg_align(8)_width(all)_u64 $d2, [%__arg_p2]; + ld_kernarg_align(8)_width(all)_u64 $d6, [%__arg_p1]; + ld_kernarg_align(8)_width(all)_u64 $d5, [%__arg_p3]; + ld_kernarg_align(8)_width(all)_u64 $d7, [%__arg_p0]; + cmp_lt_b1_s64 $c0, $d5, 1; + cbr_b1 $c0, @BB0_3; + // BB#1: // %L.preheader + mul_u64 $d1, $d5, $d3; + shl_u64 $d1, $d1, 3; + shl_u64 $d8, $d0, 3; + add_u64 $d8, $d7, $d8; + add_u64 $d6, $d6, $d1; + shl_u64 $d7, $d4, 3; + mov_f64 $d1, 0D0000000000000000; + +@BB0_2: + // %L + add_u64 $d9, $d8, $d7; + ld_global_f64 $d8, [$d8]; + ld_global_f64 $d10, [$d6]; + mul_f64 $d8, $d8, $d10; + add_f64 $d1, $d1, $d8; + add_u64 $d6, $d6, 8; + add_u64 $d5, $d5, 18446744073709551615; + cmp_ne_b1_s64 $c0, $d5, 0; + mov_b64 $d8, $d9; + cbr_b1 $c0, @BB0_2; + +@BB0_3: + // %L.7 + mul_u64 $d3, $d3, $d4; + add_u64 $d0, $d3, $d0; + shl_u64 $d0, $d0, 3; + add_u64 $d0, $d2, $d0; + st_global_f64 $d1, [$d0]; + ret; +}; + diff --git a/tests/examplefiles/test.mt b/tests/examplefiles/test.mt new file mode 100644 index 00000000..008dc88e --- /dev/null +++ b/tests/examplefiles/test.mt @@ -0,0 +1,7 @@ +exports (main) + +def main(=> currentProcess) :Int as DeepFrozen: + traceln(`Current process: $currentProcess`) + "A \r \n \x00 \u1234" + '\u1234' + return 0 diff --git a/tests/examplefiles/test.ncl b/tests/examplefiles/test.ncl new file mode 100644 index 00000000..f20f8159 --- /dev/null +++ b/tests/examplefiles/test.ncl @@ -0,0 +1,20 @@ +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +begin + int_num = 1 + float_num = 0.1 + str = "A void map" + array = (/1, 2, 3, 4, 5/) + + + wks = gsn_open_wks("X11", "test_for_pygments") + + res = True + res@mpMinLonF = 90. + res@mpMaxLonF = 180. + res@mpMinLatF = 0. + res@mpMaxLatF = 90. + + plot = gsn_csm_map_ce(wks, res) +end
\ No newline at end of file diff --git a/tests/examplefiles/test.php b/tests/examplefiles/test.php index 2ce4023e..e8efdc6a 100644 --- a/tests/examplefiles/test.php +++ b/tests/examplefiles/test.php @@ -505,11 +505,40 @@ function &byref() { return $x; } +// Test highlighting of magic methods and variables +class MagicClass { + public $magic_str; + public $ordinary_str; + + public function __construct($some_var) { + $this->magic_str = __FILE__; + $this->ordinary_str = $some_var; + } + + public function __toString() { + return $this->magic_str; + } + + public function nonMagic() { + return $this->ordinary_str; + } +} + +$magic = new MagicClass(__DIR__); +__toString(); +$magic->nonMagic(); +$magic->__toString(); + echo <<<EOF Test the heredocs... EOF; +echo <<<"some_delimiter" +more heredoc testing +continues on this line +some_delimiter; + ?> diff --git a/tests/examplefiles/test.sil b/tests/examplefiles/test.sil new file mode 100644 index 00000000..3bcee835 --- /dev/null +++ b/tests/examplefiles/test.sil @@ -0,0 +1,206 @@ +domain Option__Node { + unique function Option__Node__Some(): Option__Node + unique function Option__Node__None(): Option__Node + + function variantOfOptionNode(self: Ref): Option__Node + + function isOptionNode(self: Ref): Bool + + axiom ax_variantOfOptionNodeChoices { + forall x: Ref :: { variantOfOptionNode(x) } + (variantOfOptionNode(x) == Option__Node__Some() || variantOfOptionNode(x) == Option__Node__None()) + } + + axiom ax_isCounterState { + forall x: Ref :: { variantOfOptionNode(x) } + isOptionNode(x) == (variantOfOptionNode(x) == Option__Node__Some() || + variantOfOptionNode(x) == Option__Node__None()) + } +} + +predicate validOption(this: Ref) { + isOptionNode(this) && + variantOfOptionNode(this) == Option__Node__Some() ==> ( + acc(this.Option__Node__Some__1, write) && + acc(validNode(this.Option__Node__Some__1)) + ) +} + +field Option__Node__Some__1: Ref + +field Node__v: Int +field Node__next: Ref + +predicate validNode(this: Ref) { + acc(this.Node__v) && + acc(this.Node__next) && + acc(validOption(this.Node__next)) +} + + +function length(this: Ref): Int + requires acc(validNode(this), write) + ensures result >= 1 +{ + (unfolding acc(validNode(this), write) in + unfolding acc(validOption(this.Node__next)) in + (variantOfOptionNode(this.Node__next) == Option__Node__None()) ? + 1 : 1 + length(this.Node__next.Option__Node__Some__1) + ) +} + +function itemAt(this: Ref, i: Int): Int + requires acc(validNode(this), write) + requires 0 <= i && i < length(this) +{ + unfolding acc(validNode(this), write) in unfolding acc(validOption(this.Node__next)) in ( + (i == 0) ? + this.Node__v: + (variantOfOptionNode(this.Node__next) == Option__Node__Some()) ? + itemAt(this.Node__next.Option__Node__Some__1, i-1) : this.Node__v + ) +} + +function sum(this$1: Ref): Int + requires acc(validNode(this$1), write) +{ + (unfolding acc(validNode(this$1), write) in unfolding acc(validOption(this$1.Node__next)) in + (variantOfOptionNode(this$1.Node__next) == Option__Node__None()) ? this$1.Node__v : this$1.Node__v + sum(this$1.Node__next.Option__Node__Some__1)) +} + +method append(this: Ref, val: Int) + requires acc(validNode(this), write) + ensures acc(validNode(this), write) /* POST1 */ + ensures length(this) == (old(length(this)) + 1) /* POST2 */ + ensures (forall i: Int :: (0 <= i && i < old(length(this))) ==> (itemAt(this, i) == old(itemAt(this, i)))) /* POST3 */ + ensures itemAt(this, length(this) - 1) == val /* POST4 */ + ensures true ==> true +{ + var tmp_node: Ref + var tmp_option: Ref + + unfold acc(validNode(this), write) + unfold acc(validOption(this.Node__next), write) + + if (variantOfOptionNode(this.Node__next) == Option__Node__None()) { + tmp_node := new(Node__next, Node__v) + tmp_node.Node__next := null + tmp_node.Node__v := val + + assume variantOfOptionNode(tmp_node.Node__next) == Option__Node__None() + fold acc(validOption(tmp_node.Node__next)) + fold acc(validNode(tmp_node), write) + + tmp_option := new(Option__Node__Some__1) + tmp_option.Option__Node__Some__1 := tmp_node + assume variantOfOptionNode(tmp_option) == Option__Node__Some() + fold acc(validOption(tmp_option)) + + this.Node__next := tmp_option + + + unfold validOption(tmp_option) + assert length(tmp_node) == 1 /* TODO: Required by Silicon, POST2 fails otherwise */ + assert itemAt(tmp_node, 0) == val /* TODO: Required by Silicon, POST4 fails otherwise */ + fold validOption(tmp_option) + } else { + append(this.Node__next.Option__Node__Some__1, val) + fold acc(validOption(this.Node__next), write) + } + + fold acc(validNode(this), write) +} + +method prepend(tail: Ref, val: Int) returns (res: Ref) + requires acc(validNode(tail)) + ensures acc(validNode(res)) + //ensures acc(validNode(tail)) + ensures length(res) == old(length(tail)) + 1 + + ensures (forall i: Int :: (1 <= i && i < length(res)) ==> (itemAt(res, i) == old(itemAt(tail, i-1)))) /* POST3 */ + ensures itemAt(res, 0) == val +{ + var tmp_option: Ref + + res := new(Node__v, Node__next) + res.Node__v := val + + tmp_option := new(Option__Node__Some__1) + tmp_option.Option__Node__Some__1 := tail + assume variantOfOptionNode(tmp_option) == Option__Node__Some() + + res.Node__next := tmp_option + + assert acc(validNode(tail)) + fold acc(validOption(res.Node__next)) + fold acc(validNode(res)) +} + +method length_iter(list: Ref) returns (len: Int) + requires acc(validNode(list), write) + ensures old(length(list)) == len + // TODO we have to preserve this property + // ensures acc(validNode(list)) +{ + var curr: Ref := list + var tmp: Ref := list + + len := 1 + + unfold acc(validNode(curr)) + unfold acc(validOption(curr.Node__next)) + while(variantOfOptionNode(curr.Node__next) == Option__Node__Some()) + invariant acc(curr.Node__v) + invariant acc(curr.Node__next) + invariant (variantOfOptionNode(curr.Node__next) == Option__Node__Some() ==> ( + acc(curr.Node__next.Option__Node__Some__1, write) && + acc(validNode(curr.Node__next.Option__Node__Some__1)) + )) + invariant (variantOfOptionNode(curr.Node__next) == Option__Node__Some() ==> len + length(curr.Node__next.Option__Node__Some__1) == old(length(list))) + invariant (variantOfOptionNode(curr.Node__next) == Option__Node__None() ==> len == old(length(list))) + { + assert acc(validNode(curr.Node__next.Option__Node__Some__1)) + len := len + 1 + tmp := curr + curr := curr.Node__next.Option__Node__Some__1 + unfold acc(validNode(curr)) + unfold acc(validOption(curr.Node__next)) + } +} + +method t1() +{ + var l: Ref + + l := new(Node__v, Node__next) + l.Node__next := null + l.Node__v := 1 + assume variantOfOptionNode(l.Node__next) == Option__Node__None() + + fold validOption(l.Node__next) + fold validNode(l) + + assert length(l) == 1 + assert itemAt(l, 0) == 1 + + append(l, 7) + assert itemAt(l, 1) == 7 + assert itemAt(l, 0) == 1 + assert length(l) == 2 + + l := prepend(l, 10) + assert itemAt(l, 2) == 7 + assert itemAt(l, 1) == 1 + assert itemAt(l, 0) == 10 + assert length(l) == 3 + + //assert sum(l) == 18 +} + +method t2(l: Ref) returns (res: Ref) + requires acc(validNode(l), write) + ensures acc(validNode(res), write) + ensures length(res) > old(length(l)) +{ + res := prepend(l, 10) +} diff --git a/tests/examplefiles/tsql_example.sql b/tests/examplefiles/tsql_example.sql new file mode 100644 index 00000000..cbd76091 --- /dev/null +++ b/tests/examplefiles/tsql_example.sql @@ -0,0 +1,72 @@ +-- Example Transact-SQL file. + +-- Single line comment +/* A comment + * spawning two lines. */ + /* An indented comment + * spawning multiple + * lines. */ +/* A /* nested */ comment. */ + +select + left(emp.firstname, 1) + '.' + [emp.surname] as "Name", + dep.name as [Department] +into + #temp_employee +from + employee as emp + inner join department as dep on + dep.ident_code = emp.department_id +where + emp.date_of_birth >= '1990-01-01'; +go + +declare @TextToFind nvarchar(100) = N'some +text across +multiple lines'; + +set @TextToFind varchar(32) = 'hello' + ' world'; +set @TextTiFind += '!'; + +declare @Count int = 17 * (3 - 5); + +delete from + [server].[database].[schema].[table] +where + [Text] = @TextToFind and author Not LIKE '%some%'; + +goto overthere; +overthere: + +select + 123 as "int 1", + +123 as "int 2", + -123 as "int 3", + 0x20 as "hex int", + 123.45 as "float 1", + -1.23e45 as "float 2" + +1.23E+45 as "float 3", + -1.23e-45 as "float 4", + 1. as "float 5", + .1 as "float 6", + 1.e2 as "float 7", + .1e2 as "float 8"; + +Select @@Error, $PARTITion.RangePF1(10); + +select top 3 Ähnliches from Müll; + +-- Example transaction +BEGIN TRAN + +BEGIN TRY + INSERT INTO #temp_employe(Name, Department) VALUES ('L. Miller', 'Sales') + iNsErT inTO #temp_employe(Name, Department) VaLuEs ('M. Webster', 'Helpdesk') + COMMIT TRAN +END TRY +BEGIN CATCH + print 'cannot perform transaction; rolling back'; + ROLLBACK TRAN +END CATCH + +-- Comment at end without newline.
\ No newline at end of file diff --git a/tests/examplefiles/example.ts b/tests/examplefiles/typescript_example index 760e2543..760e2543 100644 --- a/tests/examplefiles/example.ts +++ b/tests/examplefiles/typescript_example diff --git a/tests/examplefiles/typoscript_example b/tests/examplefiles/typoscript_example new file mode 100644 index 00000000..e2fccf5d --- /dev/null +++ b/tests/examplefiles/typoscript_example @@ -0,0 +1,1930 @@ +# *************************************************************************** +# Notice: "styles." (and "temp.") objects are UNSET after template parsing! +# Use "lib." for persisting storage of objects. +# *************************************************************************** + +<INCLUDE_TYPOSCRIPT: source="FILE: EXT:www_tue_nl/Configuration/TypoScript/Setup/Root.ts"> + +page.80 = RECORDS +page.80 { + source = 1 + tables = tt_address + conf.tt_address = COA + conf.tt_address { + 20 = TEXT + 20.field = email + 20.typolink.parameter.field = email + } +} + + /* +page.200 = PHP_SCRIPT_EXT +page.200 { + 1 = TMENU + 1.wrap = <div style="width:200px; border: 1px solid;">|</div> + 1.expAll = 1 + 1.submenuObjSuffixes = a |*| |*| b + 1.NO.allWrap = <b>|</b><br/> + + 2 = TMENU + 2.NO.allWrap = <div style="background:red;">|</div> + + 2a = TMENU + 2a.NO.allWrap = <div style="background:yellow;">|</div> +* + 2b = TMENU + 2b.NO.allWrap = <div style="background:green;">|</div> +} +*/ + + # Add the CSS and JS files +page { + includeCSS { # comment at the end of a line + file99 = fileadmin/your-fancybox.css + } + includeJSFooter { + fancybox = fileadmin/your-fancybox.js + } +} + + # Change the default rendering of images to match lightbox requirements +tt_content.image.20.1.imageLinkWrap { + JSwindow = 0 + test = MyExtension\Path\To\Class + + directImageLink = 1 + linkParams.ATagParams { + dataWrap = class= "lightbox" rel="fancybox{field:uid}" + } +} + +tt_content.image.20.1.imageLinkWrap > +tt_content.image.20.1.imageLinkWrap = 1 +tt_content.image.20.1.imageLinkWrap { + enable = 1 + typolink { + # directly link to the recent image + parameter.cObject = IMG_RESOURCE + parameter.cObject.file.import.data = TSFE:lastImageInfo|origFile + parameter.cObject.file.maxW = {$styles.content.imgtext.maxW} + parameter.override.listNum.stdWrap.data = register : IMAGE_NUM_CURRENT + title.field = imagecaption // title + title.split.token.char = 10 + title.if.isTrue.field = imagecaption // header + title.split.token.char = 10 + title.split.returnKey.data = register : IMAGE_NUM_CURRENT + parameter.cObject = IMG_RESOURCE + parameter.cObject.file.import.data = TSFE:lastImageInfo|origFile + ATagParams = target="_blank" + } +} + +10 = IMAGE +10 { + # point to the image + file = fileadmin/demo/lorem_ipsum/images/a4.jpg + # make it rather small + file.width = 80 + # add a link to tx_cms_showpic.php that shows the original image + imageLinkWrap = 1 + imageLinkWrap { + enable = 1 + # JSwindow = 1 + } +} + +# Clear out any constants in this reserved room! +styles.content > + +# get content +styles.content.get = CONTENT +styles.content.get { + table = tt_content + select.orderBy = sorting + select.where = colPos=0 + select.languageField = sys_language_uid +} + +# get content, left +styles.content.getLeft < styles.content.get +styles.content.getLeft.select.where = colPos=1 + +# get content, right +styles.content.getRight < styles.content.get +styles.content.getRight.select.where = colPos=2 + +# get content, margin +styles.content.getBorder < styles.content.get +styles.content.getBorder.select.where = colPos=3 + +# get news +styles.content.getNews < styles.content.get +styles.content.getNews.select.pidInList = {$styles.content.getNews.newsPid} + +# Edit page object: +styles.content.editPanelPage = COA +styles.content.editPanelPage { + 10 = EDITPANEL + 10 { + allow = toolbar,move,hide + label.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.page + label.wrap = | <b>%s</b> + } +} + + + + + + + + + + + +# ********************************************************************* +# "lib." objects are preserved from unsetting after template parsing +# ********************************************************************* + +# Creates persistent ParseFunc setup for non-HTML content. This is recommended to use (as a reference!) +lib.parseFunc { + makelinks = 1 + makelinks.http.keep = {$styles.content.links.keep} + makelinks.http.extTarget = {$styles.content.links.extTarget} + makelinks.mailto.keep = path + tags { + link = TEXT + link { + current = 1 + typolink.parameter.data = parameters : allParams + typolink.extTarget = {$styles.content.links.extTarget} + typolink.target = {$styles.content.links.target} + parseFunc.constants =1 + } + } + allowTags = {$styles.content.links.allowTags} + denyTags = * + sword = <span class="csc-sword">|</span> + constants = 1 + + nonTypoTagStdWrap.HTMLparser = 1 + nonTypoTagStdWrap.HTMLparser { + keepNonMatchedTags = 1 + htmlSpecialChars = 2 + } +} + +# good old parsefunc in "styles.content.parseFunc" is created for backwards compatibility. Don't use it, just ignore. +styles.content.parseFunc < lib.parseFunc + +# Creates persistent ParseFunc setup for RTE content (which is mainly HTML) based on the "ts_css" transformation. +lib.parseFunc_RTE < lib.parseFunc +lib.parseFunc_RTE { + // makelinks > + # Processing <table> and <blockquote> blocks separately + externalBlocks = table, blockquote, dd, dl, ol, ul, div + externalBlocks { + # The blockquote content is passed into parseFunc again... + blockquote.stripNL=1 + blockquote.callRecursive=1 + blockquote.callRecursive.tagStdWrap.HTMLparser = 1 + blockquote.callRecursive.tagStdWrap.HTMLparser.tags.blockquote.overrideAttribs = style="margin-bottom:0;margin-top:0;" + + ol.stripNL=1 + ol.stdWrap.parseFunc = < lib.parseFunc + + ul.stripNL=1 + ul.stdWrap.parseFunc = < lib.parseFunc + + table.stripNL=1 + table.stdWrap.HTMLparser = 1 + table.stdWrap.HTMLparser.tags.table.fixAttrib.class { + default = contenttable + always = 1 + list = contenttable + } + table.stdWrap.HTMLparser.keepNonMatchedTags = 1 + table.HTMLtableCells=1 + table.HTMLtableCells { + default.callRecursive=1 + addChr10BetweenParagraphs=1 + } + div.stripNL = 1 + div.callRecursive = 1 + + # Definition list processing + dl < .div + dd < .div + } + nonTypoTagStdWrap.encapsLines { + encapsTagList = p,pre,h1,h2,h3,h4,h5,h6,hr,dt + remapTag.DIV = P + nonWrappedTag = P + innerStdWrap_all.ifBlank = + addAttributes.P.class = bodytext + addAttributes.P.class.setOnly=blank + } + nonTypoTagStdWrap.HTMLparser = 1 + nonTypoTagStdWrap.HTMLparser { + keepNonMatchedTags = 1 + htmlSpecialChars = 2 + } +} + + +# Content header: +lib.stdheader = COA +lib.stdheader { + + # Create align style-attribute for <Hx> tags + 2 = LOAD_REGISTER + 2.headerStyle.field = header_position + 2.headerStyle.required = 1 + 2.headerStyle.noTrimWrap = | style="text-align:|;"| + + # Create class="csc-firstHeader" attribute for <Hx> tags + 3 = LOAD_REGISTER + 3.headerClass = csc-firstHeader + 3.headerClass.if.value=1 + 3.headerClass.if.equals.data = cObj:parentRecordNumber + 3.headerClass.noTrimWrap = | class="|"| + + # Date format: + 5 = TEXT + 5.field = date + 5.if.isTrue.field = date + 5.strftime = %x + 5.wrap = <p class="csc-header-date">|</p> + 5.prefixComment = 2 | Header date: + + # This CASE cObject renders the header content: + # currentValue is set to the header data, possibly wrapped in link-tags. + 10 = CASE + 10.setCurrent { + field = header + htmlSpecialChars = 1 + typolink.parameter.field = header_link + } + 10.key.field = header_layout + 10.key.ifEmpty = {$content.defaultHeaderType} + 10.key.ifEmpty.override.data = register: defaultHeaderType + + 10.1 = TEXT + 10.1.current = 1 + 10.1.dataWrap = <h1{register:headerStyle}{register:headerClass}>|</h1> + + 10.2 < .10.1 + 10.2.dataWrap = <h2{register:headerStyle}{register:headerClass}>|</h2> + + 10.3 < .10.1 + 10.3.dataWrap = <h3{register:headerStyle}{register:headerClass}>|</h3> + + 10.4 < .10.1 + 10.4.dataWrap = <h4{register:headerStyle}{register:headerClass}>|</h4> + + 10.5 < .10.1 + 10.5.dataWrap = <h5{register:headerStyle}{register:headerClass}>|</h5> + + # Pops the used registers off the stack: + 98 = RESTORE_REGISTER + 99 = RESTORE_REGISTER + + # Post-processing: + stdWrap.fieldRequired = header + stdWrap.if { + equals.field = header_layout + value = 100 + negate = 1 + } + + stdWrap.editIcons = tt_content : header, [header_layout | header_position], [header_link|date] + stdWrap.editIcons.beforeLastTag = 1 + stdWrap.editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.header + + stdWrap.dataWrap = <div class="csc-header csc-header-n{cObj:parentRecordNumber}">|</div> + stdWrap.prefixComment = 2 | Header: +} + + + + + + + + + + + + + + + +#****************************************************** +# Including library for processing of some elements: +#****************************************************** +includeLibs.tx_cssstyledcontent_pi1 = EXT:css_styled_content/pi1/class.tx_cssstyledcontent_pi1.php + + +#********************************** +# tt_content is started +#********************************** +tt_content > +tt_content = CASE +tt_content.key.field = CType +tt_content.stdWrap { + innerWrap.cObject = CASE + innerWrap.cObject { + key.field = section_frame + + default = COA + default { + 10 = TEXT + 10 { + value = <div id="c{field:uid}" + override.cObject = TEXT + override.cObject { + value = <div + if.value = div + if.equals.field = CType + } + insertData = 1 + } + + 15 = TEXT + 15 { + value = csc-default + noTrimWrap = | class="|" | + required = 1 + } + + 20 = COA + 20 { + 10 = COA + 10 { + 10 = TEXT + 10 { + value = {$content.spaceBefore} + wrap = |+ + if.isTrue = {$content.spaceBefore} + } + + 20 = TEXT + 20 { + field = spaceBefore + } + + stdWrap { + prioriCalc = intval + wrap = margin-top:|px; + required = 1 + ifEmpty.value = + } + } + + 20 = COA + 20 { + 10 = TEXT + 10 { + value = {$content.spaceAfter} + wrap = |+ + if.isTrue = {$content.spaceAfter} + } + + 20 = TEXT + 20 { + field = spaceAfter + } + + stdWrap { + prioriCalc = intval + wrap = margin-bottom:|px; + required = 1 + ifEmpty.value = + } + } + + stdWrap.noTrimWrap = | style="|" | + stdWrap.required = 1 + } + 30 = TEXT + 30.value = >|</div> + } + + 1 =< tt_content.stdWrap.innerWrap.cObject.default + 1.15.value = csc-frame csc-frame-invisible + + 5 =< tt_content.stdWrap.innerWrap.cObject.default + 5.15.value = csc-frame csc-frame-rulerBefore + + 6 =< tt_content.stdWrap.innerWrap.cObject.default + 6.15.value = csc-frame csc-frame-rulerAfter + + 10 =< tt_content.stdWrap.innerWrap.cObject.default + 10.15.value = csc-frame csc-frame-indent + + 11 =< tt_content.stdWrap.innerWrap.cObject.default + 11.15.value = csc-frame csc-frame-indent3366 + + 12 =< tt_content.stdWrap.innerWrap.cObject.default + 12.15.value = csc-frame csc-frame-indent6633 + + 20 =< tt_content.stdWrap.innerWrap.cObject.default + 20.15.value = csc-frame csc-frame-frame1 + + 21 =< tt_content.stdWrap.innerWrap.cObject.default + 21.15.value = csc-frame csc-frame-frame2 + + 66 = COA + 66 { + 10 = TEXT + 10 { + value = <a id="c{field:uid}"></a> + insertData = 1 + } + + 20 = COA + 20 { + 10 = TEXT + 10 { + value = {$content.spaceBefore} + wrap = |+ + if.isTrue = {$content.spaceBefore} + } + + 20 = TEXT + 20 { + field = spaceBefore + } + + stdWrap { + prioriCalc = intval + wrap = margin-top:|px; + required = 1 + ifEmpty.value = + wrap2 = <div style="|"></div> + } + } + + 30 = TEXT + 30 { + value = | + } + + 40 < .20 + 40 { + 10 { + value = {$content.spaceAfter} + if.isTrue = {$content.spaceAfter} + } + 20.field = spaceAfter + stdWrap.wrap = margin-bottom:|px; + } + } + + } + + innerWrap2 = | <p class="csc-linkToTop"><a href="#">{LLL:EXT:css_styled_content/pi1/locallang.xml:label.toTop}</a></p> + innerWrap2.insertData = 1 + innerWrap2.fieldRequired = linkToTop + + prepend = TEXT + prepend.dataWrap = <a id="c{field:_LOCALIZED_UID}"></a> + prepend.if.isTrue.field = _LOCALIZED_UID + + editPanel = 1 + editPanel { + allow = move,new,edit,hide,delete + line = 5 + label = %s + onlyCurrentPid = 1 + previewBorder = 4 + edit.displayRecord = 1 + } + + prefixComment = 1 | CONTENT ELEMENT, uid:{field:uid}/{field:CType} +} + + + +# ***************** +# CType: header +# ***************** +# See Object path "lib.stdheader" +tt_content.header = COA +tt_content.header { + 10 = < lib.stdheader + + 20 = TEXT + 20 { + field = subheader + required = 1 + + dataWrap = <p class="csc-subheader csc-subheader-{field:layout}">|</p> + htmlSpecialChars = 1 + + editIcons = tt_content:subheader,layout + editIcons.beforeLastTag = 1 + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.subheader + + prefixComment = 2 | Subheader: + } +} + + + +# ***************** +# CType: text +# ***************** +tt_content.text = COA +tt_content.text { + 10 = < lib.stdheader + + 20 = TEXT + 20 { + field = bodytext + required = 1 + + parseFunc = < lib.parseFunc_RTE + + editIcons = tt_content:bodytext, rte_enabled + editIcons.beforeLastTag = 1 + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.bodytext + + prefixComment = 2 | Text: + } +} + + + +# ***************** +# CType: image +# ***************** +# (also used for rendering 'textpic' type): +tt_content.image = COA +tt_content.image.10 = < lib.stdheader +tt_content.image.20 = USER +tt_content.image.20 { + userFunc = tx_cssstyledcontent_pi1->render_textpic + + # Image source + imgList.field = image + imgPath = uploads/pics/ + + # Single image rendering + imgObjNum = 1 + 1 { + file.import.current = 1 + file.width.field = imagewidth + imageLinkWrap = 1 + imageLinkWrap { + bodyTag = <body style="margin:0; background:#fff;"> + wrap = <a href="javascript:close();"> | </a> + width = {$styles.content.imgtext.linkWrap.width} + height = {$styles.content.imgtext.linkWrap.height} + effects = {$styles.content.imgtext.linkWrap.effects} + + JSwindow = 1 + JSwindow.newWindow = {$styles.content.imgtext.linkWrap.newWindow} + JSwindow.if.isFalse = {$styles.content.imgtext.linkWrap.lightboxEnabled} + + directImageLink = {$styles.content.imgtext.linkWrap.lightboxEnabled} + + enable.field = image_zoom + enable.ifEmpty.typolink.parameter.field = image_link + enable.ifEmpty.typolink.parameter.listNum.splitChar = 10 + enable.ifEmpty.typolink.parameter.listNum.stdWrap.data = register : IMAGE_NUM_CURRENT + enable.ifEmpty.typolink.returnLast = url + + typolink.parameter.field = image_link + typolink.parameter.listNum.splitChar = 10 + typolink.parameter.listNum.stdWrap.data = register : IMAGE_NUM_CURRENT + typolink.target = {$styles.content.links.target} + typolink.extTarget = {$styles.content.links.extTarget} + + linkParams.ATagParams.dataWrap = class="{$styles.content.imgtext.linkWrap.lightboxCssClass}" rel="{$styles.content.imgtext.linkWrap.lightboxRelAttribute}" + } + + altText = TEXT + altText { + field = altText + stripHtml = 1 + split.token.char = 10 + split.token.if.isTrue = {$styles.content.imgtext.imageTextSplit} + split.returnKey.data = register : IMAGE_NUM_CURRENT + } + + titleText < .altText + titleText.field = titleText + + longdescURL < .altText + longdescURL.field = longdescURL + + emptyTitleHandling = {$styles.content.imgtext.emptyTitleHandling} + titleInLink = {$styles.content.imgtext.titleInLink} + titleInLinkAndImg = {$styles.content.imgtext.titleInLinkAndImg} + } + + textPos.field = imageorient + maxW = {$styles.content.imgtext.maxW} + maxW.override.data = register:maxImageWidth + maxWInText = {$styles.content.imgtext.maxWInText} + maxWInText.override.data = register:maxImageWidthInText + + equalH.field = imageheight + + image_compression.field = image_compression + image_effects.field = image_effects + + noRows.field = image_noRows + + cols.field = imagecols + border.field = imageborder + + caption { + 1 = TEXT + 1 { + field = imagecaption + required = 1 + parseFunc =< lib.parseFunc + br = 1 + split.token.char = 10 + split.token.if.isPositive = {$styles.content.imgtext.imageTextSplit} + {$styles.content.imgtext.captionSplit} + split.returnKey.data = register : IMAGE_NUM_CURRENT + } + } + # captionSplit is deprecated, use imageTextSplit instead + captionSplit = {$styles.content.imgtext.captionSplit} + captionAlign.field = imagecaption_position + # caption/alttext/title/longdescURL splitting + imageTextSplit = {$styles.content.imgtext.imageTextSplit} + + borderCol = {$styles.content.imgtext.borderColor} + borderThick = {$styles.content.imgtext.borderThick} + borderClass = {$styles.content.imgtext.borderClass} + colSpace = {$styles.content.imgtext.colSpace} + rowSpace = {$styles.content.imgtext.rowSpace} + textMargin = {$styles.content.imgtext.textMargin} + + borderSpace = {$styles.content.imgtext.borderSpace} + separateRows = {$styles.content.imgtext.separateRows} + addClasses = + addClassesImage = + addClassesImage.ifEmpty = csc-textpic-firstcol csc-textpic-lastcol + addClassesImage.override = csc-textpic-firstcol |*| |*| csc-textpic-lastcol + addClassesImage.override.if { + isGreaterThan.field = imagecols + value = 1 + } + + # + imageStdWrap.dataWrap = <div class="csc-textpic-imagewrap" style="width:{register:totalwidth}px;"> | </div> + imageStdWrapNoWidth.wrap = <div class="csc-textpic-imagewrap"> | </div> + + # if noRows is set, wrap around each column: + imageColumnStdWrap.dataWrap = <div class="csc-textpic-imagecolumn" style="width:{register:columnwidth}px;"> | </div> + + layout = CASE + layout { + key.field = imageorient + # above-center + default = TEXT + default.value = <div class="csc-textpic csc-textpic-center csc-textpic-above###CLASSES###">###IMAGES######TEXT###</div><div class="csc-textpic-clear"><!-- --></div> + # above-right + 1 = TEXT + 1.value = <div class="csc-textpic csc-textpic-right csc-textpic-above###CLASSES###">###IMAGES######TEXT###</div><div class="csc-textpic-clear"><!-- --></div> + # above-left + 2 = TEXT + 2.value = <div class="csc-textpic csc-textpic-left csc-textpic-above###CLASSES###">###IMAGES######TEXT###</div><div class="csc-textpic-clear"><!-- --></div> + # below-center + 8 = TEXT + 8.value = <div class="csc-textpic csc-textpic-center csc-textpic-below###CLASSES###">###TEXT######IMAGES###</div><div class="csc-textpic-clear"><!-- --></div> + # below-right + 9 = TEXT + 9.value = <div class="csc-textpic csc-textpic-right csc-textpic-below###CLASSES###">###TEXT######IMAGES###</div><div class="csc-textpic-clear"><!-- --></div> + # below-left + 10 = TEXT + 10.value = <div class="csc-textpic csc-textpic-left csc-textpic-below###CLASSES###">###TEXT######IMAGES###</div><div class="csc-textpic-clear"><!-- --></div> + # intext-right + 17 = TEXT + 17.value = <div class="csc-textpic csc-textpic-intext-right###CLASSES###">###IMAGES######TEXT###</div> + 17.override = <div class="csc-textpic csc-textpic-intext-right###CLASSES###">###IMAGES######TEXT###</div><div class="csc-textpic-clear"><!-- --></div> + 17.override.if.isTrue = {$styles.content.imgtext.addIntextClearer} + # intext-left + 18 = TEXT + 18.value = <div class="csc-textpic csc-textpic-intext-left###CLASSES###">###IMAGES######TEXT###</div> + 18.override = <div class="csc-textpic csc-textpic-intext-left###CLASSES###">###IMAGES######TEXT###</div><div class="csc-textpic-clear"><!-- --></div> + 18.override.if.isTrue = {$styles.content.imgtext.addIntextClearer} + # intext-right-nowrap + 25 = TEXT + 25.value = <div class="csc-textpic csc-textpic-intext-right-nowrap###CLASSES###">###IMAGES###<div style="margin-right:{register:rowWidthPlusTextMargin}px;">###TEXT###</div></div><div class="csc-textpic-clear"><!-- --></div> + 25.insertData = 1 + # intext-left-nowrap + 26 = TEXT + 26.value = <div class="csc-textpic csc-textpic-intext-left-nowrap###CLASSES###">###IMAGES###<div style="margin-left:{register:rowWidthPlusTextMargin}px;">###TEXT###</div></div><div class="csc-textpic-clear"><!-- --></div> + 26.insertData = 1 + } + + rendering { + dl { + # Choose another rendering for special edge cases + fallbackRendering = COA + fallbackRendering { + # Just one image without a caption => don't need the dl-overhead, use the "simple" rendering + 10 = TEXT + 10 { + if { + isFalse.field = imagecaption + value = 1 + equals.data = register:imageCount + } + value = simple + } + + # Multiple images and one global caption => "ul" + 20 = TEXT + 20 { + if { + value = 1 + isGreaterThan.data = register:imageCount + isTrue.if.isTrue.data = register:renderGlobalCaption + isTrue.field = imagecaption + } + value = ul + } + + # Multiple images and no caption at all => "ul" + 30 = TEXT + 30 { + if { + value = 1 + isGreaterThan.data = register:imageCount + isFalse.field = imagecaption + } + value = ul + } + } + imageRowStdWrap.dataWrap = <div class="csc-textpic-imagerow" style="width:{register:rowwidth}px;"> | </div> + imageLastRowStdWrap.dataWrap = <div class="csc-textpic-imagerow csc-textpic-imagerow-last" style="width:{register:rowwidth}px;"> | </div> + noRowsStdWrap.wrap = + oneImageStdWrap.dataWrap = <dl class="csc-textpic-image###CLASSES###" style="width:{register:imagespace}px;"> | </dl> + imgTagStdWrap.wrap = <dt> | </dt> + editIconsStdWrap.wrap = <dd> | </dd> + caption { + required = 1 + wrap = <dd class="csc-textpic-caption"> | </dd> + } + } + ul { + # Just one image without a caption => don't need the ul-overhead, use the "simple" rendering + fallbackRendering < tt_content.image.20.rendering.dl.fallbackRendering.10 + imageRowStdWrap.dataWrap = <div class="csc-textpic-imagerow" style="width:{register:rowwidth}px;"><ul> | </ul></div> + imageLastRowStdWrap.dataWrap = <div class="csc-textpic-imagerow csc-textpic-imagerow-last" style="width:{register:rowwidth}px;"><ul> | </ul></div> + noRowsStdWrap.wrap = <ul> | </ul> + oneImageStdWrap.dataWrap = <li class="csc-textpic-image###CLASSES###" style="width:{register:imagespace}px;"> | </li> + imgTagStdWrap.wrap = + editIconsStdWrap.wrap = <div> | </div> + caption.wrap = <div class="csc-textpic-caption"> | </div> + } + div { + # Just one image without a caption => don't need the div-overhead, use the "simple" rendering + fallbackRendering < tt_content.image.20.rendering.dl.fallbackRendering.10 + imageRowStdWrap.dataWrap = <div class="csc-textpic-imagerow" style="width:{register:rowwidth}px;"> | </div> + imageLastRowStdWrap.dataWrap = <div class="csc-textpic-imagerow csc-textpic-imagerow-last" style="width:{register:rowwidth}px;"> | </div> + noRowsStdWrap.wrap = + oneImageStdWrap.dataWrap = <div class="csc-textpic-image###CLASSES###" style="width:{register:imagespace}px;"> | </div> + imgTagStdWrap.wrap = <div> | </div> + editIconsStdWrap.wrap = <div> | </div> + caption.wrap = <div class="csc-textpic-caption"> | </div> + } + simple { + imageRowStdWrap.dataWrap = | + imageLastRowStdWrap.dataWrap = | + noRowsStdWrap.wrap = + oneImageStdWrap.dataWrap = | + imgTagStdWrap.wrap = | + editIconsStdWrap.wrap = | + caption.wrap = <div class="csc-textpic-caption"> | </div> + imageStdWrap.dataWrap = <div class="csc-textpic-imagewrap csc-textpic-single-image" style="width:{register:totalwidth}px;"> | </div> + imageStdWrapNoWidth.wrap = <div class="csc-textpic-imagewrap csc-textpic-single-image"> | </div> + } + } + renderMethod = dl + + editIcons = tt_content : image [imageorient|imagewidth|imageheight], [imagecols|image_noRows|imageborder],[image_link|image_zoom],[image_compression|image_effects|image_frames],imagecaption[imagecaption_position] + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.images + + caption.editIcons = tt_content : imagecaption[imagecaption_position] + caption.editIcons.beforeLastTag=1 + caption.editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.caption + + stdWrap.prefixComment = 2 | Image block: +} + +# ***************** +# CType: textpic +# ***************** +tt_content.textpic = COA +tt_content.textpic { + 10 = COA + 10.if.value = 25 + 10.if.isLessThan.field = imageorient + 10.10 = < lib.stdheader + + 20 = < tt_content.image.20 + 20 { + text.10 = COA + text.10 { + if.value = 24 + if.isGreaterThan.field = imageorient + 10 = < lib.stdheader + 10.stdWrap.dataWrap = <div class="csc-textpicHeader csc-textpicHeader-{field:imageorient}">|</div> + } + text.20 = < tt_content.text.20 + text.wrap = <div class="csc-textpic-text"> | </div> + } +} + + + +# ***************** +# CType: bullet +# ***************** +tt_content.bullets = COA +tt_content.bullets { + 10 = < lib.stdheader + + 20 = TEXT + 20 { + field = bodytext + trim = 1 + split{ + token.char = 10 + cObjNum = |*|1|| 2|*| + 1.current = 1 + 1.parseFunc =< lib.parseFunc + 1.wrap = <li class="odd">|</li> + + 2.current = 1 + 2.parseFunc =< lib.parseFunc + 2.wrap = <li class="even">|</li> + } + dataWrap = <ul class="csc-bulletlist csc-bulletlist-{field:layout}">|</ul> + editIcons = tt_content: bodytext, [layout] + editIcons.beforeLastTag = 1 + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.php:eIcon.bullets + + prefixComment = 2 | Bullet list: + } +} + + + +# ***************** +# CType: table +# ***************** +# Rendered by a PHP function specifically written to handle CE tables. See css_styled_content/pi1/class.tx_cssstyledcontent_pi1.php +tt_content.table = COA +tt_content.table { + 10 = < lib.stdheader + + 20 = USER + 20.userFunc = tx_cssstyledcontent_pi1->render_table + 20.field = bodytext + + 20.color { + default = + 1 = #EDEBF1 + 2 = #F5FFAA + } + 20.tableParams_0 { + border = + cellpadding = + cellspacing = + } + 20.tableParams_1 { + border = + cellpadding = + cellspacing = + } + 20.tableParams_2 { + border = + cellpadding = + cellspacing = + } + 20.tableParams_3 { + border = + cellpadding = + cellspacing = + } + 20.innerStdWrap.wrap = | + 20.innerStdWrap.parseFunc = < lib.parseFunc + + 20.stdWrap { + editIcons = tt_content: cols, bodytext, [layout], [table_bgColor|table_border|table_cellspacing|table_cellpadding] + editIcons.beforeLastTag = 1 + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.table + + prefixComment = 2 | Table: + } +} + + +# ***************** +# CType: uploads +# ***************** +# Rendered by a PHP function specifically written to handle CE filelists. See css_styled_content/pi1/class.tx_cssstyledcontent_pi1.php +tt_content.uploads = COA +tt_content.uploads { + 10 = < lib.stdheader + + 20 = USER + 20.userFunc = tx_cssstyledcontent_pi1->render_uploads + 20.field = media + 20.filePath.field = select_key + + 20 { + # Rendering for each file (e.g. rows of the table) as a cObject + itemRendering = COA + itemRendering { + wrap = <tr class="tr-odd tr-first">|</tr> |*| <tr class="tr-even">|</tr> || <tr class="tr-odd">|</tr> |*| + + 10 = TEXT + 10.data = register:linkedIcon + 10.wrap = <td class="csc-uploads-icon">|</td> + 10.if.isPositive.field = layout + + 20 = COA + 20.wrap = <td class="csc-uploads-fileName">|</td> + 20.1 = TEXT + 20.1 { + data = register:linkedLabel + wrap = <p>|</p> + } + 20.2 = TEXT + 20.2 { + data = register:description + wrap = <p class="csc-uploads-description">|</p> + required = 1 + htmlSpecialChars = 1 + } + + 30 = TEXT + 30.if.isTrue.field = filelink_size + 30.data = register:fileSize + 30.wrap = <td class="csc-uploads-fileSize">|</td> + 30.bytes = 1 + 30.bytes.labels = {$styles.content.uploads.filesizeBytesLabels} + } + useSpacesInLinkText = 0 + stripFileExtensionFromLinkText = 0 + } + + 20.color { + default = + 1 = #EDEBF1 + 2 = #F5FFAA + } + 20.tableParams_0 { + border = + cellpadding = + cellspacing = + } + 20.tableParams_1 { + border = + cellpadding = + cellspacing = + } + 20.tableParams_2 { + border = + cellpadding = + cellspacing = + } + 20.tableParams_3 { + border = + cellpadding = + cellspacing = + } + + 20.linkProc { + target = _blank + jumpurl = {$styles.content.uploads.jumpurl} + jumpurl.secure = {$styles.content.uploads.jumpurl_secure} + jumpurl.secure.mimeTypes = {$styles.content.uploads.jumpurl_secure_mimeTypes} + removePrependedNumbers = 1 + + iconCObject = IMAGE + iconCObject.file.import.data = register : ICON_REL_PATH + iconCObject.file.width = 150 + } + + 20.filesize { + bytes = 1 + bytes.labels = {$styles.content.uploads.filesizeBytesLabels} + } + + 20.stdWrap { + editIcons = tt_content: media, layout [table_bgColor|table_border|table_cellspacing|table_cellpadding], filelink_size, imagecaption + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.filelist + + prefixComment = 2 | File list: + } +} + + +# ****************** +# CType: multimedia +# ****************** +tt_content.multimedia = COA +tt_content.multimedia { + 10 = < lib.stdheader + + 20 = MULTIMEDIA + 20.file.field = multimedia + 20.file.wrap = uploads/media/ + 20.file.listNum = 0 + 20.params.field = bodytext + + 20.stdWrap { + editIcons = tt_content: multimedia, bodytext + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.multimedia + + prefixComment = 2 | Multimedia element: + } +} + +# ***************** +# CType: swfobject +# ***************** +tt_content.swfobject = COA +tt_content.swfobject { + 10 = < lib.stdheader + + 20 = SWFOBJECT + 20 { + file = + width = + height = + + flexParams.field = pi_flexform + + alternativeContent.field = bodytext + + layout = ###SWFOBJECT### + + video { + player = {$styles.content.media.videoPlayer} + + defaultWidth = {$styles.content.media.defaultVideoWidth} + defaultHeight = {$styles.content.media.defaultVideoHeight} + + default { + params.quality = high + params.menu = false + params.allowScriptAccess = sameDomain + params.allowFullScreen = true + } + mapping { + + } + } + + audio { + player = {$styles.content.media.audioPlayer} + + defaultWidth = {$styles.content.media.defaultAudioWidth} + defaultHeight = {$styles.content.media.defaultAudioHeight} + + default { + params.quality = high + params.allowScriptAccess = sameDomain + params.menu = false + } + mapping { + flashvars.file = soundFile + } + } + + } + 20.stdWrap { + editIcons = tt_content: multimedia, imagewidth, imageheight, pi_flexform, bodytext + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.multimedia + + prefixComment = 2 | SWFobject element: + } +} + +# ***************** +# CType: qtobject +# ***************** +tt_content.qtobject = COA +tt_content.qtobject { + 10 = < lib.stdheader + + 20 = QTOBJECT + 20 { + file = + width = + height = + + flexParams.field = pi_flexform + + alternativeContent.field = bodytext + + layout = ###QTOBJECT### + + video { + player = {$styles.content.media.videoPlayer} + + defaultWidth = {$styles.content.media.defaultVideoWidth} + defaultHeight = {$styles.content.media.defaultVideoHeight} + + default { + params.quality = high + params.menu = false + params.allowScriptAccess = sameDomain + params.allowFullScreen = true + } + mapping { + + } + } + + audio { + player = {$styles.content.media.audioPlayer} + + defaultWidth = {$styles.content.media.defaultAudioWidth} + defaultHeight = {$styles.content.media.defaultAudioHeight} + + default { + params.quality = high + params.allowScriptAccess = sameDomain + params.menu = false + } + mapping { + flashvars.file = soundFile + } + } + } + 20.stdWrap { + editIcons = tt_content: multimedia, imagewidth, imageheight, pi_flexform, bodytext + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.multimedia + + prefixComment = 2 | QTobject element: + } +} + +# ***************** +# CType: media +# ***************** +tt_content.media = COA +tt_content.media { + 10 = < lib.stdheader + + 20 = MEDIA + 20 { + + flexParams.field = pi_flexform + alternativeContent < tt_content.text.20 + alternativeContent.field = bodytext + + type = video + renderType = auto + allowEmptyUrl = 0 + forcePlayer = 1 + + fileExtHandler { + default = MEDIA + avi = MEDIA + asf = MEDIA + class = MEDIA + wmv = MEDIA + mp3 = SWF + mp4 = SWF + m4v = SWF + swa = SWF + flv = SWF + swf = SWF + mov = QT + m4v = QT + m4a = QT + } + + mimeConf.swfobject < tt_content.swfobject.20 + mimeConf.qtobject < tt_content.qtobject.20 + + } + 20.stdWrap { + editIcons = tt_content: pi_flexform, bodytext + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.multimedia + + prefixComment = 2 | Media element: + } +} + +# ****************** +# CType: mailform +# ****************** +tt_content.mailform = COA +tt_content.mailform.10 = < lib.stdheader +tt_content.mailform.20 = FORM +tt_content.mailform.20 { + accessibility = 1 + noWrapAttr=1 + formName = mailform + dontMd5FieldNames = 1 + layout = <div class="csc-mailform-field">###LABEL### ###FIELD###</div> + labelWrap.wrap = | + commentWrap.wrap = | + radioWrap.wrap = |<br /> + radioWrap.accessibilityWrap = <fieldset###RADIO_FIELD_ID###><legend>###RADIO_GROUP_LABEL###</legend>|</fieldset> + REQ = 1 + REQ.labelWrap.wrap = | + COMMENT.layout = <div class="csc-mailform-label">###LABEL###</div> + RADIO.layout = <div class="csc-mailform-field">###LABEL### <span class="csc-mailform-radio">###FIELD###</span></div> + LABEL.layout = <div class="csc-mailform-field">###LABEL### <span class="csc-mailform-label">###FIELD###</span></div> + target = {$styles.content.mailform.target} + goodMess = {$styles.content.mailform.goodMess} + badMess = {$styles.content.mailform.badMess} + redirect.field = pages + redirect.listNum = 0 + recipient.field = subheader + data.field = bodytext + locationData = 1 + hiddenFields.stdWrap.wrap = <div style="display:none;">|</div> + + params.radio = class="csc-mailform-radio" + params.check = class="csc-mailform-check" + params.submit = class="csc-mailform-submit" + + stdWrap.wrap = <fieldset class="csc-mailform"> | </fieldset> + stdWrap { + editIcons = tt_content: bodytext, pages, subheader + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.form + + prefixComment = 2 | Mail form inserted: + } +} + + +# ****************** +# CType: search +# ****************** +tt_content.search = COA +tt_content.search.10 = < lib.stdheader +# Result: +tt_content.search.20 = SEARCHRESULT +tt_content.search.20 { + allowedCols = pages.title-subtitle-keywords-description : tt_content.header-bodytext-imagecaption : tt_address.name-title-address-email-company-city-country : tt_links.title-note-note2-url : tt_board.subject-message-author-email : tt_calender.title-note : tt_products.title-note-itemnumber + languageField.tt_content = sys_language_uid + renderObj = COA + renderObj { + + 10 = TEXT + 10.field = pages_title + 10.htmlSpecialChars = 1 + 10.typolink { + parameter.field = uid + target = {$styles.content.searchresult.resultTarget} + additionalParams.data = register:SWORD_PARAMS + additionalParams.required = 1 + additionalParams.wrap = &no_cache=1 + } + 10.htmlSpecialChars = 1 + 10.wrap = <h3 class="csc-searchResultHeader">|</h3> + + 20 = COA + 20 { + 10 = TEXT + 10.field = tt_content_bodytext + 10.stripHtml = 1 + 10.htmlSpecialChars = 1 + } + 20.stdWrap.crop = 200 | ... + 20.stdWrap.wrap = <p class="csc-searchResult">|</p> + } + + layout = COA + layout { + wrap = <table border="0" cellspacing="0" cellpadding="2" class="csc-searchResultInfo"><tr> | </tr></table> ###RESULT### + + 10 = TEXT + 10.data = LLL:EXT:css_styled_content/pi1/locallang.xml:search.resultRange + 10.wrap = <td class="csc-searchResultRange"><p>|</p></td> + + 20 = TEXT + 20.value = ###PREV### ###NEXT### + 20.wrap = <td class="csc-searchResultPrevNext"><p>|</p></td> + } + + noResultObj = COA + noResultObj { + 10 = TEXT + 10.data = LLL:EXT:css_styled_content/pi1/locallang.xml:search.emptySearch + 10.wrap = <h3 class="csc-noSearchResultMsg">|</h3> + } + + next = TEXT + next.data = LLL:EXT:css_styled_content/pi1/locallang.xml:search.searchResultNext + + prev = TEXT + prev.data = LLL:EXT:css_styled_content/pi1/locallang.xml:search.searchResultPrev + + target = {$styles.content.searchresult.target} + range = 20 + + stdWrap.prefixComment = 2 | Search result: +} + +# Form: +tt_content.search.30 < tt_content.mailform.20 +tt_content.search.30 { + goodMess = {$styles.content.searchform.goodMess} + redirect > + recipient > + data > + dataArray { + 10.label.data = LLL:EXT:css_styled_content/pi1/locallang.xml:search.searchWord + 10.type = sword=input + 20.label.data = LLL:EXT:css_styled_content/pi1/locallang.xml:search.searchIn + 20.type = scols=select + 20.valueArray { + 10.label.data = LLL:EXT:css_styled_content/pi1/locallang.xml:search.headersKeywords + 10.value = pages.title-subtitle-keywords-description:tt_content.header + 20.label.data = LLL:EXT:css_styled_content/pi1/locallang.xml:search.pageContent + 20.value = tt_content.header-bodytext-imagecaption + } + 30.type = stype=hidden + 30.value = L0 + 40.type = submit=submit + 40.value.data = LLL:EXT:css_styled_content/pi1/locallang.xml:search.searchButton + } + type.field = pages + type.listNum = 0 + locationData = HTTP_POST_VARS + no_cache = 1 + + stdWrap.wrap = <table border="0" cellspacing="1" cellpadding="1" class="csc-searchform"> | </table> + stdWrap { + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.search + + prefixComment = 2 | Search form inserted: + } +} + + +# ****************** +# CType: login +# ****************** +tt_content.login < tt_content.mailform +tt_content.login.10 = < lib.stdheader +tt_content.login.20 { + goodMess = {$styles.content.loginform.goodMess} + redirect > + recipient > + data > + dataArray { + 10.label.data = LLL:EXT:css_styled_content/pi1/locallang.xml:login.username + 10.type = *user=input + 20.label.data = LLL:EXT:css_styled_content/pi1/locallang.xml:login.password + 20.type = *pass=password + 30.type = logintype=hidden + 30.value = login + 40.type = submit=submit + 40.value.data = LLL:EXT:css_styled_content/pi1/locallang.xml:login.login + } + type.field = pages + type.listNum = 0 + target = {$styles.content.loginform.target} + locationData = 0 + hiddenFields.pid = TEXT + hiddenFields.pid { + value = {$styles.content.loginform.pid} + override.field = pages + override.listNum = 1 + } + + stdWrap.wrap = <div class="csc-loginform"> | </div> + stdWrap { + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.login + + prefixComment = 2 | Login/Logout form: + } +} +[loginUser = *] +tt_content.login.20 { + dataArray > + dataArray { + 10.label.data = LLL:EXT:css_styled_content/pi1/locallang.xml:login.username + 10.label.wrap = | <!--###USERNAME###--> + 30.type = logintype=hidden + 30.value = logout + 40.type = submit=submit + 40.value.data = LLL:EXT:css_styled_content/pi1/locallang.xml:login.logout + } +} +[global] + + +# ****************** +# CType: splash +# ****************** +# Deprecated element. +# Still here for backwards compliance with plugins using the "text box" type. +tt_content.splash = CASE +tt_content.splash.key.field = splash_layout +tt_content.splash.stdWrap { + prefixComment = 2 | Textbox inserted (Deprecated) +} +tt_content.splash.default = COA +tt_content.splash.default { + 20 = CTABLE + 20 { + c.1 = < tt_content.text + lm.1 = IMAGE + lm.1.file { + import = uploads/pics/ + import.field = image + import.listNum = 0 + maxW.field = imagewidth + maxW.ifEmpty = 200 + } + cMargins = 30,0,0,0 + } +} +tt_content.splash.1 < tt_content.splash.default +tt_content.splash.1.20.lm.1.file > +tt_content.splash.1.20.lm.1.file = GIFBUILDER +tt_content.splash.1.20.lm.1.file { + XY = [10.w]+10,[10.h]+10 + backColor = {$content.splash.bgCol} + backColor.override.data = register:pageColor + format = jpg + 5 = BOX + 5.dimensions = 3,3,[10.w],[10.h] + 5.color = #333333 + 7 = EFFECT + 7.value = blur=99|blur=99|blur=99|blur=99|blur=99|blur=99|blur=99 + 10 = IMAGE + 10.file { + import = uploads/pics/ + import.field = image + import.listNum = 0 + maxW.field = imagewidth + maxW.ifEmpty = 200 + } +} +// The image frames are not available unless TypoScript code from styles.content.imgFrames.x is provided manually: +tt_content.splash.2 < tt_content.splash.default +#tt_content.splash.2.20.lm.1.file.m < styles.content.imgFrames.1 +tt_content.splash.3 < tt_content.splash.default +#tt_content.splash.3.20.lm.1.file.m < styles.content.imgFrames.2 + +// From plugin.postit1, if included: +tt_content.splash.20 = < plugin.postit1 + + + +# **************** +# CType: menu +# **************** +tt_content.menu = COA +tt_content.menu { + 10 = < lib.stdheader + + 20 = CASE + 20 { + key.field = menu_type + + # "Menu of these pages" + default = HMENU + default { + special = list + special.value.field = pages + wrap = <ul class="csc-menu csc-menu-def">|</ul> + 1 = TMENU + 1 { + target = {$PAGE_TARGET} + NO { + stdWrap.htmlSpecialChars = 1 + wrapItemAndSub = <li>|</li> + ATagTitle.field = description // title + } + noBlur = 1 + } + } + + # "Menu of subpages to these pages" + 1 < .default + 1 { + special = directory + wrap = <ul class="csc-menu csc-menu-1">|</ul> + } + + # "Sitemap - liststyle" + 2 = HMENU + 2 { + wrap = <div class="csc-sitemap">|</div> + 1 = TMENU + 1 { + target = {$PAGE_TARGET} + noBlur = 1 + expAll = 1 + wrap = <ul>|</ul> + NO { + stdWrap.htmlSpecialChars = 1 + wrapItemAndSub = <li>|</li> + ATagTitle.field = description // title + } + } + 2 < .1 + 3 < .1 + 4 < .1 + 5 < .1 + 6 < .1 + 7 < .1 + } + + # "Section index (pagecontent w/Index checked - liststyle)" + 3 < styles.content.get + 3 { + wrap = <ul class="csc-menu csc-menu-3">|</ul> + select.andWhere = sectionIndex!=0 + select.pidInList.override.field = pages + renderObj = TEXT + renderObj { + fieldRequired = header + trim = 1 + field = header + htmlSpecialChars = 1 + noBlur = 1 + wrap = <li class="csc-section">|</li> + typolink.parameter.field = pid + typolink.section.field = uid + } + } + + # "Menu of subpages to these pages (with abstract)" + 4 < .1 + 4 { + wrap = <dl class="csc-menu csc-menu-4">|</dl> + 1.NO { + wrapItemAndSub > + linkWrap = <dt>|</dt> + after { + data = field : abstract // field : description // field : subtitle + required = 1 + htmlSpecialChars = 1 + wrap = <dd>|</dd> + } + ATagTitle.field = description // title + } + } + + # "Recently updated pages" + 5 < .default + 5 { + wrap = <ul class="csc-menu csc-menu-5">|</ul> + special = updated + special { + maxAge = 3600*24*7 + excludeNoSearchPages = 1 + } + } + + # "Related pages (based on keywords)" + 6 < .default + 6 { + wrap = <ul class="csc-menu csc-menu-6">|</ul> + special = keywords + special { + excludeNoSearchPages = 1 + } + } + + # "Menu of subpages to these pages + sections - liststyle" + 7 < .1 + 7 { + wrap = <ul class="csc-menu csc-menu-7">|</ul> + 1.expAll = 1 + 2 < .1 + 2 { + sectionIndex = 1 + sectionIndex.type = header + wrap = <ul>|</ul> + NO.wrapItemAndSub = <li class="csc-section">|</li> + } + } + } + + 20.stdWrap { + editIcons = tt_content: menu_type, pages + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.menuSitemap + + prefixComment = 2 | Menu/Sitemap element: + } +} + + + +# **************** +# CType: shortcut +# **************** +# Should be a complete copy from the old static template "content (default)" +tt_content.shortcut = COA +tt_content.shortcut { + 20 = CASE + 20.key.field = layout + 20.0= RECORDS + 20.0 { + source.field = records + tables = {$content.shortcut.tables} + # THESE are OLD plugins. Modern plugins registers themselves automatically! + conf.tt_content = < tt_content + conf.tt_address = < tt_address + conf.tt_links = < tt_links + conf.tt_guest = < tt_guest + conf.tt_board = < tt_board + conf.tt_calender = < tt_calender + conf.tt_rating < tt_rating + conf.tt_products = < tt_products + conf.tt_news = < tt_news + conf.tt_poll = < plugin.tt_poll + } + 20.1= RECORDS + 20.1 { + source.field = records + tables = {$content.shortcut.tables} + conf.tt_poll = < plugin.tt_poll + conf.tt_poll.code = RESULT,SUBMITTEDVOTE + } + + 20.stdWrap { + editIcons = tt_content: records + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.recordList + + prefixComment = 2 | Inclusion of other records (by reference): + } +} + + +# **************** +# CType: list +# **************** +# Should be a complete copy from the old static template "content (default)" (except "lib.stdheader") +tt_content.list = COA +tt_content.list { + 10 = < lib.stdheader + + 20 = CASE + 20.key.field = list_type + 20 { + # LIST element references (NOT copy of objects!) + # THESE are OLD plugins. Modern plugins registers themselves automatically! + 3 = CASE + 3.key.field = layout + 3.0 = < plugin.tt_guest + + 4 = CASE + 4.key.field = layout + 4.0 = < plugin.tt_board_list + 4.1 = < plugin.tt_board_tree + + 2 = CASE + 2.key.field = layout + 2.0 = < plugin.tt_board_tree + + 5 = CASE + 5.key.field = layout + 5.0 = < plugin.tt_products + + 7 = CASE + 7.key.field = layout + 7.0 = < plugin.tt_calender + + 8 = CASE + 8.key.field = layout + 8.0 = < plugin.tt_rating + + 9 = CASE + 9.key.field = layout + 9.0 = < plugin.tt_news + + 11 = CASE + 11.key.field = layout + 11.0 = < plugin.tipafriend + + 20 = CASE + 20.key.field = layout + 20.0 = < plugin.feadmin.fe_users + + 21 = CASE + 21.key.field = layout + 21.0 = < plugin.feadmin.dmailsubscription + } + + 20.stdWrap { + editIcons = tt_content: list_type, layout, select_key, pages [recursive] + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.plugin + + prefixComment = 2 | Plugin inserted: + } +} + + +# **************** +# CType: script +# **************** +# OBSOLETE! Please make extensions instead. The "script" content element was meant for these custom purposes in the past. Today extensions will do the job better. +tt_content.script = TEXT +tt_content.script { + value = + + prefixComment = 2 | Script element (Deprecated) +} + + +# **************** +# CType: div +# **************** +tt_content.div = TEXT +tt_content.div { + value = <hr /> + wrap = <div class="divider">|</div> + prefixComment = 2 | Div element +} + + +# **************** +# CType: html +# **************** +# This truely IS a content object, launched from inside the PHP class of course. +# Should be a complete copy from the old static template "content (default)" +tt_content.html = TEXT +tt_content.html { + field = bodytext + + editIcons = tt_content: pages + editIcons.iconTitle.data = LLL:EXT:css_styled_content/pi1/locallang.xml:eIcon.html + + prefixComment = 2 | Raw HTML content: +} + + +# **************** +# Default error msg: +# **************** +tt_content.default = TEXT +tt_content.default { + field = CType + wrap = <p style="background-color: yellow;"><b>ERROR:</b> Content Element type "|" has no rendering definition!</p> + + prefixComment = 2 | Unknown element message: +} + +# ********************************************************************* +# ACCESSIBILTY MODE +# ********************************************************************* + + + + + + + +plugin.tx_cssstyledcontent._CSS_DEFAULT_STYLE ( + /* Captions */ + DIV.csc-textpic-caption-c .csc-textpic-caption { text-align: center; } + DIV.csc-textpic-caption-r .csc-textpic-caption { text-align: right; } + DIV.csc-textpic-caption-l .csc-textpic-caption { text-align: left; } + + /* Needed for noRows setting */ + DIV.csc-textpic DIV.csc-textpic-imagecolumn { float: left; display: inline; } + + /* Border just around the image */ + {$styles.content.imgtext.borderSelector} { + border: {$styles.content.imgtext.borderThick}px solid {$styles.content.imgtext.borderColor}; + padding: {$styles.content.imgtext.borderSpace}px {$styles.content.imgtext.borderSpace}px; + } + + DIV.csc-textpic-imagewrap { padding: 0; } + + DIV.csc-textpic IMG { border: none; } + + /* DIV: This will place the images side by side */ + DIV.csc-textpic DIV.csc-textpic-imagewrap DIV.csc-textpic-image { float: left; } + + /* UL: This will place the images side by side */ + DIV.csc-textpic DIV.csc-textpic-imagewrap UL { list-style: none; margin: 0; padding: 0; } + DIV.csc-textpic DIV.csc-textpic-imagewrap UL LI { float: left; margin: 0; padding: 0; } + + /* DL: This will place the images side by side */ + DIV.csc-textpic DIV.csc-textpic-imagewrap DL.csc-textpic-image { float: left; } + DIV.csc-textpic DIV.csc-textpic-imagewrap DL.csc-textpic-image DT { float: none; } + DIV.csc-textpic DIV.csc-textpic-imagewrap DL.csc-textpic-image DD { float: none; } + DIV.csc-textpic DIV.csc-textpic-imagewrap DL.csc-textpic-image DD IMG { border: none; } /* FE-Editing Icons */ + DL.csc-textpic-image { margin: 0; } + DL.csc-textpic-image DT { margin: 0; display: inline; } + DL.csc-textpic-image DD { margin: 0; } + + /* Clearer */ + DIV.csc-textpic-clear { clear: both; } + + /* Margins around images: */ + + /* Pictures on left, add margin on right */ + DIV.csc-textpic-left DIV.csc-textpic-imagewrap .csc-textpic-image, + DIV.csc-textpic-intext-left-nowrap DIV.csc-textpic-imagewrap .csc-textpic-image, + DIV.csc-textpic-intext-left DIV.csc-textpic-imagewrap .csc-textpic-image { + display: inline; /* IE fix for double-margin bug */ + margin-right: {$styles.content.imgtext.colSpace}px; + } + + /* Pictures on right, add margin on left */ + DIV.csc-textpic-right DIV.csc-textpic-imagewrap .csc-textpic-image, + DIV.csc-textpic-intext-right-nowrap DIV.csc-textpic-imagewrap .csc-textpic-image, + DIV.csc-textpic-intext-right DIV.csc-textpic-imagewrap .csc-textpic-image { + display: inline; /* IE fix for double-margin bug */ + margin-left: {$styles.content.imgtext.colSpace}px; + } + + /* Pictures centered, add margin on left */ + DIV.csc-textpic-center DIV.csc-textpic-imagewrap .csc-textpic-image { + display: inline; /* IE fix for double-margin bug */ + margin-left: {$styles.content.imgtext.colSpace}px; + } + DIV.csc-textpic DIV.csc-textpic-imagewrap .csc-textpic-image .csc-textpic-caption { margin: 0; } + DIV.csc-textpic DIV.csc-textpic-imagewrap .csc-textpic-image IMG { margin: 0; vertical-align:bottom; } + + /* Space below each image (also in-between rows) */ + DIV.csc-textpic DIV.csc-textpic-imagewrap .csc-textpic-image { margin-bottom: {$styles.content.imgtext.rowSpace}px; } + DIV.csc-textpic-equalheight DIV.csc-textpic-imagerow { margin-bottom: {$styles.content.imgtext.rowSpace}px; display: block; } + DIV.csc-textpic DIV.csc-textpic-imagerow { clear: both; } + DIV.csc-textpic DIV.csc-textpic-single-image IMG { margin-bottom: {$styles.content.imgtext.rowSpace}px; } + + /* IE7 hack for margin between image rows */ + *+html DIV.csc-textpic DIV.csc-textpic-imagerow .csc-textpic-image { margin-bottom: 0; } + *+html DIV.csc-textpic DIV.csc-textpic-imagerow { margin-bottom: {$styles.content.imgtext.rowSpace}px; } + + /* No margins around the whole image-block */ + DIV.csc-textpic DIV.csc-textpic-imagewrap .csc-textpic-firstcol { margin-left: 0px !important; } + DIV.csc-textpic DIV.csc-textpic-imagewrap .csc-textpic-lastcol { margin-right: 0px !important; } + + /* Add margin from image-block to text (in case of "Text w/ images") */ + DIV.csc-textpic-intext-left DIV.csc-textpic-imagewrap, + DIV.csc-textpic-intext-left-nowrap DIV.csc-textpic-imagewrap { + margin-right: {$styles.content.imgtext.textMargin}px !important; + } + DIV.csc-textpic-intext-right DIV.csc-textpic-imagewrap, + DIV.csc-textpic-intext-right-nowrap DIV.csc-textpic-imagewrap { + margin-left: {$styles.content.imgtext.textMargin}px !important; + } + + /* Positioning of images: */ + + /* Above */ + DIV.csc-textpic-above DIV.csc-textpic-text { clear: both; } + + /* Center (above or below) */ + DIV.csc-textpic-center { text-align: center; /* IE-hack */ } + DIV.csc-textpic-center DIV.csc-textpic-imagewrap { margin: 0 auto; } + DIV.csc-textpic-center DIV.csc-textpic-imagewrap .csc-textpic-image { text-align: left; /* Remove IE-hack */ } + DIV.csc-textpic-center DIV.csc-textpic-text { text-align: left; /* Remove IE-hack */ } + + /* Right (above or below) */ + DIV.csc-textpic-right DIV.csc-textpic-imagewrap { float: right; } + DIV.csc-textpic-right DIV.csc-textpic-text { clear: right; } + + /* Left (above or below) */ + DIV.csc-textpic-left DIV.csc-textpic-imagewrap { float: left; } + DIV.csc-textpic-left DIV.csc-textpic-text { clear: left; } + + /* Left (in text) */ + DIV.csc-textpic-intext-left DIV.csc-textpic-imagewrap { float: left; } + + /* Right (in text) */ + DIV.csc-textpic-intext-right DIV.csc-textpic-imagewrap { float: right; } + + /* Right (in text, no wrap around) */ + DIV.csc-textpic-intext-right-nowrap DIV.csc-textpic-imagewrap { float: right; clear: both; } + /* Hide from IE5-mac. Only IE-win sees this. \*/ + * html DIV.csc-textpic-intext-right-nowrap .csc-textpic-text { height: 1%; } + /* End hide from IE5/mac */ + + /* Left (in text, no wrap around) */ + DIV.csc-textpic-intext-left-nowrap DIV.csc-textpic-imagewrap { float: left; clear: both; } + /* Hide from IE5-mac. Only IE-win sees this. \*/ + * html DIV.csc-textpic-intext-left-nowrap .csc-textpic-text, + * html .csc-textpic-intext-left ol, + * html .csc-textpic-intext-left ul { height: 1%; } + /* End hide from IE5/mac */ + + DIV.csc-textpic DIV.csc-textpic-imagerow-last { margin-bottom: 0; } + + /* Browser fixes: */ + + /* Fix for unordered and ordered list with image "In text, left" */ + .csc-textpic-intext-left ol, .csc-textpic-intext-left ul {padding-left: 40px; overflow: auto; } +) + +# TYPO3 SVN ID: $Id$ + diff --git a/tests/examplefiles/varnish.vcl b/tests/examplefiles/varnish.vcl new file mode 100644 index 00000000..6258c313 --- /dev/null +++ b/tests/examplefiles/varnish.vcl @@ -0,0 +1,187 @@ +# This is the VCL configuration Varnish will automatically append to your VCL +# file during compilation/loading. See the vcl(7) man page for details on syntax +# and semantics. +# New users is recommended to use the example.vcl file as a starting point. + +vcl 4.0; + +backend foo { .host = "192.168.1.1"; } + +probe blatti { .url = "foo"; } +probe fooy { + .url = "beh"; + +} + +acl foo { + "192.168.1.1"; + "192.168.0.0"/24; + ! "192.168.0.1"; +} + +include "foo.vcl"; + +import std; + +sub vcl_init { + new b = director.foo(); +} + +sub vcl_recv { + ban(req.url ~ "foo"); + rollback(); +} +sub vcl_recv { + if (req.method == "PRI") { + /* We do not support SPDY or HTTP/2.0 */ + return (synth(405)); + } + if (req.method != "GET" && + req.method != "HEAD" && + req.method != "PUT" && + req.method != "POST" && + req.method != "TRACE" && + req.method != "OPTIONS" && + req.method != "DELETE") { + /* Non-RFC2616 or CONNECT which is weird. */ + return (pipe); + } + + if (req.method != "GET" && req.method != "HEAD") { + /* We only deal with GET and HEAD by default */ + return (pass); + } + if (req.http.Authorization || req.http.Cookie) { + /* Not cacheable by default */ + return (pass); + } + return (hash); +} + +sub vcl_pipe { + # By default Connection: close is set on all piped requests, to stop + # connection reuse from sending future requests directly to the + # (potentially) wrong backend. If you do want this to happen, you can undo + # it here. + # unset bereq.http.connection; + return (pipe); +} + +sub vcl_pass { + return (fetch); +} + +sub vcl_hash { + hash_data(req.url); + if (req.http.host) { + hash_data(req.http.host); + } else { + hash_data(server.ip); + } + return (lookup); +} + +sub vcl_purge { + return (synth(200, "Purged")); +} + +sub vcl_hit { + if (obj.ttl >= 0s) { + // A pure unadultered hit, deliver it + return (deliver); + } + if (obj.ttl + obj.grace > 0s) { + // Object is in grace, deliver it + // Automatically triggers a background fetch + return (deliver); + } + // fetch & deliver once we get the result + return (miss); +} + +sub vcl_miss { + return (fetch); +} + +sub vcl_deliver { + set resp.http.x-storage = storage.s0.free; + return (deliver); +} + +/* + * We can come here "invisibly" with the following errors: 413, 417 & 503 + */ +sub vcl_synth { + set resp.http.Content-Type = "text/html; charset=utf-8"; + set resp.http.Retry-After = "5"; + synthetic( {"<!DOCTYPE html> +<html> + <head> + <title>"} + resp.status + " " + resp.reason + {"</title> + </head> + <body> + <h1>Error "} + resp.status + " " + resp.reason + {"</h1> + <p>"} + resp.reason + {"</p> + <h3>Guru Meditation:</h3> + <p>XID: "} + req.xid + {"</p> + <hr> + <p>Varnish cache server</p> + </body> +</html> +"} ); + return (deliver); +} + +####################################################################### +# Backend Fetch + +sub vcl_backend_fetch { + return (fetch); +} + +sub vcl_backend_response { + if (beresp.ttl <= 0s || + beresp.http.Set-Cookie || + beresp.http.Surrogate-control ~ "no-store" || + (!beresp.http.Surrogate-Control && + beresp.http.Cache-Control ~ "no-cache|no-store|private") || + beresp.http.Vary == "*") { + /* + * Mark as "Hit-For-Pass" for the next 2 minutes + */ + set beresp.ttl = 120s; + set beresp.uncacheable = true; + } + return (deliver); +} + +sub vcl_backend_error { + set beresp.http.Content-Type = "text/html; charset=utf-8"; + set beresp.http.Retry-After = "5"; + synthetic( {"<!DOCTYPE html> +<html> + <head> + <title>"} + beresp.status + " " + beresp.reason + {"</title> + </head> + <body> + <h1>Error "} + beresp.status + " " + beresp.reason + {"</h1> + <p>"} + beresp.reason + {"</p> + <h3>Guru Meditation:</h3> + <p>XID: "} + bereq.xid + {"</p> + <hr> + <p>Varnish cache server</p> + </body> +</html> +"} ); + return (deliver); +} + +####################################################################### +# Housekeeping + +sub vcl_init { +} + +sub vcl_fini { + return (ok); +} diff --git a/tests/examplefiles/wdiff_example1.wdiff b/tests/examplefiles/wdiff_example1.wdiff new file mode 100644 index 00000000..ca760812 --- /dev/null +++ b/tests/examplefiles/wdiff_example1.wdiff @@ -0,0 +1,731 @@ +.. -*- mode: rst -*- + +{+.. highlight:: python+} + +==================== +Write your own lexer +==================== + +If a lexer for your favorite language is missing in the Pygments package, you +can easily write your own and extend Pygments. + +All you need can be found inside the :mod:`pygments.lexer` module. As you can +read in the :doc:`API documentation <api>`, a lexer is a class that is +initialized with some keyword arguments (the lexer options) and that provides a +:meth:`.get_tokens_unprocessed()` method which is given a string or unicode +object with the data to [-parse.-] {+lex.+} + +The :meth:`.get_tokens_unprocessed()` method must return an iterator or iterable +containing tuples in the form ``(index, token, value)``. Normally you don't +need to do this since there are [-numerous-] base lexers {+that do most of the work and that+} +you can subclass. + + +RegexLexer +========== + +[-A very powerful (but quite easy to use)-] + +{+The+} lexer {+base class used by almost all of Pygments' lexers+} is the +:class:`RegexLexer`. This +[-lexer base-] class allows you to define lexing rules in terms of +*regular expressions* for different *states*. + +States are groups of regular expressions that are matched against the input +string at the *current position*. If one of these expressions matches, a +corresponding action is performed [-(normally-] {+(such as+} yielding a token with a specific +[-type),-] +{+type, or changing state),+} the current position is set to where the last match +ended and the matching process continues with the first regex of the current +state. + +Lexer states are kept [-in-] {+on+} a [-state-] stack: each time a new state is entered, the new +state is pushed onto the stack. The most basic lexers (like the `DiffLexer`) +just need one state. + +Each state is defined as a list of tuples in the form (`regex`, `action`, +`new_state`) where the last item is optional. In the most basic form, `action` +is a token type (like `Name.Builtin`). That means: When `regex` matches, emit a +token with the match text and type `tokentype` and push `new_state` on the state +stack. If the new state is ``'#pop'``, the topmost state is popped from the +stack instead. [-(To-] {+To+} pop more than one state, use ``'#pop:2'`` and so [-on.)-] {+on.+} +``'#push'`` is a synonym for pushing the current state on the stack. + +The following example shows the `DiffLexer` from the builtin lexers. Note that +it contains some additional attributes `name`, `aliases` and `filenames` which +aren't required for a lexer. They are used by the builtin lexer lookup +functions. + +[-.. sourcecode:: python-] {+::+} + + from pygments.lexer import RegexLexer + from pygments.token import * + + class DiffLexer(RegexLexer): + name = 'Diff' + aliases = ['diff'] + filenames = ['*.diff'] + + tokens = { + 'root': [ + (r' .*\n', Text), + (r'\+.*\n', Generic.Inserted), + (r'-.*\n', Generic.Deleted), + (r'@.*\n', Generic.Subheading), + (r'Index.*\n', Generic.Heading), + (r'=.*\n', Generic.Heading), + (r'.*\n', Text), + ] + } + +As you can see this lexer only uses one state. When the lexer starts scanning +the text, it first checks if the current character is a space. If this is true +it scans everything until newline and returns the [-parsed-] data as {+a+} `Text` [-token.-] {+token (which +is the "no special highlighting" token).+} + +If this rule doesn't match, it checks if the current char is a plus sign. And +so on. + +If no rule matches at the current position, the current char is emitted as an +`Error` token that indicates a [-parsing-] {+lexing+} error, and the position is increased by +[-1.-] +{+one.+} + + +Adding and testing a new lexer +============================== + +To make [-pygments-] {+Pygments+} aware of your new lexer, you have to perform the following +steps: + +First, change to the current directory containing the [-pygments-] {+Pygments+} source code: + +.. [-sourcecode::-] {+code-block::+} console + + $ cd .../pygments-main + +{+Select a matching module under ``pygments/lexers``, or create a new module for +your lexer class.+} + +Next, make sure the lexer is known from outside of the module. All modules in +the ``pygments.lexers`` specify ``__all__``. For example, [-``other.py`` sets: + +.. sourcecode:: python-] {+``esoteric.py`` sets::+} + + __all__ = ['BrainfuckLexer', 'BefungeLexer', ...] + +Simply add the name of your lexer class to this list. + +Finally the lexer can be made [-publically-] {+publicly+} known by rebuilding the lexer mapping: + +.. [-sourcecode::-] {+code-block::+} console + + $ make mapfiles + +To test the new lexer, store an example file with the proper extension in +``tests/examplefiles``. For example, to test your ``DiffLexer``, add a +``tests/examplefiles/example.diff`` containing a sample diff output. + +Now you can use pygmentize to render your example to HTML: + +.. [-sourcecode::-] {+code-block::+} console + + $ ./pygmentize -O full -f html -o /tmp/example.html tests/examplefiles/example.diff + +Note that this [-explicitely-] {+explicitly+} calls the ``pygmentize`` in the current directory +by preceding it with ``./``. This ensures your modifications are used. +Otherwise a possibly already installed, unmodified version without your new +lexer would have been called from the system search path (``$PATH``). + +To view the result, open ``/tmp/example.html`` in your browser. + +Once the example renders as expected, you should run the complete test suite: + +.. [-sourcecode::-] {+code-block::+} console + + $ make test + +{+It also tests that your lexer fulfills the lexer API and certain invariants, +such as that the concatenation of all token text is the same as the input text.+} + + +Regex Flags +=========== + +You can either define regex flags {+locally+} in the regex (``r'(?x)foo bar'``) or +{+globally+} by adding a `flags` attribute to your lexer class. If no attribute is +defined, it defaults to `re.MULTILINE`. For more [-informations-] {+information+} about regular +expression flags see the {+page about+} `regular expressions`_ [-help page-] in the [-python-] {+Python+} +documentation. + +.. _regular expressions: [-http://docs.python.org/lib/re-syntax.html-] {+http://docs.python.org/library/re.html#regular-expression-syntax+} + + +Scanning multiple tokens at once +================================ + +{+So far, the `action` element in the rule tuple of regex, action and state has +been a single token type. Now we look at the first of several other possible +values.+} + +Here is a more complex lexer that highlights INI files. INI files consist of +sections, comments and [-key-] {+``key+} = [-value pairs: + +.. sourcecode:: python-] {+value`` pairs::+} + + from pygments.lexer import RegexLexer, bygroups + from pygments.token import * + + class IniLexer(RegexLexer): + name = 'INI' + aliases = ['ini', 'cfg'] + filenames = ['*.ini', '*.cfg'] + + tokens = { + 'root': [ + (r'\s+', Text), + (r';.*?$', Comment), + (r'\[.*?\]$', Keyword), + (r'(.*?)(\s*)(=)(\s*)(.*?)$', + bygroups(Name.Attribute, Text, Operator, Text, String)) + ] + } + +The lexer first looks for whitespace, comments and section names. [-And later-] {+Later+} it +looks for a line that looks like a key, value pair, separated by an ``'='`` +sign, and optional whitespace. + +The `bygroups` helper [-makes sure that-] {+yields+} each {+capturing+} group [-is yielded-] {+in the regex+} with a different +token type. First the `Name.Attribute` token, then a `Text` token for the +optional whitespace, after that a `Operator` token for the equals sign. Then a +`Text` token for the whitespace again. The rest of the line is returned as +`String`. + +Note that for this to work, every part of the match must be inside a capturing +group (a ``(...)``), and there must not be any nested capturing groups. If you +nevertheless need a group, use a non-capturing group defined using this syntax: +[-``r'(?:some|words|here)'``-] +{+``(?:some|words|here)``+} (note the ``?:`` after the beginning parenthesis). + +If you find yourself needing a capturing group inside the regex which shouldn't +be part of the output but is used in the regular expressions for backreferencing +(eg: ``r'(<(foo|bar)>)(.*?)(</\2>)'``), you can pass `None` to the bygroups +function and [-it will skip-] that group will be skipped in the output. + + +Changing states +=============== + +Many lexers need multiple states to work as expected. For example, some +languages allow multiline comments to be nested. Since this is a recursive +pattern it's impossible to lex just using regular expressions. + +Here is [-the solution: + +.. sourcecode:: python-] {+a lexer that recognizes C++ style comments (multi-line with ``/* */`` +and single-line with ``//`` until end of line)::+} + + from pygments.lexer import RegexLexer + from pygments.token import * + + class [-ExampleLexer(RegexLexer):-] {+CppCommentLexer(RegexLexer):+} + name = 'Example Lexer with states' + + tokens = { + 'root': [ + (r'[^/]+', Text), + (r'/\*', Comment.Multiline, 'comment'), + (r'//.*?$', Comment.Singleline), + (r'/', Text) + ], + 'comment': [ + (r'[^*/]', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline) + ] + } + +This lexer starts lexing in the ``'root'`` state. It tries to match as much as +possible until it finds a slash (``'/'``). If the next character after the slash +is [-a star-] {+an asterisk+} (``'*'``) the `RegexLexer` sends those two characters to the +output stream marked as `Comment.Multiline` and continues [-parsing-] {+lexing+} with the rules +defined in the ``'comment'`` state. + +If there wasn't [-a star-] {+an asterisk+} after the slash, the `RegexLexer` checks if it's a +[-singleline-] +{+Singleline+} comment [-(eg:-] {+(i.e.+} followed by a second slash). If this also wasn't the +case it must be a single [-slash-] {+slash, which is not a comment starter+} (the separate +regex for a single slash must also be given, else the slash would be marked as +an error token). + +Inside the ``'comment'`` state, we do the same thing again. Scan until the +lexer finds a star or slash. If it's the opening of a multiline comment, push +the ``'comment'`` state on the stack and continue scanning, again in the +``'comment'`` state. Else, check if it's the end of the multiline comment. If +yes, pop one state from the stack. + +Note: If you pop from an empty stack you'll get an `IndexError`. (There is an +easy way to prevent this from happening: don't ``'#pop'`` in the root state). + +If the `RegexLexer` encounters a newline that is flagged as an error token, the +stack is emptied and the lexer continues scanning in the ``'root'`` state. This +[-helps-] +{+can help+} producing error-tolerant highlighting for erroneous input, e.g. when a +single-line string is not closed. + + +Advanced state tricks +===================== + +There are a few more things you can do with states: + +- You can push multiple states onto the stack if you give a tuple instead of a + simple string as the third item in a rule tuple. For example, if you want to + match a comment containing a directive, something [-like::-] {+like: + + .. code-block:: text+} + + /* <processing directive> rest of comment */ + + you can use this [-rule: + + .. sourcecode:: python-] {+rule::+} + + tokens = { + 'root': [ + (r'/\* <', Comment, ('comment', 'directive')), + ... + ], + 'directive': [ + (r'[^>]*', Comment.Directive), + (r'>', Comment, '#pop'), + ], + 'comment': [ + (r'[^*]+', Comment), + (r'\*/', Comment, '#pop'), + (r'\*', Comment), + ] + } + + When this encounters the above sample, first ``'comment'`` and ``'directive'`` + are pushed onto the stack, then the lexer continues in the directive state + until it finds the closing ``>``, then it continues in the comment state until + the closing ``*/``. Then, both states are popped from the stack again and + lexing continues in the root state. + + .. versionadded:: 0.9 + The tuple can contain the special ``'#push'`` and ``'#pop'`` (but not + ``'#pop:n'``) directives. + + +- You can include the rules of a state in the definition of another. This is + done by using `include` from [-`pygments.lexer`: + + .. sourcecode:: python-] {+`pygments.lexer`::+} + + from pygments.lexer import RegexLexer, bygroups, include + from pygments.token import * + + class ExampleLexer(RegexLexer): + tokens = { + 'comments': [ + (r'/\*.*?\*/', Comment), + (r'//.*?\n', Comment), + ], + 'root': [ + include('comments'), + (r'(function )(\w+)( {)', + bygroups(Keyword, Name, Keyword), 'function'), + (r'.', Text), + ], + 'function': [ + (r'[^}/]+', Text), + include('comments'), + (r'/', Text), + [-(r'}',-] + {+(r'\}',+} Keyword, '#pop'), + ] + } + + This is a hypothetical lexer for a language that consist of functions and + comments. Because comments can occur at toplevel and in functions, we need + rules for comments in both states. As you can see, the `include` helper saves + repeating rules that occur more than once (in this example, the state + ``'comment'`` will never be entered by the lexer, as it's only there to be + included in ``'root'`` and ``'function'``). + +- Sometimes, you may want to "combine" a state from existing ones. This is + possible with the [-`combine`-] {+`combined`+} helper from `pygments.lexer`. + + If you, instead of a new state, write ``combined('state1', 'state2')`` as the + third item of a rule tuple, a new anonymous state will be formed from state1 + and state2 and if the rule matches, the lexer will enter this state. + + This is not used very often, but can be helpful in some cases, such as the + `PythonLexer`'s string literal processing. + +- If you want your lexer to start lexing in a different state you can modify the + stack by [-overloading-] {+overriding+} the `get_tokens_unprocessed()` [-method: + + .. sourcecode:: python-] {+method::+} + + from pygments.lexer import RegexLexer + + class [-MyLexer(RegexLexer):-] {+ExampleLexer(RegexLexer):+} + tokens = {...} + + def get_tokens_unprocessed(self, [-text): + stack = ['root', 'otherstate']-] {+text, stack=('root', 'otherstate')):+} + for item in RegexLexer.get_tokens_unprocessed(text, stack): + yield item + + Some lexers like the `PhpLexer` use this to make the leading ``<?php`` + preprocessor comments optional. Note that you can crash the lexer easily by + putting values into the stack that don't exist in the token map. Also + removing ``'root'`` from the stack can result in strange errors! + +- [-An-] {+In some lexers, a state should be popped if anything is encountered that isn't + matched by a rule in the state. You could use an+} empty regex at the end of [-a-] + {+the+} state list, [-combined with ``'#pop'``, can + act as-] {+but Pygments provides+} a [-return point-] {+more obvious way of spelling that: + ``default('#pop')`` is equivalent to ``('', Text, '#pop')``. + + .. versionadded:: 2.0 + + +Subclassing lexers derived+} from {+RegexLexer +========================================== + +.. versionadded:: 1.6 + +Sometimes multiple languages are very similar, but should still be lexed by +different lexer classes. + +When subclassing+} a {+lexer derived from RegexLexer, the ``tokens`` dictionaries +defined in the parent and child class are merged. For example:: + + from pygments.lexer import RegexLexer, inherit + from pygments.token import * + + class BaseLexer(RegexLexer): + tokens = { + 'root': [ + ('[a-z]+', Name), + (r'/\*', Comment, 'comment'), + ('"', String, 'string'), + ('\s+', Text), + ], + 'string': [ + ('[^"]+', String), + ('"', String, '#pop'), + ], + 'comment': [ + ... + ], + } + + class DerivedLexer(BaseLexer): + tokens = { + 'root': [ + ('[0-9]+', Number), + inherit, + ], + 'string': [ + (r'[^"\\]+', String), + (r'\\.', String.Escape), + ('"', String, '#pop'), + ], + } + +The `BaseLexer` defines two states, lexing names and strings. The +`DerivedLexer` defines its own tokens dictionary, which extends the definitions +of the base lexer: + +* The "root"+} state {+has an additional rule and then the special object `inherit`, + which tells Pygments to insert the token definitions of the parent class at+} + that [-doesn't have a clear end marker.-] {+point. + +* The "string" state is replaced entirely, since there is not `inherit` rule. + +* The "comment" state is inherited entirely.+} + + +Using multiple lexers +===================== + +Using multiple lexers for the same input can be tricky. One of the easiest +combination techniques is shown here: You can replace the [-token type-] {+action+} entry in a rule +tuple [-(the second item)-] with a lexer class. The matched text will then be lexed with that lexer, +and the resulting tokens will be yielded. + +For example, look at this stripped-down HTML [-lexer: + +.. sourcecode:: python-] {+lexer::+} + + from pygments.lexer import RegexLexer, bygroups, using + from pygments.token import * + from [-pygments.lexers.web-] {+pygments.lexers.javascript+} import JavascriptLexer + + class HtmlLexer(RegexLexer): + name = 'HTML' + aliases = ['html'] + filenames = ['*.html', '*.htm'] + + flags = re.IGNORECASE | re.DOTALL + tokens = { + 'root': [ + ('[^<&]+', Text), + ('&.*?;', Name.Entity), + (r'<\s*script\s*', Name.Tag, ('script-content', 'tag')), + (r'<\s*[a-zA-Z0-9:]+', Name.Tag, 'tag'), + (r'<\s*/\s*[a-zA-Z0-9:]+\s*>', Name.Tag), + ], + 'script-content': [ + (r'(.+?)(<\s*/\s*script\s*>)', + bygroups(using(JavascriptLexer), Name.Tag), + '#pop'), + ] + } + +Here the content of a ``<script>`` tag is passed to a newly created instance of +a `JavascriptLexer` and not processed by the `HtmlLexer`. This is done using +the `using` helper that takes the other lexer class as its parameter. + +Note the combination of `bygroups` and `using`. This makes sure that the +content up to the ``</script>`` end tag is processed by the `JavascriptLexer`, +while the end tag is yielded as a normal token with the `Name.Tag` type. + +[-As an additional goodie, if the lexer class is replaced by `this` (imported from +`pygments.lexer`), the "other" lexer will be the current one (because you cannot +refer to the current class within the code that runs at class definition time).-] + +Also note the ``(r'<\s*script\s*', Name.Tag, ('script-content', 'tag'))`` rule. +Here, two states are pushed onto the state stack, ``'script-content'`` and +``'tag'``. That means that first ``'tag'`` is processed, which will [-parse-] {+lex+} +attributes and the closing ``>``, then the ``'tag'`` state is popped and the +next state on top of the stack will be ``'script-content'``. + +{+Since you cannot refer to the class currently being defined, use `this` +(imported from `pygments.lexer`) to refer to the current lexer class, i.e. +``using(this)``. This construct may seem unnecessary, but this is often the +most obvious way of lexing arbitrary syntax between fixed delimiters without +introducing deeply nested states.+} + +The `using()` helper has a special keyword argument, `state`, which works as +follows: if given, the lexer to use initially is not in the ``"root"`` state, +but in the state given by this argument. This [-*only* works-] {+does not work+} with [-a `RegexLexer`.-] {+advanced +`RegexLexer` subclasses such as `ExtendedRegexLexer` (see below).+} + +Any other keywords arguments passed to `using()` are added to the keyword +arguments used to create the lexer. + + +Delegating Lexer +================ + +Another approach for nested lexers is the `DelegatingLexer` which is for example +used for the template engine lexers. It takes two lexers as arguments on +initialisation: a `root_lexer` and a `language_lexer`. + +The input is processed as follows: First, the whole text is lexed with the +`language_lexer`. All tokens yielded with [-a-] {+the special+} type of ``Other`` are +then concatenated and given to the `root_lexer`. The language tokens of the +`language_lexer` are then inserted into the `root_lexer`'s token stream at the +appropriate positions. + +[-.. sourcecode:: python-] {+::+} + + from pygments.lexer import DelegatingLexer + from pygments.lexers.web import HtmlLexer, PhpLexer + + class HtmlPhpLexer(DelegatingLexer): + def __init__(self, **options): + super(HtmlPhpLexer, self).__init__(HtmlLexer, PhpLexer, **options) + +This procedure ensures that e.g. HTML with template tags in it is highlighted +correctly even if the template tags are put into HTML tags or attributes. + +If you want to change the needle token ``Other`` to something else, you can give +the lexer another token type as the third [-parameter: + +.. sourcecode:: python-] {+parameter::+} + + DelegatingLexer.__init__(MyLexer, OtherLexer, Text, **options) + + +Callbacks +========= + +Sometimes the grammar of a language is so complex that a lexer would be unable +to [-parse-] {+process+} it just by using regular expressions and stacks. + +For this, the `RegexLexer` allows callbacks to be given in rule tuples, instead +of token types (`bygroups` and `using` are nothing else but preimplemented +callbacks). The callback must be a function taking two arguments: + +* the lexer itself +* the match object for the last matched rule + +The callback must then return an iterable of (or simply yield) ``(index, +tokentype, value)`` tuples, which are then just passed through by +`get_tokens_unprocessed()`. The ``index`` here is the position of the token in +the input string, ``tokentype`` is the normal token type (like `Name.Builtin`), +and ``value`` the associated part of the input string. + +You can see an example [-here: + +.. sourcecode:: python-] {+here::+} + + from pygments.lexer import RegexLexer + from pygments.token import Generic + + class HypotheticLexer(RegexLexer): + + def headline_callback(lexer, match): + equal_signs = match.group(1) + text = match.group(2) + yield match.start(), Generic.Headline, equal_signs + text + equal_signs + + tokens = { + 'root': [ + (r'(=+)(.*?)(\1)', headline_callback) + ] + } + +If the regex for the `headline_callback` matches, the function is called with +the match object. Note that after the callback is done, processing continues +normally, that is, after the end of the previous match. The callback has no +possibility to influence the position. + +There are not really any simple examples for lexer callbacks, but you can see +them in action e.g. in the [-`compiled.py`_ source code-] {+`SMLLexer` class+} in [-the `CLexer` and +`JavaLexer` classes.-] {+`ml.py`_.+} + +.. [-_compiled.py: http://bitbucket.org/birkenfeld/pygments-main/src/tip/pygments/lexers/compiled.py-] {+_ml.py: http://bitbucket.org/birkenfeld/pygments-main/src/tip/pygments/lexers/ml.py+} + + +The ExtendedRegexLexer class +============================ + +The `RegexLexer`, even with callbacks, unfortunately isn't powerful enough for +the funky syntax rules of [-some-] languages [-that will go unnamed,-] such as Ruby. + +But fear not; even then you don't have to abandon the regular expression +[-approach. For-] +{+approach:+} Pygments has a subclass of `RegexLexer`, the `ExtendedRegexLexer`. +All features known from RegexLexers are available here too, and the tokens are +specified in exactly the same way, *except* for one detail: + +The `get_tokens_unprocessed()` method holds its internal state data not as local +variables, but in an instance of the `pygments.lexer.LexerContext` class, and +that instance is passed to callbacks as a third argument. This means that you +can modify the lexer state in callbacks. + +The `LexerContext` class has the following members: + +* `text` -- the input text +* `pos` -- the current starting position that is used for matching regexes +* `stack` -- a list containing the state stack +* `end` -- the maximum position to which regexes are matched, this defaults to + the length of `text` + +Additionally, the `get_tokens_unprocessed()` method can be given a +`LexerContext` instead of a string and will then process this context instead of +creating a new one for the string argument. + +Note that because you can set the current position to anything in the callback, +it won't be automatically be set by the caller after the callback is finished. +For example, this is how the hypothetical lexer above would be written with the +[-`ExtendedRegexLexer`: + +.. sourcecode:: python-] +{+`ExtendedRegexLexer`::+} + + from pygments.lexer import ExtendedRegexLexer + from pygments.token import Generic + + class ExHypotheticLexer(ExtendedRegexLexer): + + def headline_callback(lexer, match, ctx): + equal_signs = match.group(1) + text = match.group(2) + yield match.start(), Generic.Headline, equal_signs + text + equal_signs + ctx.pos = match.end() + + tokens = { + 'root': [ + (r'(=+)(.*?)(\1)', headline_callback) + ] + } + +This might sound confusing (and it can really be). But it is needed, and for an +example look at the Ruby lexer in [-`agile.py`_.-] {+`ruby.py`_.+} + +.. [-_agile.py: https://bitbucket.org/birkenfeld/pygments-main/src/tip/pygments/lexers/agile.py + + +Filtering-] {+_ruby.py: https://bitbucket.org/birkenfeld/pygments-main/src/tip/pygments/lexers/ruby.py + + +Handling Lists of Keywords +========================== + +For a relatively short list (hundreds) you can construct an optimized regular +expression directly using ``words()`` (longer lists, see next section). This +function handles a few things for you automatically, including escaping +metacharacters and Python's first-match rather than longest-match in +alternations. Feel free to put the lists themselves in +``pygments/lexers/_$lang_builtins.py`` (see examples there), and generated by +code if possible. + +An example of using ``words()`` is something like:: + + from pygments.lexer import RegexLexer, words, Name + + class MyLexer(RegexLexer): + + tokens = { + 'root': [ + (words(('else', 'elseif'), suffix=r'\b'), Name.Builtin), + (r'\w+', Name), + ], + } + +As you can see, you can add ``prefix`` and ``suffix`` parts to the constructed +regex. + + +Modifying+} Token Streams +======================= + +Some languages ship a lot of builtin functions (for example PHP). The total +amount of those functions differs from system to system because not everybody +has every extension installed. In the case of PHP there are over 3000 builtin +functions. That's an [-incredible-] {+incredibly+} huge amount of functions, much more than you +[-can-] +{+want to+} put into a regular expression. + +But because only `Name` tokens can be function names [-it's-] {+this is+} solvable by +overriding the ``get_tokens_unprocessed()`` method. The following lexer +subclasses the `PythonLexer` so that it highlights some additional names as +pseudo [-keywords: + +.. sourcecode:: python-] {+keywords::+} + + from [-pygments.lexers.agile-] {+pygments.lexers.python+} import PythonLexer + from pygments.token import Name, Keyword + + class MyPythonLexer(PythonLexer): + EXTRA_KEYWORDS = [-['foo',-] {+set(('foo',+} 'bar', 'foobar', 'barfoo', 'spam', [-'eggs']-] {+'eggs'))+} + + def get_tokens_unprocessed(self, text): + for index, token, value in PythonLexer.get_tokens_unprocessed(self, text): + if token is Name and value in self.EXTRA_KEYWORDS: + yield index, Keyword.Pseudo, value + else: + yield index, token, value + +The `PhpLexer` and `LuaLexer` use this method to resolve builtin functions. + +[-.. note:: Do not confuse this with the :doc:`filter <filters>` system.-] diff --git a/tests/examplefiles/wdiff_example3.wdiff b/tests/examplefiles/wdiff_example3.wdiff new file mode 100644 index 00000000..0bbd6d65 --- /dev/null +++ b/tests/examplefiles/wdiff_example3.wdiff @@ -0,0 +1,10 @@ +This example is unbalanced open-close. +We can't treat these easily. + +{+ added? -] +[- deleted? +} + +suddenly closed -] +suddenly closed +} + +{+ added? [- deleted? diff --git a/tests/run.py b/tests/run.py index 8167b911..07665b2a 100644 --- a/tests/run.py +++ b/tests/run.py @@ -8,7 +8,7 @@ python run.py [testfile ...] - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/string_asserts.py b/tests/string_asserts.py index 11f5c7f0..05e95e6a 100644 --- a/tests/string_asserts.py +++ b/tests/string_asserts.py @@ -3,7 +3,7 @@ Pygments string assert utility ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/support/empty.py b/tests/support/empty.py new file mode 100644 index 00000000..40a96afc --- /dev/null +++ b/tests/support/empty.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/support/html_formatter.py b/tests/support/html_formatter.py new file mode 100644 index 00000000..169cd4af --- /dev/null +++ b/tests/support/html_formatter.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +from pygments.formatters import HtmlFormatter + + +class HtmlFormatterWrapper(HtmlFormatter): + name = 'HtmlWrapper' diff --git a/tests/support/python_lexer.py b/tests/support/python_lexer.py new file mode 100644 index 00000000..565ee674 --- /dev/null +++ b/tests/support/python_lexer.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# pygments.lexers.python (as CustomLexer) for test_cmdline.py + +from pygments.lexers import PythonLexer + + +class CustomLexer(PythonLexer): + name = 'PythonLexerWrapper' + + +class LexerWrapper(CustomLexer): + name = 'PythonLexerWrapperWrapper' diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 022e6c55..ac3b4a51 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -3,7 +3,7 @@ Pygments basic API tests ~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -161,8 +161,8 @@ def test_formatter_public_api(): try: inst = formatter(opt1="val1") - except (ImportError, FontNotFound): - raise support.SkipTest + except (ImportError, FontNotFound) as e: + raise support.SkipTest(e) try: inst.get_style_defs() @@ -209,9 +209,9 @@ def test_formatter_unicode_handling(): def verify(formatter): try: inst = formatter(encoding=None) - except (ImportError, FontNotFound): + except (ImportError, FontNotFound) as e: # some dependency or font not installed - raise support.SkipTest + raise support.SkipTest(e) if formatter.name != 'Raw tokens': out = format(tokens, inst) diff --git a/tests/test_bibtex.py b/tests/test_bibtex.py new file mode 100644 index 00000000..5ad92db4 --- /dev/null +++ b/tests/test_bibtex.py @@ -0,0 +1,236 @@ +# -*- coding: utf-8 -*- +""" + BibTeX Test + ~~~~~~~~~~~ + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import textwrap +import unittest + +from pygments.lexers import BibTeXLexer, BSTLexer +from pygments.token import Token + + +class BibTeXTest(unittest.TestCase): + def setUp(self): + self.lexer = BibTeXLexer() + + def testPreamble(self): + data = u'@PREAMBLE{"% some LaTeX code here"}' + tokens = [ + (Token.Name.Class, u'@PREAMBLE'), + (Token.Punctuation, u'{'), + (Token.String, u'"'), + (Token.String, u'% some LaTeX code here'), + (Token.String, u'"'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + ] + self.assertEqual(list(self.lexer.get_tokens(data)), tokens) + + def testString(self): + data = u'@STRING(SCI = "Science")' + tokens = [ + (Token.Name.Class, u'@STRING'), + (Token.Punctuation, u'('), + (Token.Name.Attribute, u'SCI'), + (Token.Text, u' '), + (Token.Punctuation, u'='), + (Token.Text, u' '), + (Token.String, u'"'), + (Token.String, u'Science'), + (Token.String, u'"'), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + ] + self.assertEqual(list(self.lexer.get_tokens(data)), tokens) + + def testEntry(self): + data = u""" + This is a comment. + + @ARTICLE{ruckenstein-diffusion, + author = "Liu, Hongquin" # and # "Ruckenstein, Eli", + year = 1997, + month = JAN, + pages = "888-895" + } + """ + + tokens = [ + (Token.Comment, u'This is a comment.'), + (Token.Text, u'\n\n'), + (Token.Name.Class, u'@ARTICLE'), + (Token.Punctuation, u'{'), + (Token.Name.Label, u'ruckenstein-diffusion'), + (Token.Punctuation, u','), + (Token.Text, u'\n '), + (Token.Name.Attribute, u'author'), + (Token.Text, u' '), + (Token.Punctuation, u'='), + (Token.Text, u' '), + (Token.String, u'"'), + (Token.String, u'Liu, Hongquin'), + (Token.String, u'"'), + (Token.Text, u' '), + (Token.Punctuation, u'#'), + (Token.Text, u' '), + (Token.Name.Variable, u'and'), + (Token.Text, u' '), + (Token.Punctuation, u'#'), + (Token.Text, u' '), + (Token.String, u'"'), + (Token.String, u'Ruckenstein, Eli'), + (Token.String, u'"'), + (Token.Punctuation, u','), + (Token.Text, u'\n '), + (Token.Name.Attribute, u'year'), + (Token.Text, u' '), + (Token.Punctuation, u'='), + (Token.Text, u' '), + (Token.Number, u'1997'), + (Token.Punctuation, u','), + (Token.Text, u'\n '), + (Token.Name.Attribute, u'month'), + (Token.Text, u' '), + (Token.Punctuation, u'='), + (Token.Text, u' '), + (Token.Name.Variable, u'JAN'), + (Token.Punctuation, u','), + (Token.Text, u'\n '), + (Token.Name.Attribute, u'pages'), + (Token.Text, u' '), + (Token.Punctuation, u'='), + (Token.Text, u' '), + (Token.String, u'"'), + (Token.String, u'888-895'), + (Token.String, u'"'), + (Token.Text, u'\n'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + ] + self.assertEqual(list(self.lexer.get_tokens(textwrap.dedent(data))), tokens) + + def testComment(self): + data = '@COMMENT{test}' + tokens = [ + (Token.Comment, u'@COMMENT'), + (Token.Comment, u'{test}'), + (Token.Text, u'\n'), + ] + self.assertEqual(list(self.lexer.get_tokens(data)), tokens) + + def testMissingBody(self): + data = '@ARTICLE xxx' + tokens = [ + (Token.Name.Class, u'@ARTICLE'), + (Token.Text, u' '), + (Token.Error, u'x'), + (Token.Error, u'x'), + (Token.Error, u'x'), + (Token.Text, u'\n'), + ] + self.assertEqual(list(self.lexer.get_tokens(data)), tokens) + + def testMismatchedBrace(self): + data = '@PREAMBLE(""}' + tokens = [ + (Token.Name.Class, u'@PREAMBLE'), + (Token.Punctuation, u'('), + (Token.String, u'"'), + (Token.String, u'"'), + (Token.Error, u'}'), + (Token.Text, u'\n'), + ] + self.assertEqual(list(self.lexer.get_tokens(data)), tokens) + + +class BSTTest(unittest.TestCase): + def setUp(self): + self.lexer = BSTLexer() + + def testBasicBST(self): + data = """ + % BibTeX standard bibliography style `plain' + + INTEGERS { output.state before.all } + + FUNCTION {sort.format.title} + { 't := + "A " #2 + "An " #3 + "The " #4 t chop.word + chop.word + chop.word + sortify + #1 global.max$ substring$ + } + + ITERATE {call.type$} + """ + tokens = [ + (Token.Comment.SingleLine, "% BibTeX standard bibliography style `plain'"), + (Token.Text, u'\n\n'), + (Token.Keyword, u'INTEGERS'), + (Token.Text, u' '), + (Token.Punctuation, u'{'), + (Token.Text, u' '), + (Token.Name.Variable, u'output.state'), + (Token.Text, u' '), + (Token.Name.Variable, u'before.all'), + (Token.Text, u' '), + (Token.Punctuation, u'}'), + (Token.Text, u'\n\n'), + (Token.Keyword, u'FUNCTION'), + (Token.Text, u' '), + (Token.Punctuation, u'{'), + (Token.Name.Variable, u'sort.format.title'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + (Token.Punctuation, u'{'), + (Token.Text, u' '), + (Token.Name.Function, u"'t"), + (Token.Text, u' '), + (Token.Name.Variable, u':='), + (Token.Text, u'\n'), + (Token.Literal.String, u'"A "'), + (Token.Text, u' '), + (Token.Literal.Number, u'#2'), + (Token.Text, u'\n '), + (Token.Literal.String, u'"An "'), + (Token.Text, u' '), + (Token.Literal.Number, u'#3'), + (Token.Text, u'\n '), + (Token.Literal.String, u'"The "'), + (Token.Text, u' '), + (Token.Literal.Number, u'#4'), + (Token.Text, u' '), + (Token.Name.Variable, u't'), + (Token.Text, u' '), + (Token.Name.Variable, u'chop.word'), + (Token.Text, u'\n '), + (Token.Name.Variable, u'chop.word'), + (Token.Text, u'\n'), + (Token.Name.Variable, u'chop.word'), + (Token.Text, u'\n'), + (Token.Name.Variable, u'sortify'), + (Token.Text, u'\n'), + (Token.Literal.Number, u'#1'), + (Token.Text, u' '), + (Token.Name.Builtin, u'global.max$'), + (Token.Text, u' '), + (Token.Name.Builtin, u'substring$'), + (Token.Text, u'\n'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n\n'), + (Token.Keyword, u'ITERATE'), + (Token.Text, u' '), + (Token.Punctuation, u'{'), + (Token.Name.Builtin, u'call.type$'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + ] + self.assertEqual(list(self.lexer.get_tokens(textwrap.dedent(data))), tokens) diff --git a/tests/test_cfm.py b/tests/test_cfm.py index 2585489a..0ff1b167 100644 --- a/tests/test_cfm.py +++ b/tests/test_cfm.py @@ -3,7 +3,7 @@ Basic ColdfusionHtmlLexer Test ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_clexer.py b/tests/test_clexer.py index fd7f58fc..5095b797 100644 --- a/tests/test_clexer.py +++ b/tests/test_clexer.py @@ -3,7 +3,7 @@ Basic CLexer Test ~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py index 5883fb5c..1500c875 100644 --- a/tests/test_cmdline.py +++ b/tests/test_cmdline.py @@ -3,7 +3,7 @@ Command line test ~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -110,6 +110,32 @@ class CmdLineTest(unittest.TestCase): finally: os.unlink(name) + def test_load_from_file(self): + lexer_file = os.path.join(TESTDIR, 'support', 'python_lexer.py') + formatter_file = os.path.join(TESTDIR, 'support', 'html_formatter.py') + + # By default, use CustomLexer + o = self.check_success('-l', lexer_file, '-f', 'html', + '-x', stdin=TESTCODE) + o = re.sub('<[^>]*>', '', o) + # rstrip is necessary since HTML inserts a \n after the last </div> + self.assertEqual(o.rstrip(), TESTCODE.rstrip()) + + # If user specifies a name, use it + o = self.check_success('-f', 'html', '-x', '-l', + lexer_file + ':LexerWrapper', stdin=TESTCODE) + o = re.sub('<[^>]*>', '', o) + # rstrip is necessary since HTML inserts a \n after the last </div> + self.assertEqual(o.rstrip(), TESTCODE.rstrip()) + + # Should also work for formatters + o = self.check_success('-lpython', '-f', + formatter_file + ':HtmlFormatterWrapper', + '-x', stdin=TESTCODE) + o = re.sub('<[^>]*>', '', o) + # rstrip is necessary since HTML inserts a \n after the last </div> + self.assertEqual(o.rstrip(), TESTCODE.rstrip()) + def test_stream_opt(self): o = self.check_success('-lpython', '-s', '-fterminal', stdin=TESTCODE) o = re.sub(r'\x1b\[.*?m', '', o) @@ -211,6 +237,20 @@ class CmdLineTest(unittest.TestCase): e = self.check_failure('-lfooo', TESTFILE) self.assertTrue('Error: no lexer for alias' in e) + # cannot load .py file without load_from_file flag + e = self.check_failure('-l', 'nonexistent.py', TESTFILE) + self.assertTrue('Error: no lexer for alias' in e) + + # lexer file is missing/unreadable + e = self.check_failure('-l', 'nonexistent.py', + '-x', TESTFILE) + self.assertTrue('Error: cannot read' in e) + + # lexer file is malformed + e = self.check_failure('-l', 'support/empty.py', + '-x', TESTFILE) + self.assertTrue('Error: no valid CustomLexer class found' in e) + # formatter not found e = self.check_failure('-lpython', '-ffoo', TESTFILE) self.assertTrue('Error: no formatter found for name' in e) @@ -219,6 +259,20 @@ class CmdLineTest(unittest.TestCase): e = self.check_failure('-ofoo.foo', TESTFILE) self.assertTrue('Error: no formatter found for file name' in e) + # cannot load .py file without load_from_file flag + e = self.check_failure('-f', 'nonexistent.py', TESTFILE) + self.assertTrue('Error: no formatter found for name' in e) + + # formatter file is missing/unreadable + e = self.check_failure('-f', 'nonexistent.py', + '-x', TESTFILE) + self.assertTrue('Error: cannot read' in e) + + # formatter file is malformed + e = self.check_failure('-f', 'support/empty.py', + '-x', TESTFILE) + self.assertTrue('Error: no valid CustomFormatter class found' in e) + # output file not writable e = self.check_failure('-o', os.path.join('nonexistent', 'dir', 'out.html'), '-lpython', TESTFILE) diff --git a/tests/test_cpp.py b/tests/test_cpp.py new file mode 100644 index 00000000..ef59965c --- /dev/null +++ b/tests/test_cpp.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" + CPP Tests + ~~~~~~~~~ + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest + +from pygments.lexers import CppLexer +from pygments.token import Token + + +class CppTest(unittest.TestCase): + def setUp(self): + self.lexer = CppLexer() + + def testGoodComment(self): + fragment = u'/* foo */\n' + tokens = [ + (Token.Comment.Multiline, u'/* foo */'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testOpenComment(self): + fragment = u'/* foo\n' + tokens = [ + (Token.Comment.Multiline, u'/* foo\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_crystal.py b/tests/test_crystal.py new file mode 100644 index 00000000..9a1588f2 --- /dev/null +++ b/tests/test_crystal.py @@ -0,0 +1,308 @@ +# -*- coding: utf-8 -*- +""" + Basic CrystalLexer Test + ~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from __future__ import unicode_literals +import unittest + +from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ + Number, Punctuation, Error +from pygments.lexers import CrystalLexer + + +class CrystalTest(unittest.TestCase): + + def setUp(self): + self.lexer = CrystalLexer() + self.maxDiff = None + + def testRangeSyntax1(self): + fragment = '1...3\n' + tokens = [ + (Number.Integer, '1'), + (Operator, '...'), + (Number.Integer, '3'), + (Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testRangeSyntax2(self): + fragment = '1 .. 3\n' + tokens = [ + (Number.Integer, '1'), + (Text, ' '), + (Operator, '..'), + (Text, ' '), + (Number.Integer, '3'), + (Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testInterpolationNestedCurly(self): + fragment = ( + '"A#{ (3..5).group_by { |x| x/2}.map ' + 'do |k,v| "#{k}" end.join }" + "Z"\n') + tokens = [ + (String.Double, '"'), + (String.Double, 'A'), + (String.Interpol, '#{'), + (Text, ' '), + (Punctuation, '('), + (Number.Integer, '3'), + (Operator, '..'), + (Number.Integer, '5'), + (Punctuation, ')'), + (Operator, '.'), + (Name, 'group_by'), + (Text, ' '), + (String.Interpol, '{'), + (Text, ' '), + (Operator, '|'), + (Name, 'x'), + (Operator, '|'), + (Text, ' '), + (Name, 'x'), + (Operator, '/'), + (Number.Integer, '2'), + (String.Interpol, '}'), + (Operator, '.'), + (Name, 'map'), + (Text, ' '), + (Keyword, 'do'), + (Text, ' '), + (Operator, '|'), + (Name, 'k'), + (Punctuation, ','), + (Name, 'v'), + (Operator, '|'), + (Text, ' '), + (String.Double, '"'), + (String.Interpol, '#{'), + (Name, 'k'), + (String.Interpol, '}'), + (String.Double, '"'), + (Text, ' '), + (Keyword, 'end'), + (Operator, '.'), + (Name, 'join'), + (Text, ' '), + (String.Interpol, '}'), + (String.Double, '"'), + (Text, ' '), + (Operator, '+'), + (Text, ' '), + (String.Double, '"'), + (String.Double, 'Z'), + (String.Double, '"'), + (Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testOperatorMethods(self): + fragment = '([] of Int32).[]?(5)\n' + tokens = [ + (Punctuation, '('), + (Operator, '['), + (Operator, ']'), + (Text, ' '), + (Keyword, 'of'), + (Text, ' '), + (Name.Builtin, 'Int32'), + (Punctuation, ')'), + (Operator, '.'), + (Name.Operator, '[]?'), + (Punctuation, '('), + (Number.Integer, '5'), + (Punctuation, ')'), + (Text, '\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testArrayAccess(self): + fragment = '[5][5]?\n' + tokens = [ + (Operator, '['), + (Number.Integer, '5'), + (Operator, ']'), + (Operator, '['), + (Number.Integer, '5'), + (Operator, ']?'), + (Text, '\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testNumbers(self): + for kind, testset in [ + (Number.Integer, '0 1 1_000_000 1u8 11231231231121312i64'), + (Number.Float, '0.0 1.0_f32 1_f32 0f64 1e+4 1e111 1_234.567_890'), + (Number.Bin, '0b1001_0110 0b0u8'), + (Number.Oct, '0o17 0o7_i32'), + (Number.Hex, '0xdeadBEEF'), + ]: + for fragment in testset.split(): + self.assertEqual([(kind, fragment), (Text, '\n')], + list(self.lexer.get_tokens(fragment + '\n'))) + + for fragment in '01 0b2 0x129g2 0o12358'.split(): + self.assertEqual(next(self.lexer.get_tokens(fragment + '\n'))[0], + Error) + + def testChars(self): + for fragment in ["'a'", "'я'", "'\\u{1234}'", "'\n'"]: + self.assertEqual([(String.Char, fragment), (Text, '\n')], + list(self.lexer.get_tokens(fragment + '\n'))) + self.assertEqual(next(self.lexer.get_tokens("'abc'"))[0], Error) + + def testMacro(self): + fragment = ( + 'def<=>(other : self) : Int\n' + '{%for field in %w(first_name middle_name last_name)%}\n' + 'cmp={{field.id}}<=>other.{{field.id}}\n' + 'return cmp if cmp!=0\n' + '{%end%}\n' + '0\n' + 'end\n') + tokens = [ + (Keyword, 'def'), + (Name.Function, '<=>'), + (Punctuation, '('), + (Name, 'other'), + (Text, ' '), + (Punctuation, ':'), + (Text, ' '), + (Keyword.Pseudo, 'self'), + (Punctuation, ')'), + (Text, ' '), + (Punctuation, ':'), + (Text, ' '), + (Name.Builtin, 'Int'), + (Text, '\n'), + (String.Interpol, '{%'), + (Keyword, 'for'), + (Text, ' '), + (Name, 'field'), + (Text, ' '), + (Keyword, 'in'), + (Text, ' '), + (String.Other, '%w('), + (String.Other, 'first_name middle_name last_name'), + (String.Other, ')'), + (String.Interpol, '%}'), + (Text, '\n'), + (Name, 'cmp'), + (Operator, '='), + (String.Interpol, '{{'), + (Name, 'field'), + (Operator, '.'), + (Name, 'id'), + (String.Interpol, '}}'), + (Operator, '<=>'), + (Name, 'other'), + (Operator, '.'), + (String.Interpol, '{{'), + (Name, 'field'), + (Operator, '.'), + (Name, 'id'), + (String.Interpol, '}}'), + (Text, '\n'), + (Keyword, 'return'), + (Text, ' '), + (Name, 'cmp'), + (Text, ' '), + (Keyword, 'if'), + (Text, ' '), + (Name, 'cmp'), + (Operator, '!='), + (Number.Integer, '0'), + (Text, '\n'), + (String.Interpol, '{%'), + (Keyword, 'end'), + (String.Interpol, '%}'), + (Text, '\n'), + (Number.Integer, '0'), + (Text, '\n'), + (Keyword, 'end'), + (Text, '\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testLib(self): + fragment = ( + '@[Link("some")]\nlib LibSome\n' + '@[CallConvention("X86_StdCall")]\nfun foo="some.foo"(thing : Void*) : LibC::Int\n' + 'end\n') + tokens = [ + (Operator, '@['), + (Name.Decorator, 'Link'), + (Punctuation, '('), + (String.Double, '"'), + (String.Double, 'some'), + (String.Double, '"'), + (Punctuation, ')'), + (Operator, ']'), + (Text, '\n'), + (Keyword, 'lib'), + (Text, ' '), + (Name.Namespace, 'LibSome'), + (Text, '\n'), + (Operator, '@['), + (Name.Decorator, 'CallConvention'), + (Punctuation, '('), + (String.Double, '"'), + (String.Double, 'X86_StdCall'), + (String.Double, '"'), + (Punctuation, ')'), + (Operator, ']'), + (Text, '\n'), + (Keyword, 'fun'), + (Text, ' '), + (Name.Function, 'foo'), + (Operator, '='), + (String.Double, '"'), + (String.Double, 'some.foo'), + (String.Double, '"'), + (Punctuation, '('), + (Name, 'thing'), + (Text, ' '), + (Punctuation, ':'), + (Text, ' '), + (Name.Builtin, 'Void'), + (Operator, '*'), + (Punctuation, ')'), + (Text, ' '), + (Punctuation, ':'), + (Text, ' '), + (Name, 'LibC'), + (Operator, '::'), + (Name.Builtin, 'Int'), + (Text, '\n'), + (Keyword, 'end'), + (Text, '\n') + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testEscapedBracestring(self): + fragment = 'str.gsub(%r{\\\\\\\\}, "/")\n' + tokens = [ + (Name, 'str'), + (Operator, '.'), + (Name, 'gsub'), + (Punctuation, '('), + (String.Regex, '%r{'), + (String.Regex, '\\\\'), + (String.Regex, '\\\\'), + (String.Regex, '}'), + (Punctuation, ','), + (Text, ' '), + (String.Double, '"'), + (String.Double, '/'), + (String.Double, '"'), + (Punctuation, ')'), + (Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_data.py b/tests/test_data.py new file mode 100644 index 00000000..be371419 --- /dev/null +++ b/tests/test_data.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +""" + Data Tests + ~~~~~~~~~~ + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest + +from pygments.lexers import JsonLexer, JsonBareObjectLexer +from pygments.token import Token + + +class JsonTest(unittest.TestCase): + def setUp(self): + self.lexer = JsonLexer() + + def testBasic(self): + fragment = u'{"foo": "bar", "foo2": [1, 2, 3]}\n' + tokens = [ + (Token.Punctuation, u'{'), + (Token.Name.Tag, u'"foo"'), + (Token.Punctuation, u':'), + (Token.Text, u' '), + (Token.Literal.String.Double, u'"bar"'), + (Token.Punctuation, u','), + (Token.Text, u' '), + (Token.Name.Tag, u'"foo2"'), + (Token.Punctuation, u':'), + (Token.Text, u' '), + (Token.Punctuation, u'['), + (Token.Literal.Number.Integer, u'1'), + (Token.Punctuation, u','), + (Token.Text, u' '), + (Token.Literal.Number.Integer, u'2'), + (Token.Punctuation, u','), + (Token.Text, u' '), + (Token.Literal.Number.Integer, u'3'), + (Token.Punctuation, u']'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + +class JsonBareObjectTest(unittest.TestCase): + def setUp(self): + self.lexer = JsonBareObjectLexer() + + def testBasic(self): + # This is the same as testBasic for JsonLexer above, except the + # enclosing curly braces are removed. + fragment = u'"foo": "bar", "foo2": [1, 2, 3]\n' + tokens = [ + (Token.Name.Tag, u'"foo"'), + (Token.Punctuation, u':'), + (Token.Text, u' '), + (Token.Literal.String.Double, u'"bar"'), + (Token.Punctuation, u','), + (Token.Text, u' '), + (Token.Name.Tag, u'"foo2"'), + (Token.Punctuation, u':'), + (Token.Text, u' '), + (Token.Punctuation, u'['), + (Token.Literal.Number.Integer, u'1'), + (Token.Punctuation, u','), + (Token.Text, u' '), + (Token.Literal.Number.Integer, u'2'), + (Token.Punctuation, u','), + (Token.Text, u' '), + (Token.Literal.Number.Integer, u'3'), + (Token.Punctuation, u']'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testClosingCurly(self): + # This can be an Error token, but should not be a can't-pop-from-stack + # exception. + fragment = '}"a"\n' + tokens = [ + (Token.Error, '}'), + (Token.Name.Tag, '"a"'), + (Token.Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testClosingCurlyInValue(self): + fragment = '"": ""}\n' + tokens = [ + (Token.Name.Tag, '""'), + (Token.Punctuation, ':'), + (Token.Text, ' '), + (Token.Literal.String.Double, '""'), + (Token.Error, '}'), + (Token.Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py index 924e1184..2fae1125 100644 --- a/tests/test_examplefiles.py +++ b/tests/test_examplefiles.py @@ -3,7 +3,7 @@ Pygments tests with example files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -46,6 +46,10 @@ def test_example_files(): if not os.path.isfile(absfn): continue + extension = os.getenv('TEST_EXT') + if extension and not absfn.endswith(extension): + continue + print(absfn) with open(absfn, 'rb') as f: code = f.read() @@ -85,7 +89,7 @@ def test_example_files(): def check_lexer(lx, fn): if os.name == 'java' and fn in BAD_FILES_FOR_JYTHON: - raise support.SkipTest + raise support.SkipTest('%s is a known bad file on Jython' % fn) absfn = os.path.join(TESTDIR, 'examplefiles', fn) with open(absfn, 'rb') as fp: text = fp.read() diff --git a/tests/test_ezhil.py b/tests/test_ezhil.py index 23b9cb41..15cc13b1 100644 --- a/tests/test_ezhil.py +++ b/tests/test_ezhil.py @@ -94,7 +94,8 @@ class EzhilTest(unittest.TestCase): முடி\n""" tokens = [ (Token.Comment.Single, - u'# (C) \u0bae\u0bc1\u0ba4\u0bcd\u0ba4\u0bc8\u0baf\u0bbe \u0b85\u0ba3\u0bcd\u0ba3\u0bbe\u0bae\u0bb2\u0bc8 2013, 2015\n'), + u'# (C) \u0bae\u0bc1\u0ba4\u0bcd\u0ba4\u0bc8\u0baf\u0bbe \u0b85' + u'\u0ba3\u0bcd\u0ba3\u0bbe\u0bae\u0bb2\u0bc8 2013, 2015\n'), (Token.Keyword,u'நிரல்பாகம்'), (Token.Text, u' '), (Token.Name, u'gcd'), diff --git a/tests/test_html_formatter.py b/tests/test_html_formatter.py index 596d9fbc..79990edd 100644 --- a/tests/test_html_formatter.py +++ b/tests/test_html_formatter.py @@ -3,7 +3,7 @@ Pygments HTML formatter tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_inherit.py b/tests/test_inherit.py index 34033a08..5da57dd9 100644 --- a/tests/test_inherit.py +++ b/tests/test_inherit.py @@ -3,7 +3,7 @@ Tests for inheritance in RegexLexer ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_irc_formatter.py b/tests/test_irc_formatter.py index 16a8fd30..3b34f0bc 100644 --- a/tests/test_irc_formatter.py +++ b/tests/test_irc_formatter.py @@ -3,7 +3,7 @@ Pygments IRC formatter tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_java.py b/tests/test_java.py index f4096647..6e5e8992 100644 --- a/tests/test_java.py +++ b/tests/test_java.py @@ -3,7 +3,7 @@ Basic JavaLexer Test ~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_javascript.py b/tests/test_javascript.py new file mode 100644 index 00000000..21dff7c4 --- /dev/null +++ b/tests/test_javascript.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +""" + Javascript tests + ~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest + +from pygments.lexers import CoffeeScriptLexer +from pygments.token import Token + +COFFEE_SLASH_GOLDEN = [ + # input_str, slashes_are_regex_here + (r'/\\/', True), + (r'/\\/i', True), + (r'/\//', True), + (r'/(\s)/', True), + ('/a{2,8}/', True), + ('/b*c?d+/', True), + ('/(capture-match)/', True), + ('/(?:do-not-capture-match)/', True), + ('/this|or|that/', True), + ('/[char-set]/', True), + ('/[^neg-char_st]/', True), + ('/^.*$/', True), + (r'/\n(\f)\0\1\d\b\cm\u1234/', True), + (r'/^.?([^/\\\n\w]*)a\1+$/.something(or_other) # something more complex', True), + ("foo = (str) ->\n /'|\"/.test str", True), + ('a = a / b / c', False), + ('a = a/b/c', False), + ('a = a/b/ c', False), + ('a = a /b/c', False), + ('a = 1 + /d/.test(a)', True), +] + +def test_coffee_slashes(): + for input_str, slashes_are_regex_here in COFFEE_SLASH_GOLDEN: + yield coffee_runner, input_str, slashes_are_regex_here + +def coffee_runner(input_str, slashes_are_regex_here): + lex = CoffeeScriptLexer() + output = list(lex.get_tokens(input_str)) + print(output) + for t, s in output: + if '/' in s: + is_regex = t is Token.String.Regex + assert is_regex == slashes_are_regex_here, (t, s) + +class CoffeeTest(unittest.TestCase): + def setUp(self): + self.lexer = CoffeeScriptLexer() + + def testMixedSlashes(self): + fragment = u'a?/foo/:1/2;\n' + tokens = [ + (Token.Name.Other, u'a'), + (Token.Operator, u'?'), + (Token.Literal.String.Regex, u'/foo/'), + (Token.Operator, u':'), + (Token.Literal.Number.Integer, u'1'), + (Token.Operator, u'/'), + (Token.Literal.Number.Integer, u'2'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testBewareInfiniteLoop(self): + # This demonstrates the case that "This isn't really guarding" comment + # refers to. + fragment = '/a/x;\n' + tokens = [ + (Token.Text, ''), + (Token.Operator, '/'), + (Token.Name.Other, 'a'), + (Token.Operator, '/'), + (Token.Name.Other, 'x'), + (Token.Punctuation, ';'), + (Token.Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_julia.py b/tests/test_julia.py new file mode 100644 index 00000000..ed46f27e --- /dev/null +++ b/tests/test_julia.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +""" + Julia Tests + ~~~~~~~~~~~ + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest + +from pygments.lexers import JuliaLexer +from pygments.token import Token + + +class JuliaTests(unittest.TestCase): + def setUp(self): + self.lexer = JuliaLexer() + + def test_unicode(self): + """ + Test that unicode character, √, in an expression is recognized + """ + fragment = u's = \u221a((1/n) * sum(count .^ 2) - mu .^2)\n' + tokens = [ + (Token.Name, u's'), + (Token.Text, u' '), + (Token.Operator, u'='), + (Token.Text, u' '), + (Token.Operator, u'\u221a'), + (Token.Punctuation, u'('), + (Token.Punctuation, u'('), + (Token.Literal.Number.Integer, u'1'), + (Token.Operator, u'/'), + (Token.Name, u'n'), + (Token.Punctuation, u')'), + (Token.Text, u' '), + (Token.Operator, u'*'), + (Token.Text, u' '), + (Token.Name, u'sum'), + (Token.Punctuation, u'('), + (Token.Name, u'count'), + (Token.Text, u' '), + (Token.Operator, u'.^'), + (Token.Text, u' '), + (Token.Literal.Number.Integer, u'2'), + (Token.Punctuation, u')'), + (Token.Text, u' '), + (Token.Operator, u'-'), + (Token.Text, u' '), + (Token.Name, u'mu'), + (Token.Text, u' '), + (Token.Operator, u'.^'), + (Token.Literal.Number.Integer, u'2'), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_latex_formatter.py b/tests/test_latex_formatter.py index 05a6c3ac..ebed7964 100644 --- a/tests/test_latex_formatter.py +++ b/tests/test_latex_formatter.py @@ -3,7 +3,7 @@ Pygments LaTeX formatter tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -42,9 +42,9 @@ class LatexFormatterTest(unittest.TestCase): ret = po.wait() output = po.stdout.read() po.stdout.close() - except OSError: + except OSError as e: # latex not available - raise support.SkipTest + raise support.SkipTest(e) else: if ret: print(output) diff --git a/tests/test_lexers_other.py b/tests/test_lexers_other.py index bb667c05..3716fb72 100644 --- a/tests/test_lexers_other.py +++ b/tests/test_lexers_other.py @@ -3,7 +3,7 @@ Tests for other lexers ~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import glob @@ -13,6 +13,7 @@ import unittest from pygments.lexers import guess_lexer from pygments.lexers.scripting import EasytrieveLexer, JclLexer, RexxLexer + def _exampleFilePath(filename): return os.path.join(os.path.dirname(__file__), 'examplefiles', filename) @@ -28,8 +29,8 @@ class AnalyseTextTest(unittest.TestCase): text = fp.read().decode('utf-8') probability = lexer.analyse_text(text) self.assertTrue(probability > 0, - '%s must recognize %r' % ( - lexer.name, exampleFilePath)) + '%s must recognize %r' % ( + lexer.name, exampleFilePath)) guessedLexer = guess_lexer(text) self.assertEqual(guessedLexer.name, lexer.name) @@ -45,25 +46,24 @@ class AnalyseTextTest(unittest.TestCase): class EasyTrieveLexerTest(unittest.TestCase): def testCanGuessFromText(self): - self.assertLess(0, EasytrieveLexer.analyse_text('MACRO')) - self.assertLess(0, EasytrieveLexer.analyse_text('\nMACRO')) - self.assertLess(0, EasytrieveLexer.analyse_text(' \nMACRO')) - self.assertLess(0, EasytrieveLexer.analyse_text(' \n MACRO')) - self.assertLess(0, EasytrieveLexer.analyse_text('*\nMACRO')) - self.assertLess(0, EasytrieveLexer.analyse_text( + self.assertTrue(EasytrieveLexer.analyse_text('MACRO')) + self.assertTrue(EasytrieveLexer.analyse_text('\nMACRO')) + self.assertTrue(EasytrieveLexer.analyse_text(' \nMACRO')) + self.assertTrue(EasytrieveLexer.analyse_text(' \n MACRO')) + self.assertTrue(EasytrieveLexer.analyse_text('*\nMACRO')) + self.assertTrue(EasytrieveLexer.analyse_text( '*\n *\n\n \n*\n MACRO')) class RexxLexerTest(unittest.TestCase): def testCanGuessFromText(self): - self.assertAlmostEqual(0.01, - RexxLexer.analyse_text('/* */')) + self.assertAlmostEqual(0.01, RexxLexer.analyse_text('/* */')) self.assertAlmostEqual(1.0, - RexxLexer.analyse_text('''/* Rexx */ + RexxLexer.analyse_text('''/* Rexx */ say "hello world"''')) val = RexxLexer.analyse_text('/* */\n' - 'hello:pRoceduRe\n' - ' say "hello world"') + 'hello:pRoceduRe\n' + ' say "hello world"') self.assertTrue(val > 0.5, val) val = RexxLexer.analyse_text('''/* */ if 1 > 0 then do diff --git a/tests/test_modeline.py b/tests/test_modeline.py new file mode 100644 index 00000000..efe038df --- /dev/null +++ b/tests/test_modeline.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +""" + Tests for the vim modeline feature + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from __future__ import print_function + +from pygments import modeline + + +def test_lexer_classes(): + def verify(buf): + assert modeline.get_filetype_from_buffer(buf) == 'python' + + for buf in [ + 'vi: ft=python' + '\n' * 8, + 'vi: ft=python' + '\n' * 8, + '\n\n\n\nvi=8: syntax=python' + '\n' * 8, + '\n' * 8 + 'ex: filetype=python', + '\n' * 8 + 'vim: some,other,syn=python\n\n\n\n' + ]: + yield verify, buf diff --git a/tests/test_objectiveclexer.py b/tests/test_objectiveclexer.py index 90bd680f..aee7db66 100644 --- a/tests/test_objectiveclexer.py +++ b/tests/test_objectiveclexer.py @@ -3,7 +3,7 @@ Basic CLexer Test ~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_perllexer.py b/tests/test_perllexer.py index 26b2d0a7..102f0a9f 100644 --- a/tests/test_perllexer.py +++ b/tests/test_perllexer.py @@ -3,14 +3,14 @@ Pygments regex lexer tests ~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import time import unittest -from pygments.token import String +from pygments.token import Keyword, Name, String, Text from pygments.lexers.perl import PerlLexer @@ -135,3 +135,23 @@ class RunawayRegexTest(unittest.TestCase): def test_substitution_with_parenthesis(self): self.assert_single_token(r's(aaa)', String.Regex) self.assert_fast_tokenization('s(' + '\\'*999) + + ### Namespaces/modules + + def test_package_statement(self): + self.assert_tokens(['package', ' ', 'Foo'], [Keyword, Text, Name.Namespace]) + self.assert_tokens(['package', ' ', 'Foo::Bar'], [Keyword, Text, Name.Namespace]) + + def test_use_statement(self): + self.assert_tokens(['use', ' ', 'Foo'], [Keyword, Text, Name.Namespace]) + self.assert_tokens(['use', ' ', 'Foo::Bar'], [Keyword, Text, Name.Namespace]) + + def test_no_statement(self): + self.assert_tokens(['no', ' ', 'Foo'], [Keyword, Text, Name.Namespace]) + self.assert_tokens(['no', ' ', 'Foo::Bar'], [Keyword, Text, Name.Namespace]) + + def test_require_statement(self): + self.assert_tokens(['require', ' ', 'Foo'], [Keyword, Text, Name.Namespace]) + self.assert_tokens(['require', ' ', 'Foo::Bar'], [Keyword, Text, Name.Namespace]) + self.assert_tokens(['require', ' ', '"Foo/Bar.pm"'], [Keyword, Text, String]) + diff --git a/tests/test_php.py b/tests/test_php.py new file mode 100644 index 00000000..b4117381 --- /dev/null +++ b/tests/test_php.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" + PHP Tests + ~~~~~~~~~ + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest + +from pygments.lexers import PhpLexer +from pygments.token import Token + + +class PhpTest(unittest.TestCase): + def setUp(self): + self.lexer = PhpLexer() + + def testStringEscapingRun(self): + fragment = '<?php $x="{\\""; ?>\n' + tokens = [ + (Token.Comment.Preproc, '<?php'), + (Token.Text, ' '), + (Token.Name.Variable, '$x'), + (Token.Operator, '='), + (Token.Literal.String.Double, '"'), + (Token.Literal.String.Double, '{'), + (Token.Literal.String.Escape, '\\"'), + (Token.Literal.String.Double, '"'), + (Token.Punctuation, ';'), + (Token.Text, ' '), + (Token.Comment.Preproc, '?>'), + (Token.Other, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_praat.py b/tests/test_praat.py new file mode 100644 index 00000000..1ca97d1e --- /dev/null +++ b/tests/test_praat.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +""" + Praat lexer tests + ~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest + +from pygments.token import Token +from pygments.lexers import PraatLexer + +class PraatTest(unittest.TestCase): + + def setUp(self): + self.lexer = PraatLexer() + self.maxDiff = None + + def testNumericAssignment(self): + fragment = u'var = -15e4\n' + tokens = [ + (Token.Text, u'var'), + (Token.Text, u' '), + (Token.Operator, u'='), + (Token.Text, u' '), + (Token.Operator, u'-'), + (Token.Literal.Number, u'15e4'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testStringAssignment(self): + fragment = u'var$ = "foo"\n' + tokens = [ + (Token.Text, u'var$'), + (Token.Text, u' '), + (Token.Operator, u'='), + (Token.Text, u' '), + (Token.Literal.String, u'"'), + (Token.Literal.String, u'foo'), + (Token.Literal.String, u'"'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testStringEscapedQuotes(self): + fragment = u'"it said ""foo"""\n' + tokens = [ + (Token.Literal.String, u'"'), + (Token.Literal.String, u'it said '), + (Token.Literal.String, u'"'), + (Token.Literal.String, u'"'), + (Token.Literal.String, u'foo'), + (Token.Literal.String, u'"'), + (Token.Literal.String, u'"'), + (Token.Literal.String, u'"'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testFunctionCall(self): + fragment = u'selected("Sound", i+(a*b))\n' + tokens = [ + (Token.Name.Function, u'selected'), + (Token.Punctuation, u'('), + (Token.Literal.String, u'"'), + (Token.Literal.String, u'Sound'), + (Token.Literal.String, u'"'), + (Token.Punctuation, u','), + (Token.Text, u' '), + (Token.Text, u'i'), + (Token.Operator, u'+'), + (Token.Text, u'('), + (Token.Text, u'a'), + (Token.Operator, u'*'), + (Token.Text, u'b'), + (Token.Text, u')'), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testBrokenUnquotedString(self): + fragment = u'printline string\n... \'interpolated\' string\n' + tokens = [ + (Token.Keyword, u'printline'), + (Token.Text, u' '), + (Token.Literal.String, u'string'), + (Token.Text, u'\n'), + (Token.Punctuation, u'...'), + (Token.Text, u' '), + (Token.Literal.String.Interpol, u"'"), + (Token.Literal.String.Interpol, u'interpolated'), + (Token.Literal.String.Interpol, u"'"), + (Token.Text, u' '), + (Token.Literal.String, u'string'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testInlinIf(self): + fragment = u'var = if true == 1 then -1 else 0 fi' + tokens = [ + (Token.Text, u'var'), + (Token.Text, u' '), + (Token.Operator, u'='), + (Token.Text, u' '), + (Token.Keyword, u'if'), + (Token.Text, u' '), + (Token.Text, u'true'), + (Token.Text, u' '), + (Token.Operator, u'=='), + (Token.Text, u' '), + (Token.Literal.Number, u'1'), + (Token.Text, u' '), + (Token.Keyword, u'then'), + (Token.Text, u' '), + (Token.Operator, u'-'), + (Token.Literal.Number, u'1'), + (Token.Text, u' '), + (Token.Keyword, u'else'), + (Token.Text, u' '), + (Token.Literal.Number, u'0'), + (Token.Text, u' '), + (Token.Keyword, u'fi'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_properties.py b/tests/test_properties.py new file mode 100644 index 00000000..562778ba --- /dev/null +++ b/tests/test_properties.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +""" + Properties Tests + ~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest + +from pygments.lexers.configs import PropertiesLexer +from pygments.token import Token + + +class PropertiesTest(unittest.TestCase): + def setUp(self): + self.lexer = PropertiesLexer() + + def test_comments(self): + """ + Assures lines lead by either # or ! are recognized as a comment + """ + fragment = '! a comment\n# also a comment\n' + tokens = [ + (Token.Comment, '! a comment'), + (Token.Text, '\n'), + (Token.Comment, '# also a comment'), + (Token.Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def test_leading_whitespace_comments(self): + fragment = ' # comment\n' + tokens = [ + (Token.Text, ' '), + (Token.Comment, '# comment'), + (Token.Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def test_escaped_space_in_key(self): + fragment = 'key = value\n' + tokens = [ + (Token.Name.Attribute, 'key'), + (Token.Text, ' '), + (Token.Operator, '='), + (Token.Text, ' '), + (Token.Literal.String, 'value'), + (Token.Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def test_escaped_space_in_value(self): + fragment = 'key = doubleword\\ value\n' + tokens = [ + (Token.Name.Attribute, 'key'), + (Token.Text, ' '), + (Token.Operator, '='), + (Token.Text, ' '), + (Token.Literal.String, 'doubleword\\ value'), + (Token.Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def test_space_delimited_kv_pair(self): + fragment = 'key value\n' + tokens = [ + (Token.Name.Attribute, 'key'), + (Token.Text, ' '), + (Token.Literal.String, 'value\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def test_just_key(self): + fragment = 'justkey\n' + tokens = [ + (Token.Name.Attribute, 'justkey'), + (Token.Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def test_just_key_with_space(self): + fragment = 'just\\ key\n' + tokens = [ + (Token.Name.Attribute, 'just\\ key'), + (Token.Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_python.py b/tests/test_python.py new file mode 100644 index 00000000..e99687a6 --- /dev/null +++ b/tests/test_python.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +""" + Python Tests + ~~~~~~~~~~~~ + + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest + +from pygments.lexers import PythonLexer, Python3Lexer +from pygments.token import Token + + +class PythonTest(unittest.TestCase): + def setUp(self): + self.lexer = PythonLexer() + + def test_cls_builtin(self): + """ + Tests that a cls token gets interpreted as a Token.Name.Builtin.Pseudo + + """ + fragment = 'class TestClass():\n @classmethod\n def hello(cls):\n pass\n' + tokens = [ + (Token.Keyword, 'class'), + (Token.Text, ' '), + (Token.Name.Class, 'TestClass'), + (Token.Punctuation, '('), + (Token.Punctuation, ')'), + (Token.Punctuation, ':'), + (Token.Text, '\n'), + (Token.Text, ' '), + (Token.Name.Decorator, '@classmethod'), + (Token.Text, '\n'), + (Token.Text, ' '), + (Token.Keyword, 'def'), + (Token.Text, ' '), + (Token.Name.Function, 'hello'), + (Token.Punctuation, '('), + (Token.Name.Builtin.Pseudo, 'cls'), + (Token.Punctuation, ')'), + (Token.Punctuation, ':'), + (Token.Text, '\n'), + (Token.Text, ' '), + (Token.Keyword, 'pass'), + (Token.Text, '\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + +class Python3Test(unittest.TestCase): + def setUp(self): + self.lexer = Python3Lexer() + + def testNeedsName(self): + """ + Tests that '@' is recognized as an Operator + """ + fragment = u'S = (H @ beta - r).T @ inv(H @ V @ H.T) @ (H @ beta - r)\n' + tokens = [ + (Token.Name, u'S'), + (Token.Text, u' '), + (Token.Operator, u'='), + (Token.Text, u' '), + (Token.Punctuation, u'('), + (Token.Name, u'H'), + (Token.Text, u' '), + (Token.Operator, u'@'), + (Token.Text, u' '), + (Token.Name, u'beta'), + (Token.Text, u' '), + (Token.Operator, u'-'), + (Token.Text, u' '), + (Token.Name, u'r'), + (Token.Punctuation, u')'), + (Token.Operator, u'.'), + (Token.Name, u'T'), + (Token.Text, u' '), + (Token.Operator, u'@'), + (Token.Text, u' '), + (Token.Name, u'inv'), + (Token.Punctuation, u'('), + (Token.Name, u'H'), + (Token.Text, u' '), + (Token.Operator, u'@'), + (Token.Text, u' '), + (Token.Name, u'V'), + (Token.Text, u' '), + (Token.Operator, u'@'), + (Token.Text, u' '), + (Token.Name, u'H'), + (Token.Operator, u'.'), + (Token.Name, u'T'), + (Token.Punctuation, u')'), + (Token.Text, u' '), + (Token.Operator, u'@'), + (Token.Text, u' '), + (Token.Punctuation, u'('), + (Token.Name, u'H'), + (Token.Text, u' '), + (Token.Operator, u'@'), + (Token.Text, u' '), + (Token.Name, u'beta'), + (Token.Text, u' '), + (Token.Operator, u'-'), + (Token.Text, u' '), + (Token.Name, u'r'), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_qbasiclexer.py b/tests/test_qbasiclexer.py index 8b790cee..0ea221a1 100644 --- a/tests/test_qbasiclexer.py +++ b/tests/test_qbasiclexer.py @@ -3,7 +3,7 @@ Tests for QBasic ~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_regexlexer.py b/tests/test_regexlexer.py index eb25be61..d919a950 100644 --- a/tests/test_regexlexer.py +++ b/tests/test_regexlexer.py @@ -3,7 +3,7 @@ Pygments regex lexer tests ~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_regexopt.py b/tests/test_regexopt.py index dd56a446..5cfb62a3 100644 --- a/tests/test_regexopt.py +++ b/tests/test_regexopt.py @@ -3,7 +3,7 @@ Tests for pygments.regexopt ~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -46,6 +46,7 @@ class RegexOptTestCase(unittest.TestCase): random.randint(1, len(kwlist) - 1)) no_match = set(kwlist) - set(to_match) rex = re.compile(regex_opt(to_match)) + self.assertEqual(rex.groups, 1) for w in to_match: self.assertTrue(rex.match(w)) for w in no_match: @@ -74,3 +75,36 @@ class RegexOptTestCase(unittest.TestCase): rex = re.compile(opt) m = rex.match('abfoo') self.assertEqual(5, m.end()) + + def test_different_length_grouping(self): + opt = regex_opt(('a', 'xyz')) + print(opt) + rex = re.compile(opt) + self.assertTrue(rex.match('a')) + self.assertTrue(rex.match('xyz')) + self.assertFalse(rex.match('b')) + self.assertEqual(1, rex.groups) + + def test_same_length_grouping(self): + opt = regex_opt(('a', 'b')) + print(opt) + rex = re.compile(opt) + self.assertTrue(rex.match('a')) + self.assertTrue(rex.match('b')) + self.assertFalse(rex.match('x')) + + self.assertEqual(1, rex.groups) + groups = rex.match('a').groups() + self.assertEqual(('a',), groups) + + def test_same_length_suffix_grouping(self): + opt = regex_opt(('a', 'b'), suffix='(m)') + print(opt) + rex = re.compile(opt) + self.assertTrue(rex.match('am')) + self.assertTrue(rex.match('bm')) + self.assertFalse(rex.match('xm')) + self.assertFalse(rex.match('ax')) + self.assertEqual(2, rex.groups) + groups = rex.match('am').groups() + self.assertEqual(('a', 'm'), groups) diff --git a/tests/test_rtf_formatter.py b/tests/test_rtf_formatter.py index 25784743..756c03a9 100644 --- a/tests/test_rtf_formatter.py +++ b/tests/test_rtf_formatter.py @@ -3,7 +3,7 @@ Pygments RTF formatter tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_ruby.py b/tests/test_ruby.py index ab210bad..b7d4110a 100644 --- a/tests/test_ruby.py +++ b/tests/test_ruby.py @@ -3,7 +3,7 @@ Basic RubyLexer Test ~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_shell.py b/tests/test_shell.py index 4eb5a15a..e283793e 100644 --- a/tests/test_shell.py +++ b/tests/test_shell.py @@ -3,14 +3,14 @@ Basic Shell Tests ~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import unittest from pygments.token import Token -from pygments.lexers import BashLexer +from pygments.lexers import BashLexer, BashSessionLexer class BashTest(unittest.TestCase): @@ -87,3 +87,56 @@ class BashTest(unittest.TestCase): (Token.Text, u'\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testArrayNums(self): + fragment = u'a=(1 2 3)\n' + tokens = [ + (Token.Name.Variable, u'a'), + (Token.Operator, u'='), + (Token.Operator, u'('), + (Token.Literal.Number, u'1'), + (Token.Text, u' '), + (Token.Literal.Number, u'2'), + (Token.Text, u' '), + (Token.Literal.Number, u'3'), + (Token.Operator, u')'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + + def testEndOfLineNums(self): + fragment = u'a=1\nb=2 # comment\n' + tokens = [ + (Token.Name.Variable, u'a'), + (Token.Operator, u'='), + (Token.Literal.Number, u'1'), + (Token.Text, u'\n'), + (Token.Name.Variable, u'b'), + (Token.Operator, u'='), + (Token.Literal.Number, u'2'), + (Token.Text, u' '), + (Token.Comment.Single, u'# comment\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + +class BashSessionTest(unittest.TestCase): + + def setUp(self): + self.lexer = BashSessionLexer() + self.maxDiff = None + + def testNeedsName(self): + fragment = u'$ echo \\\nhi\nhi\n' + tokens = [ + (Token.Text, u''), + (Token.Generic.Prompt, u'$'), + (Token.Text, u' '), + (Token.Name.Builtin, u'echo'), + (Token.Text, u' '), + (Token.Literal.String.Escape, u'\\\n'), + (Token.Text, u'hi'), + (Token.Text, u'\n'), + (Token.Generic.Output, u'hi\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + diff --git a/tests/test_smarty.py b/tests/test_smarty.py index 450e4e6b..e1e079d9 100644 --- a/tests/test_smarty.py +++ b/tests/test_smarty.py @@ -3,7 +3,7 @@ Basic SmartyLexer Test ~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_sql.py b/tests/test_sql.py new file mode 100644 index 00000000..c5f5c758 --- /dev/null +++ b/tests/test_sql.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +""" + Pygments SQL lexers tests + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import unittest + +from pygments.lexers.sql import TransactSqlLexer +from pygments.token import Comment, Name, Number, Punctuation, Whitespace + + +class TransactSqlLexerTest(unittest.TestCase): + + def setUp(self): + self.lexer = TransactSqlLexer() + + def _assertAreTokensOfType(self, examples, expected_token_type): + for test_number, example in enumerate(examples.split(), 1): + token_count = 0 + for token_type, token_value in self.lexer.get_tokens(example): + if token_type != Whitespace: + token_count += 1 + self.assertEqual( + token_type, expected_token_type, + 'token_type #%d for %s is be %s but must be %s' % + (test_number, token_value, token_type, expected_token_type)) + self.assertEqual( + token_count, 1, + '%s must yield exactly 1 token instead of %d' % + (example, token_count)) + + def _assertTokensMatch(self, text, expected_tokens_without_trailing_newline): + actual_tokens = tuple(self.lexer.get_tokens(text)) + if (len(actual_tokens) >= 1) and (actual_tokens[-1] == (Whitespace, '\n')): + actual_tokens = tuple(actual_tokens[:-1]) + self.assertEqual( + expected_tokens_without_trailing_newline, actual_tokens, + 'text must yield expected tokens: %s' % text) + + def test_can_lex_float(self): + self._assertAreTokensOfType( + '1. 1.e1 .1 1.2 1.2e3 1.2e+3 1.2e-3 1e2', Number.Float) + self._assertTokensMatch( + '1e2.1e2', + ((Number.Float, '1e2'), (Number.Float, '.1e2')) + ) + + def test_can_reject_almost_float(self): + self._assertTokensMatch( + '.e1', + ((Punctuation, '.'), (Name, 'e1'))) + + def test_can_lex_integer(self): + self._assertAreTokensOfType( + '1 23 456', Number.Integer) + + def test_can_lex_names(self): + self._assertAreTokensOfType( + u'thingy thingy123 _thingy _ _123 Ähnliches Müll #temp1 ##temp2', Name) + + def test_can_lex_comments(self): + self._assertTokensMatch('--\n', ((Comment.Single, '--\n'),)) + self._assertTokensMatch('/**/', ( + (Comment.Multiline, '/*'), (Comment.Multiline, '*/') + )) + self._assertTokensMatch('/*/**/*/', ( + (Comment.Multiline, '/*'), + (Comment.Multiline, '/*'), + (Comment.Multiline, '*/'), + (Comment.Multiline, '*/'), + )) diff --git a/tests/test_string_asserts.py b/tests/test_string_asserts.py index ba7b37fa..5e9e5617 100644 --- a/tests/test_string_asserts.py +++ b/tests/test_string_asserts.py @@ -3,7 +3,7 @@ Pygments string assert utility tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_terminal_formatter.py b/tests/test_terminal_formatter.py index 07337cd5..ee0ac380 100644 --- a/tests/test_terminal_formatter.py +++ b/tests/test_terminal_formatter.py @@ -3,7 +3,7 @@ Pygments terminal formatter tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -14,7 +14,13 @@ import re from pygments.util import StringIO from pygments.lexers.sql import PlPgsqlLexer -from pygments.formatters import TerminalFormatter +from pygments.formatters import TerminalFormatter, Terminal256Formatter, \ + HtmlFormatter, LatexFormatter + +from pygments.style import Style +from pygments.token import Token +from pygments.lexers import Python3Lexer +from pygments import highlight DEMO_TEXT = '''\ -- comment @@ -26,9 +32,11 @@ DEMO_TOKENS = list(DEMO_LEXER().get_tokens(DEMO_TEXT)) ANSI_RE = re.compile(r'\x1b[\w\W]*?m') + def strip_ansi(x): return ANSI_RE.sub('', x) + class TerminalFormatterTest(unittest.TestCase): def test_reasonable_output(self): out = StringIO() @@ -49,3 +57,46 @@ class TerminalFormatterTest(unittest.TestCase): for a, b in zip(DEMO_TEXT.splitlines(), plain.splitlines()): self.assertTrue(a in b) + + +class MyStyle(Style): + styles = { + Token.Comment: '#ansidarkgray', + Token.String: '#ansiblue bg:#ansidarkred', + Token.Number: '#ansigreen bg:#ansidarkgreen', + Token.Number.Hex: '#ansidarkgreen bg:#ansired', + } + + +class Terminal256FormatterTest(unittest.TestCase): + code = ''' +# this should be a comment +print("Hello World") +async def function(a,b,c, *d, **kwarg:Bool)->Bool: + pass + return 123, 0xb3e3 + +''' + + def test_style_html(self): + style = HtmlFormatter(style=MyStyle).get_style_defs() + self.assertTrue('#555555' in style, + "ansigray for comment not html css style") + + def test_others_work(self): + """check other formatters don't crash""" + highlight(self.code, Python3Lexer(), LatexFormatter(style=MyStyle)) + highlight(self.code, Python3Lexer(), HtmlFormatter(style=MyStyle)) + + def test_256esc_seq(self): + """ + test that a few escape sequences are actualy used when using #ansi<> color codes + """ + def termtest(x): + return highlight(x, Python3Lexer(), + Terminal256Formatter(style=MyStyle)) + + self.assertTrue('32;41' in termtest('0x123')) + self.assertTrue('32;42' in termtest('123')) + self.assertTrue('30;01' in termtest('#comment')) + self.assertTrue('34;41' in termtest('"String"')) diff --git a/tests/test_textfmts.py b/tests/test_textfmts.py index d355ab68..453dd61f 100644 --- a/tests/test_textfmts.py +++ b/tests/test_textfmts.py @@ -3,7 +3,7 @@ Basic Tests for textfmts ~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_token.py b/tests/test_token.py index 0c6b02bf..94522373 100644 --- a/tests/test_token.py +++ b/tests/test_token.py @@ -3,7 +3,7 @@ Test suite for the token module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_unistring.py b/tests/test_unistring.py index a414347c..c56b68c7 100644 --- a/tests/test_unistring.py +++ b/tests/test_unistring.py @@ -3,7 +3,7 @@ Test suite for the unistring module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_using_api.py b/tests/test_using_api.py index 16d865e6..7517ce7d 100644 --- a/tests/test_using_api.py +++ b/tests/test_using_api.py @@ -3,7 +3,7 @@ Pygments tests for using() ~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_util.py b/tests/test_util.py index 720b384a..cdb58b3f 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -3,7 +3,7 @@ Test suite for the util module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_whiley.py b/tests/test_whiley.py new file mode 100644 index 00000000..f447ffec --- /dev/null +++ b/tests/test_whiley.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" + Whiley Test + ~~~~~~~~~~~ + + :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest + +from pygments.lexers import WhileyLexer +from pygments.token import Token + + +class WhileyTest(unittest.TestCase): + def setUp(self): + self.lexer = WhileyLexer() + + def testWhileyOperator(self): + fragment = u'123 \u2200 x\n' + tokens = [ + (Token.Literal.Number.Integer, u'123'), + (Token.Text, u' '), + (Token.Operator, u'\u2200'), + (Token.Text, u' '), + (Token.Name, u'x'), + (Token.Text, u'\n'), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) |